Explicitly mark destructors with override (#13212 )

Summary: I saw these compiler warnings while preparing for the 9.10 release: ```cpp '~CompactOnDeletionCollectorFactory' overrides a destructor but is not marked 'override' [-Werror,-Wsuggest-destructor-override] '~CompactForTieringCollectorFactory' overrides a destructor but is not marked 'override' [-Werror,-Wsuggest-destructor-override] ``` This code is from a while ago so I assume that this CI check has been failing for quite some time. We should still clean this up to avoid confusion in the future. Pull Request resolved: https://github.com/facebook/rocksdb/pull/13212 Test Plan: Existing CI checks should pass, and we should not see this CI check failure the next time we try to make a release/patch. Reviewed By: jaykorean Differential Revision: D67287794 Pulled By: archang19 fbshipit-source-id: a11230a919c0b7ef21a7219bf05f567d3d44b2d1
Add files for 9.10 release
2026-11-07 19:42:37 -05:00 · 2024-12-16 12:45:14 -08:00 · 2024-12-13 10:51:47 -08:00
366 changed files with 4211 additions and 6868 deletions
@@ -0,0 +1,905 @@
+version: 2.1
+
+orbs:
+  win: circleci/windows@5.0.0
+
+commands:
+  install-cmake-on-macos:
+    steps:
+      - run:
+          name: Install cmake on macos
+          command: |
+            HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
+
+  install-jdk8-on-macos:
+    steps:
+      - run:
+          name: Install JDK 8 on macos
+          command: |
+            HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica
+            HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8
+
+  increase-max-open-files-on-macos:
+    steps:
+      - run:
+          name: Increase max open files
+          command: |
+            sudo sysctl -w kern.maxfiles=1048576
+            sudo sysctl -w kern.maxfilesperproc=1048576
+            sudo launchctl limit maxfiles 1048576
+
+  pre-steps:
+    steps:
+      - checkout
+      - run:
+          name: Setup Environment Variables
+          command: |
+            echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
+            echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
+            echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
+            echo "export GTEST_COLOR=1" >> $BASH_ENV
+            echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV
+            echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV
+            echo "export ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> $BASH_ENV
+            echo "export BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> $BASH_ENV
+            echo "export SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> $BASH_ENV
+            echo "export LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> $BASH_ENV
+            echo "export ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> $BASH_ENV
+
+  windows-build-steps:
+    steps:
+      - checkout
+      - run:
+          name: "Install thirdparty dependencies"
+          command: |
+            echo "Installing CMake..."
+            choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' -y
+            choco install liberica8jdk -y
+            mkdir $Env:THIRDPARTY_HOME
+            cd $Env:THIRDPARTY_HOME
+            echo "Building Snappy dependency..."
+            curl https://github.com/google/snappy/archive/refs/tags/1.1.8.zip -O snappy-1.1.8.zip
+            unzip -q snappy-1.1.8.zip
+            cd snappy-1.1.8
+            mkdir build
+            cd build
+            & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" ..
+            msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+      - run:
+          name: "Build RocksDB"
+          command: |
+            $env:Path = $env:JAVA_HOME + ";" + $env:Path
+            mkdir build
+            cd build
+            & $Env:CMAKE_BIN -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DJNI=1 ..
+            cd ..
+            echo "Building with VS version: $Env:CMAKE_GENERATOR"
+            msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+      - run:
+          name: "Test RocksDB"
+          shell: powershell.exe
+          command: |
+            build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
+      - run:
+          name: "Test RocksJava"
+          command: |
+            cd build\java
+            & $Env:CTEST_BIN -C Debug -j 16
+  pre-steps-macos:
+      steps:
+        - pre-steps
+
+  post-steps:
+    steps:
+      - store_test_results: # store test result if there's any
+          path: /tmp/test-results
+      - store_artifacts: # store LOG for debugging if there's any
+          path: LOG
+      - run: # on fail, compress Test Logs for diagnosing the issue
+          name: Compress Test Logs
+          command: tar -cvzf t.tar.gz t
+          when: on_fail
+      - store_artifacts: # on fail, store Test Logs for diagnosing the issue
+          path: t.tar.gz
+          destination: test_logs
+          when: on_fail
+      - run: # store core dumps if there's any
+          command: |
+            mkdir -p /tmp/core_dumps
+            cp core.* /tmp/core_dumps
+          when: on_fail
+      - store_artifacts:
+          path: /tmp/core_dumps
+          when: on_fail
+
+  post-pmd-steps:
+    steps:
+      - store_artifacts:
+          path: /home/circleci/project/java/target/pmd.xml
+          when: on_fail
+      - store_artifacts:
+          path: /home/circleci/project/java/target/site
+          when: on_fail
+
+  upgrade-cmake:
+    steps:
+      - run:
+          name: Upgrade cmake
+          command: |
+            sudo apt remove --purge cmake
+            sudo snap install cmake --classic
+
+  install-gflags:
+    steps:
+      - run:
+          name: Install gflags
+          command: |
+            sudo apt-get update -y && sudo apt-get install -y libgflags-dev
+
+  install-gflags-on-macos:
+    steps:
+      - run:
+          name: Install gflags on macos
+          command: |
+            HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
+
+  install-maven:
+    steps:
+      - run:
+          name: Install Maven
+          command: |
+            wget --no-check-certificate https://dlcdn.apache.org/maven/maven-3/3.9.6/binaries/apache-maven-3.9.6-bin.tar.gz
+            tar zxf apache-maven-3.9.6-bin.tar.gz
+            echo "export M2_HOME=$(pwd)/apache-maven-3.9.6" >> $BASH_ENV
+            echo 'export PATH=$M2_HOME/bin:$PATH' >> $BASH_ENV
+
+  setup-folly:
+    steps:
+      - run:
+          name: Checkout folly sources
+          command: |
+            make checkout_folly
+
+  build-folly:
+    steps:
+      - run:
+          name: Build folly and dependencies
+          command: |
+            make build_folly
+
+  build-for-benchmarks:
+    steps:
+      - pre-steps
+      - run:
+          name: "Linux build for benchmarks"
+          command: #sized for the resource-class rocksdb-benchmark-sys1
+            make V=1 J=8 -j8 release
+
+  perform-benchmarks:
+    steps:
+      - run:
+          name: "Test low-variance benchmarks"
+          command: ./tools/benchmark_ci.py --db_dir /tmp/rocksdb-benchmark-datadir --output_dir /tmp/benchmark-results --num_keys 20000000
+          environment:
+            LD_LIBRARY_PATH: /usr/local/lib
+            # How long to run parts of the test(s)
+            DURATION_RO: 300
+            DURATION_RW: 500
+            # Keep threads within physical capacity of server (much lower than default)
+            NUM_THREADS: 1
+            MAX_BACKGROUND_JOBS: 4
+            # Don't run a couple of "optional" initial tests
+            CI_TESTS_ONLY: "true"
+            # Reduce configured size of levels to ensure more levels in the leveled compaction LSM tree
+            WRITE_BUFFER_SIZE_MB: 16
+            TARGET_FILE_SIZE_BASE_MB: 16
+            MAX_BYTES_FOR_LEVEL_BASE_MB: 64
+            # The benchmark host has 32GB memory
+            # The following values are tailored to work with that
+            # Note, tests may not exercise the targeted issues if the memory is increased on new test hosts.
+            COMPRESSION_TYPE: "none"
+            CACHE_INDEX_AND_FILTER_BLOCKS: 1
+            MIN_LEVEL_TO_COMPRESS: 3
+            CACHE_SIZE_MB: 10240
+            MB_WRITE_PER_SEC: 2
+
+  post-benchmarks:
+    steps:
+      - store_artifacts: # store the benchmark output
+          path: /tmp/benchmark-results
+          destination: test_logs
+      - run:
+          name: Send benchmark report to visualisation
+          command: |
+            set +e
+            set +o pipefail
+            ./build_tools/benchmark_log_tool.py --tsvfile /tmp/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc
+            true
+
+executors:
+  linux-docker:
+    docker:
+      # The image configuration is build_tools/ubuntu20_image/Dockerfile
+      # To update and build the image:
+      #  $ cd build_tools/ubuntu20_image
+      #  $ docker build -t zjay437/rocksdb:0.5 .
+      #  $ docker push zjay437/rocksdb:0.5
+      # `zjay437` is the account name for zjay@meta.com which readwrite token is shared internally. To login:
+      #  $ docker login --username zjay437
+      # Or please feel free to change it to your docker hub account for hosting the image, meta employee should already have the account and able to login with SSO.
+      # To avoid impacting the existing CI runs, please bump the version every time creating a new image
+      # to run the CI image environment locally:
+      #  $ docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -it zjay437/rocksdb:0.5 bash
+      # option `--cap-add=SYS_PTRACE --security-opt seccomp=unconfined` is used to enable gdb to attach an existing process
+      - image: zjay437/rocksdb:0.6
+  linux-java-docker:
+    docker:
+      # This is the Docker Image used for building RocksJava releases, see: https://github.com/evolvedbinary/docker-rocksjava
+      - image: evolvedbinary/rocksjava:centos6_x64-be
+
+jobs:
+  build-macos:
+    macos:
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
+    environment:
+      ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc cause env_test hang, disable it for now
+    steps:
+      - increase-max-open-files-on-macos
+      - install-gflags-on-macos
+      - pre-steps-macos
+      - run: ulimit -S -n `ulimit -H -n` && make V=1 J=16 -j16 all
+      - post-steps
+
+  build-macos-cmake:
+    macos:
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
+    parameters:
+      run_even_tests:
+        description: run even or odd tests, used to split tests to 2 groups
+        type: boolean
+        default: true
+    steps:
+      - increase-max-open-files-on-macos
+      - install-cmake-on-macos
+      - install-gflags-on-macos
+      - pre-steps-macos
+      - run:
+          name: "cmake generate project file"
+          command: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
+      - run:
+          name: "Build tests"
+          command: cd build && make V=1 -j16
+      - when:
+          condition: << parameters.run_even_tests >>
+          steps:
+            - run:
+                name: "Run even tests"
+                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2
+      - when:
+          condition:
+            not: << parameters.run_even_tests >>
+          steps:
+            - run:
+                name: "Run odd tests"
+                command: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2
+      - post-steps
+
+  build-linux:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: make V=1 J=32 -j32 check
+      - post-steps
+
+  build-linux-encrypted_env-no_compression:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check
+      - run: |
+          ./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression
+      - post-steps
+
+  build-linux-static_lib-alt_namespace-status_checked:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check
+      - post-steps
+
+  build-linux-release:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: make V=1 -j32 LIB_MODE=shared release
+      - run: ls librocksdb.so # ensure shared lib built
+      - run: ./db_stress --version # ensure with gflags
+      - run: make clean
+      - run: make V=1 -j32 release
+      - run: ls librocksdb.a # ensure static lib built
+      - run: ./db_stress --version # ensure with gflags
+      - run: make clean
+      - run: apt-get remove -y libgflags-dev
+      - run: make V=1 -j32 LIB_MODE=shared release
+      - run: ls librocksdb.so # ensure shared lib built
+      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
+      - run: make clean
+      - run: make V=1 -j32 release
+      - run: ls librocksdb.a # ensure static lib built
+      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
+      - post-steps
+
+  build-linux-release-rtti:
+    executor: linux-docker
+    resource_class: xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
+      - run: ./db_stress --version # ensure with gflags
+      - run: make clean
+      - run: apt-get remove -y libgflags-dev
+      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
+      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
+
+  build-linux-clang-no_test_run:
+    executor: linux-docker
+    resource_class: xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all
+      - post-steps
+
+  build-linux-clang10-asan:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out
+      - post-steps
+
+  build-linux-clang10-mini-tsan:
+    executor: linux-docker
+    resource_class: 2xlarge+
+    steps:
+      - pre-steps
+      - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check
+      - post-steps
+
+  build-linux-clang10-ubsan:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check # aligned new doesn't work for reason we haven't figured out
+      - post-steps
+
+  build-linux-valgrind:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: PORTABLE=1 make V=1 -j32 valgrind_test
+      - post-steps
+
+  build-linux-clang10-clang-analyze:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
+      - post-steps
+      - run:
+          name: "compress test report"
+          command: tar -cvzf scan_build_report.tar.gz scan_build_report
+          when: on_fail
+      - store_artifacts:
+          path: scan_build_report.tar.gz
+          destination: scan_build_report
+          when: on_fail
+
+  build-linux-runner:
+    machine: true
+    resource_class: facebook/rocksdb-benchmark-sys1
+    steps:
+      - pre-steps
+      - run:
+          name: "Checked Linux build (Runner)"
+          command: make V=1 J=8 -j8 check
+          environment:
+            LD_LIBRARY_PATH: /usr/local/lib
+      - post-steps
+
+  build-linux-cmake-with-folly:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - setup-folly
+      - build-folly
+      - run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)
+      - post-steps
+
+  build-linux-cmake-with-folly-lite-no-test:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - setup-folly
+      - run: (mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)
+      - post-steps
+
+  build-linux-cmake-with-benchmark:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20
+      - post-steps
+
+  build-linux-unity-and-headers:
+    docker: # executor type
+      - image: gcc:latest
+    environment:
+      EXTRA_CXXFLAGS: -mno-avx512f # Warnings-as-error in avx512fintrin.h, would be used on newer hardware
+    resource_class: large
+    steps:
+      - checkout # check out the code in the project directory
+      - run: apt-get update -y && apt-get install -y libgflags-dev
+      - run:
+          name: "Unity build"
+          command: make V=1 -j8 unity_test
+          no_output_timeout: 20m
+      - run: make V=1 -j8 -k check-headers # could be moved to a different build
+      - post-steps
+
+  build-linux-gcc-7-with-folly:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - setup-folly
+      - build-folly
+      - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures
+      - post-steps
+
+  build-linux-gcc-7-with-folly-lite-no-test:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - setup-folly
+      - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all
+      - post-steps
+
+  build-linux-gcc-8-no_test_run:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all
+      - post-steps
+
+  build-linux-cmake-with-folly-coroutines:
+    executor: linux-docker
+    resource_class: 2xlarge
+    environment:
+      CC: gcc-10
+      CXX: g++-10
+    steps:
+      - pre-steps
+      - setup-folly
+      - build-folly
+      - run: (mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)
+      - post-steps
+
+  build-linux-gcc-10-cxx20-no_test_run:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all
+      - post-steps
+
+  build-linux-gcc-11-no_test_run:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench # TODO: LIB_MODE only to work around unresolved linker failures
+      - post-steps
+
+  build-linux-clang-13-no_test_run:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench
+      - post-steps
+
+  # Ensure ASAN+UBSAN with folly, and full testsuite with clang 13
+  build-linux-clang-13-asan-ubsan-with-folly:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - setup-folly
+      - build-folly
+      - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures
+      - post-steps
+
+  # This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing.
+  build-linux-run-microbench:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run: DEBUG_LEVEL=0 make -j32 run_microbench
+      - post-steps
+
+  build-linux-mini-crashtest:
+    executor: linux-docker
+    resource_class: large
+    steps:
+      - pre-steps
+      - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000' blackbox_crash_test_with_atomic_flush
+      - post-steps
+
+  build-linux-crashtest-tiered-storage-bb:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run:
+          name: "run crashtest"
+          command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800' blackbox_crash_test_with_tiered_storage
+          no_output_timeout: 100m
+      - post-steps
+
+  build-linux-crashtest-tiered-storage-wb:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run:
+          name: "run crashtest"
+          command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800' whitebox_crash_test_with_tiered_storage
+          no_output_timeout: 100m
+      - post-steps
+
+  build-windows-vs2022-avx2:
+    executor:
+      name: win/server-2022
+      size: 2xlarge
+    environment:
+      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
+      CMAKE_HOME: C:/Program Files/CMake
+      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
+      JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8
+      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
+      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
+      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
+      CMAKE_GENERATOR: Visual Studio 17 2022
+      CMAKE_PORTABLE: AVX2
+    steps:
+      - windows-build-steps
+
+  build-windows-vs2022:
+    executor:
+      name: win/server-2022
+      size: 2xlarge
+    environment:
+      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
+      CMAKE_HOME: C:/Program Files/CMake
+      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
+      JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8
+      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
+      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
+      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
+      CMAKE_GENERATOR: Visual Studio 17 2022
+      CMAKE_PORTABLE: 1
+    steps:
+      - windows-build-steps
+
+  build-windows-vs2019:
+    executor:
+      name: win/server-2019
+      size: 2xlarge
+    environment:
+      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
+      CMAKE_HOME: C:/Program Files/CMake
+      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
+      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
+      JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8
+      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.8
+      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.8;C:/Users/circleci/thirdparty/snappy-1.1.8/build
+      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.8/build/Debug/snappy.lib
+      CMAKE_GENERATOR: Visual Studio 16 2019
+      CMAKE_PORTABLE: 1
+    steps:
+      - windows-build-steps
+
+  build-linux-java:
+    executor: linux-java-docker
+    resource_class: large
+    steps:
+      - pre-steps
+      - run:
+          name: "Set Java Environment"
+          command: |
+            echo "JAVA_HOME=${JAVA_HOME}"
+            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
+            which java && java -version
+            which javac && javac -version
+      - run:
+          name: "Test RocksDBJava"
+          command: scl enable devtoolset-7 'make V=1 J=8 -j8 jtest'
+      - post-steps
+
+  build-linux-java-pmd:
+    executor: linux-java-docker
+    resource_class: large
+    steps:
+      - pre-steps
+      - run:
+          name: "Set Java Environment"
+          command: |
+            echo "JAVA_HOME=${JAVA_HOME}"
+            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
+            which java && java -version
+            which javac && javac -version
+      - install-maven
+      - run:
+          name: "PMD RocksDBJava"
+          command: scl enable devtoolset-7 'make V=1 J=8 -j8 jpmd'
+      - post-pmd-steps
+
+  build-linux-java-static:
+    executor: linux-java-docker
+    resource_class: large
+    steps:
+      - pre-steps
+      - run:
+          name: "Set Java Environment"
+          command: |
+            echo "JAVA_HOME=${JAVA_HOME}"
+            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
+            which java && java -version
+            which javac && javac -version
+      - run:
+          name: "Build RocksDBJava Static Library"
+          command: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic'
+      - post-steps
+
+  build-macos-java:
+    macos:
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
+    environment:
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
+      ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc causes java 8 crash
+    steps:
+      - increase-max-open-files-on-macos
+      - install-gflags-on-macos
+      - install-jdk8-on-macos
+      - pre-steps-macos
+      - run:
+          name: "Set Java Environment"
+          command: |
+            echo "JAVA_HOME=${JAVA_HOME}"
+            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
+            which java && java -version
+            which javac && javac -version
+      - run:
+          name: "Test RocksDBJava"
+          command: make V=1 J=16 -j16 jtest
+          no_output_timeout: 20m
+      - post-steps
+
+  build-macos-java-static:
+    macos:
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
+    environment:
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
+    steps:
+      - increase-max-open-files-on-macos
+      - install-gflags-on-macos
+      - install-cmake-on-macos
+      - install-jdk8-on-macos
+      - pre-steps-macos
+      - run:
+          name: "Set Java Environment"
+          command: |
+            echo "JAVA_HOME=${JAVA_HOME}"
+            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
+            which java && java -version
+            which javac && javac -version
+      - run:
+          name: "Build RocksDBJava x86 and ARM Static Libraries"
+          command: make V=1 J=16 -j16 rocksdbjavastaticosx
+          no_output_timeout: 20m
+      - post-steps
+
+  build-macos-java-static-universal:
+    macos:
+      xcode: 14.3.1
+    resource_class: macos.m1.medium.gen1
+    environment:
+      JAVA_HOME: /Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home
+    steps:
+      - increase-max-open-files-on-macos
+      - install-gflags-on-macos
+      - install-cmake-on-macos
+      - install-jdk8-on-macos
+      - pre-steps-macos
+      - run:
+          name: "Set Java Environment"
+          command: |
+            echo "JAVA_HOME=${JAVA_HOME}"
+            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
+            which java && java -version
+            which javac && javac -version
+      - run:
+          name: "Build RocksDBJava Universal Binary Static Library"
+          command: make V=1 J=16 -j16 rocksdbjavastaticosx_ub
+          no_output_timeout: 20m
+      - post-steps
+
+  build-examples:
+    executor: linux-docker
+    resource_class: large
+    steps:
+      - pre-steps
+      - run:
+          name: "Build examples"
+          command: |
+            make V=1 -j4 static_lib && cd examples && make V=1 -j4
+      - post-steps
+
+  build-cmake-mingw:
+    executor: linux-docker
+    resource_class: large
+    steps:
+      - pre-steps
+      - run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
+      - run:
+          name: "Build cmake-mingw"
+          command: |
+            export PATH=$JAVA_HOME/bin:$PATH
+            echo "JAVA_HOME=${JAVA_HOME}"
+            which java && java -version
+            which javac && javac -version
+            mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
+      - post-steps
+
+  build-linux-non-shm:
+    executor: linux-docker
+    resource_class: 2xlarge
+    environment:
+      TEST_TMPDIR: /tmp/rocksdb_test_tmp
+    steps:
+      - pre-steps
+      - run: make V=1 -j32 check
+      - post-steps
+
+  build-linux-arm-test-full:
+    machine:
+      image: ubuntu-2004:202111-02
+    resource_class: arm.large
+    steps:
+      - pre-steps
+      - install-gflags
+      - run: make V=1 J=4 -j4 check
+      - post-steps
+
+  build-linux-arm:
+    machine:
+      image: ubuntu-2004:202111-02
+    resource_class: arm.large
+    steps:
+      - pre-steps
+      - install-gflags
+      - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some
+      - post-steps
+
+  build-linux-arm-cmake-no_test_run:
+    machine:
+      image: ubuntu-2004:202111-02
+    resource_class: arm.large
+    environment:
+      JAVA_HOME: /usr/lib/jvm/java-8-openjdk-arm64
+    steps:
+      - pre-steps
+      - install-gflags
+      - run:
+          name: "Set Java Environment"
+          command: |
+            echo "JAVA_HOME=${JAVA_HOME}"
+            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
+            which java && java -version
+            which javac && javac -version
+      - run:
+          name: "Build with cmake"
+          command: |
+            mkdir build
+            cd build
+            cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
+            make -j4
+      - run:
+          name: "Build Java with cmake"
+          command: |
+            rm -rf build
+            mkdir build
+            cd build
+            cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 ..
+            make -j4 rocksdb rocksdbjni
+      - post-steps
+
+  build-format-compatible:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run:
+          name: "test"
+          command: |
+            export TEST_TMPDIR=/dev/shm/rocksdb
+            rm -rf /dev/shm/rocksdb
+            mkdir /dev/shm/rocksdb
+            tools/check_format_compatible.sh
+      - post-steps
+
+  build-fuzzers:
+    executor: linux-docker
+    resource_class: large
+    steps:
+      - pre-steps
+      - run:
+          name: "Build rocksdb lib"
+          command: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
+      - run:
+          name: "Build fuzzers"
+          command: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
+      - post-steps
+
+  benchmark-linux: #use a private Circle CI runner (resource_class) to run the job
+    machine: true
+    resource_class: facebook/rocksdb-benchmark-sys1
+    steps:
+      - build-for-benchmarks
+      - perform-benchmarks
+      - post-benchmarks
+
+workflows: # Only jobs that haven't been successfully migrated to GitHub Actions
+  version: 2
+  jobs-linux-arm:
+    jobs:
+      - build-linux-arm
+  benchmark-linux:
+    triggers:
+      - schedule:
+          cron: "0 * * * *"
+          filters:
+            branches:
+              only:
+                - main
+    jobs:
+      - benchmark-linux
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 9 * * *"
+          filters:
+            branches:
+              only:
+                - main
+    jobs:
+      - build-linux-arm-test-full
@@ -0,0 +1,6 @@
+# Supress UBSAN warnings related to stl_tree.h, e.g.
+# UndefinedBehaviorSanitizer: undefined-behavior /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43 in 
+# /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43:
+# runtime error: upcast of address 0x000001fa8820 with insufficient space for an object of type
+# 'std::_Rb_tree_node<std::pair<const std::__cxx11::basic_string<char>, rocksdb::(anonymous namespace)::LockHoldingInfo> >'
+src:*bits/stl_tree.h
@@ -31,7 +31,7 @@ jobs:
  # ======================== Fast Initial Checks ====================== #
  check-format-and-targets:
    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4.1.0
      with:
@@ -390,9 +390,9 @@ jobs:
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
-    - run: COMPILE_WITH_UBSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check
+    - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check
    - uses: "./.github/actions/post-steps"
-  build-linux-clang13-mini-tsan:
+  build-linux-clang10-mini-tsan:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
      labels: 32-core-ubuntu
@@ -91,7 +91,6 @@ buckifier/__pycache__
 .arcconfig

 compile_commands.json
-meta_gen_cpp_compile_commands.json
 clang-format-diff.py
 .py3/

@@ -214,7 +214,6 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "table/cuckoo/cuckoo_table_builder.cc",
        "table/cuckoo/cuckoo_table_factory.cc",
        "table/cuckoo/cuckoo_table_reader.cc",
-        "table/external_table_reader.cc",
        "table/format.cc",
        "table/get_context.cc",
        "table/iterator.cc",
@@ -835,7 +835,6 @@ set(SOURCES
        table/cuckoo/cuckoo_table_builder.cc
        table/cuckoo/cuckoo_table_factory.cc
        table/cuckoo/cuckoo_table_reader.cc
-        table/external_table_reader.cc
        table/format.cc
        table/get_context.cc
        table/iterator.cc
@@ -1,30 +1,6 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`

-## 9.11.1 (02/19/2025)
-### New Features
-* Added the ability to plug-in a custom table reader implementation. See include/rocksdb/external_table_reader.h for more details.
-
-## 9.11.0 (01/17/2025)
-### New Features
-* Introduce CancelAwaitingJobs() in CompactionService interface which will allow users to implement cancellation of running remote compactions from the primary instance
-* Experimental feature: RocksDB now supports defining secondary indices, which are automatically maintained by the storage engine. Secondary indices provide a new customization point: applications can provide their own by implementing the new `SecondaryIndex` interface. See the `SecondaryIndex` API comments for more details. Note: this feature is currently only available in conjunction with write-committed pessimistic transactions, and `Merge` is not yet supported.
-* Provide a new option `track_and_verify_wals` to track and verify various information about WAL during WAL recovery. This is intended to be a better replacement to `track_and_verify_wals_in_manifest`.
-
-### Public API Changes
-* Add `io_buffer_size` to BackupEngineOptions to enable optimal configuration of IO size
-* Clean up all the references to `random_access_max_buffer_size`, related rules and all the clients wrappers. This option has been officially deprecated in 5.4.0.
-* Add `file_ingestion_nanos` and `file_ingestion_blocking_live_writes_nanos` in PerfContext to observe file ingestions
-* Offer new DB::Open and variants that use `std::unique_ptr<DB>*` output parameters and deprecate the old versions that use `DB**` output parameters.
-* The DB::DeleteFile API is officially deprecated.
-
-### Behavior Changes
-* For leveled compaction, manual compaction (CompactRange()) will be more strict about keeping compaction size under `max_compaction_bytes`. This prevents overly large compactions in some cases (#13306).
-* Experimental tiering options `preclude_last_level_data_seconds` and `preserve_internal_time_seconds` are now mutable with `SetOptions()`. Some changes to handling of these features along with long-lived snapshots and range deletes made this possible.
-
-### Bug Fixes
-* Fix a longstanding major bug with SetOptions() in which setting changes can be quietly reverted.
-
 ## 9.10.0 (12/12/2024)
 ### New Features
 * Introduce `TransactionOptions::commit_bypass_memtable` to enable transaction commit to bypass memtable insertions. This can be beneficial for transactions with many operations, as it reduces commit time that is mostly spent on memtable insertion.
@@ -2489,7 +2489,7 @@ checkout_folly:
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically
-	cd third-party/folly && git reset --hard 62baa6ba07ff0a23ee4f2ea2f5207e4c88464deb
+	cd third-party/folly && git reset --hard 33f5b67fcaeb8705b04fd1b850873a180dc89aaa
 	@# NOTE: this hack is required for clang in some cases
 	perl -pi -e 's/int rv = syscall/int rv = (int)syscall/' third-party/folly/folly/detail/Futex.cpp
 	@# NOTE: this hack is required for gcc in some cases
@@ -118,6 +118,7 @@ Status SecondaryCache::CreateFromString(
      sec_cache = NewCompressedSecondaryCache(sec_cache_opts);
    }

+
    if (status.ok()) {
      result->swap(sec_cache);
    }
@@ -191,7 +191,8 @@ namespace {
 class SharedState {
 public:
  explicit SharedState(CacheBench* cache_bench)
-      : cv_(&mu_), cache_bench_(cache_bench) {}
+      : cv_(&mu_),
+        cache_bench_(cache_bench) {}

  ~SharedState() = default;

@@ -845,6 +845,7 @@ TEST_P(CompressedSecondaryCacheTestWithCompressionParam,
  BasicTestHelper(sec_cache, sec_cache_is_compressed_);
 }

+
 TEST_P(CompressedSecondaryCacheTestWithCompressionParam, FailsTest) {
  FailsTest(sec_cache_is_compressed_);
 }
@@ -93,8 +93,9 @@ void LRUHandleTable::Resize() {

  uint32_t old_length = uint32_t{1} << length_bits_;
  int new_length_bits = length_bits_ + 1;
-  std::unique_ptr<LRUHandle*[]> new_list{
-      new LRUHandle* [size_t{1} << new_length_bits] {}};
+  std::unique_ptr<LRUHandle* []> new_list {
+    new LRUHandle* [size_t{1} << new_length_bits] {}
+  };
  [[maybe_unused]] uint32_t count = 0;
  for (uint32_t i = 0; i < old_length; i++) {
    LRUHandle* h = list_[i];
@@ -546,7 +546,7 @@ TYPED_TEST(ClockCacheTest, Limits) {
    // verify usage tracking on detached entries.)
    {
      size_t n = kCapacity * 5 + 1;
-      std::unique_ptr<HandleImpl*[]> ha{new HandleImpl* [n] {}};
+      std::unique_ptr<HandleImpl* []> ha { new HandleImpl* [n] {} };
      Status s;
      for (size_t i = 0; i < n && s.ok(); ++i) {
        hkey[1] = i;
@@ -713,7 +713,7 @@ TYPED_TEST(ClockCacheTest, ClockEvictionEffortCapTest) {
      // evictable entries are present beyond the cache capacity, despite
      // being evictable.
      constexpr size_t kCount = kCapacity - 1;
-      std::unique_ptr<HandleImpl*[]> ha{new HandleImpl* [kCount] {}};
+      std::unique_ptr<HandleImpl* []> ha { new HandleImpl* [kCount] {} };
      for (size_t i = 0; i < 2 * kCount; ++i) {
        UniqueId64x2 hkey = this->CheapHash(i);
        ASSERT_OK(shard.Insert(
@@ -7,4 +7,6 @@

 #include "cache/cache_entry_roles.h"

-namespace ROCKSDB_NAMESPACE {}  // namespace ROCKSDB_NAMESPACE
+namespace ROCKSDB_NAMESPACE {
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -951,20 +951,12 @@ TEST_F(DBTieredSecondaryCacheTest, FSBufferTest) {
                           IODebugContext* dbg) override {
          for (size_t i = 0; i < num_reqs; ++i) {
            FSReadRequest& req = reqs[i];
-            // See https://github.com/facebook/rocksdb/pull/13195 for why we
-            // want to set up our test implementation for FSAllocationPtr this
-            // way.
-            char* internalData = new char[req.len];
-            req.status = Read(req.offset, req.len, options, &req.result,
-                              internalData, dbg);
-
-            Slice* internalSlice = new Slice(internalData, req.len);
-            FSAllocationPtr internalPtr(internalSlice, [](void* ptr) {
-              delete[] static_cast<const char*>(
-                  static_cast<Slice*>(ptr)->data_);
-              delete static_cast<Slice*>(ptr);
+            FSAllocationPtr buffer(new char[req.len], [](void* ptr) {
+              delete[] static_cast<char*>(ptr);
            });
-            req.fs_scratch = std::move(internalPtr);
+            req.fs_scratch = std::move(buffer);
+            req.status = Read(req.offset, req.len, options, &req.result,
+                              static_cast<char*>(req.fs_scratch.get()), dbg);
          }
          return IOStatus::OK();
        }
@@ -97,7 +97,7 @@ Status ArenaWrappedDBIter::Refresh(const Snapshot* snapshot) {
    if (read_callback_) {
      read_callback_->Refresh(read_seq);
    }
-    Init(env, read_options_, cfd->ioptions(), sv->mutable_cf_options,
+    Init(env, read_options_, *(cfd->ioptions()), sv->mutable_cf_options,
         sv->current, read_seq,
         sv->mutable_cf_options.max_sequential_skip_in_iterations,
         sv->version_number, read_callback_, cfh_, expose_blob_index_,
@@ -82,8 +82,8 @@ Status BlobSource::PutBlobIntoCache(
  assert(cached_blob->IsEmpty());

  TypedHandle* cache_handle = nullptr;
-  const Status s = InsertEntryIntoCache(cache_key, blob->get(), &cache_handle,
-                                        Cache::Priority::BOTTOM);
+  const Status s = InsertEntryIntoCache(cache_key, blob->get(),
+                                        &cache_handle, Cache::Priority::BOTTOM);
  if (s.ok()) {
    blob->release();

@@ -231,7 +231,7 @@ struct rocksdb_livefiles_t {
 };
 struct rocksdb_column_family_handle_t {
  ColumnFamilyHandle* rep;
-  bool immortal; /* only true for default cf */
+  bool immortal;  /* only true for default cf */
 };
 struct rocksdb_column_family_metadata_t {
  ColumnFamilyMetaData rep;
@@ -1858,8 +1858,8 @@ extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes_cf_with_flags(
  delete[] ranges;
 }

-void DEPRECATED_rocksdb_delete_file(rocksdb_t* db, const char* name) {
-  db->rep->DEPRECATED_DeleteFile(name);
+void rocksdb_delete_file(rocksdb_t* db, const char* name) {
+  db->rep->DeleteFile(name);
 }

 const rocksdb_livefiles_t* rocksdb_livefiles(rocksdb_t* db) {
@@ -5000,35 +5000,6 @@ int rocksdb_compactoptions_get_target_level(rocksdb_compactoptions_t* opt) {
  return opt->rep.target_level;
 }

-void rocksdb_compactoptions_set_target_path_id(rocksdb_compactoptions_t* opt,
-                                               int n) {
-  opt->rep.target_path_id = n;
-}
-
-int rocksdb_compactoptions_get_target_path_id(rocksdb_compactoptions_t* opt) {
-  return opt->rep.target_path_id;
-}
-
-void rocksdb_compactoptions_set_allow_write_stall(rocksdb_compactoptions_t* opt,
-                                                  unsigned char v) {
-  opt->rep.allow_write_stall = v;
-}
-
-unsigned char rocksdb_compactoptions_get_allow_write_stall(
-    rocksdb_compactoptions_t* opt) {
-  return opt->rep.allow_write_stall;
-}
-
-void rocksdb_compactoptions_set_max_subcompactions(
-    rocksdb_compactoptions_t* opt, int n) {
-  opt->rep.max_subcompactions = n;
-}
-
-int rocksdb_compactoptions_get_max_subcompactions(
-    rocksdb_compactoptions_t* opt) {
-  return opt->rep.max_subcompactions;
-}
-
 void rocksdb_compactoptions_set_full_history_ts_low(
    rocksdb_compactoptions_t* opt, char* ts, size_t tslen) {
  if (ts == nullptr) {
@@ -2938,15 +2938,6 @@ int main(int argc, char** argv) {
    rocksdb_compactoptions_set_target_level(co, 1);
    CheckCondition(1 == rocksdb_compactoptions_get_target_level(co));

-    rocksdb_compactoptions_set_target_path_id(co, 1);
-    CheckCondition(1 == rocksdb_compactoptions_get_target_path_id(co));
-
-    rocksdb_compactoptions_set_allow_write_stall(co, 1);
-    CheckCondition(1 == rocksdb_compactoptions_get_allow_write_stall(co));
-
-    rocksdb_compactoptions_set_max_subcompactions(co, 1);
-    CheckCondition(1 == rocksdb_compactoptions_get_max_subcompactions(co));
-
    rocksdb_compactoptions_destroy(co);
  }

@@ -53,7 +53,7 @@ ColumnFamilyHandleImpl::ColumnFamilyHandleImpl(

 ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
  if (cfd_ != nullptr) {
-    for (auto& listener : cfd_->ioptions().listeners) {
+    for (auto& listener : cfd_->ioptions()->listeners) {
      listener->OnColumnFamilyHandleDeletionStarted(this);
    }
    // Job id == 0 means that this is not our background process, but rather
@@ -593,7 +593,7 @@ ColumnFamilyData::ColumnFamilyData(
                                      block_cache_tracer, io_tracer,
                                      db_session_id));
    blob_file_cache_.reset(
-        new BlobFileCache(_table_cache, &ioptions(), soptions(), id_,
+        new BlobFileCache(_table_cache, ioptions(), soptions(), id_,
                          internal_stats_->GetBlobFileReadHist(), io_tracer));
    blob_source_.reset(new BlobSource(ioptions_, mutable_cf_options_, db_id,
                                      db_session_id, blob_file_cache_.get()));
@@ -968,7 +968,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
    auto write_stall_condition_and_cause = GetWriteStallConditionAndCause(
        imm()->NumNotFlushed(), vstorage->l0_delay_trigger_count(),
        vstorage->estimated_compaction_needed_bytes(), mutable_cf_options,
-        ioptions());
+        *ioptions());
    write_stall_condition = write_stall_condition_and_cause.first;
    auto write_stall_cause = write_stall_condition_and_cause.second;

@@ -1160,13 +1160,12 @@ MemTable* ColumnFamilyData::ConstructNewMemtable(
                      write_buffer_manager_, earliest_seq, id_);
 }

-void ColumnFamilyData::CreateNewMemtable(SequenceNumber earliest_seq) {
+void ColumnFamilyData::CreateNewMemtable(
+    const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
  if (mem_ != nullptr) {
    delete mem_->Unref();
  }
-  // NOTE: db mutex must be locked for SetMemtable, so safe for
-  // GetLatestMutableCFOptions
-  SetMemtable(ConstructNewMemtable(GetLatestMutableCFOptions(), earliest_seq));
+  SetMemtable(ConstructNewMemtable(mutable_cf_options, earliest_seq));
  mem_->Ref();
 }

@@ -1385,7 +1384,7 @@ void ColumnFamilyData::InstallSuperVersion(
        new_superversion->write_stall_condition) {
      sv_context->PushWriteStallNotification(
          old_superversion->write_stall_condition,
-          new_superversion->write_stall_condition, GetName(), &ioptions());
+          new_superversion->write_stall_condition, GetName(), ioptions());
    }
    if (old_superversion->Unref()) {
      old_superversion->Cleanup();
@@ -1845,7 +1844,10 @@ const ImmutableOptions& GetImmutableOptions(ColumnFamilyHandle* column_family) {
  const ColumnFamilyData* const cfd = handle->cfd();
  assert(cfd);

-  return cfd->ioptions();
+  const ImmutableOptions* ioptions = cfd->ioptions();
+  assert(ioptions);
+
+  return *ioptions;
 }

 }  // namespace ROCKSDB_NAMESPACE
@@ -221,8 +221,7 @@ struct SuperVersion {
  // enable UDT feature, this is an empty string.
  std::string full_history_ts_low;

-  // An immutable snapshot of the DB's seqno to time mapping, usually shared
-  // between SuperVersions.
+  // A shared copy of the DB's seqno to time mapping.
  std::shared_ptr<const SeqnoToTimeMapping> seqno_to_time_mapping{nullptr};

  // should be called outside the mutex
@@ -343,17 +342,17 @@ class ColumnFamilyData {

  // thread-safe
  const FileOptions* soptions() const;
-  const ImmutableOptions& ioptions() const { return ioptions_; }
+  const ImmutableOptions* ioptions() const { return &ioptions_; }
  // REQUIRES: DB mutex held
  // This returns the MutableCFOptions used by current SuperVersion
  // You should use this API to reference MutableCFOptions most of the time.
-  const MutableCFOptions& GetCurrentMutableCFOptions() const {
-    return super_version_->mutable_cf_options;
+  const MutableCFOptions* GetCurrentMutableCFOptions() const {
+    return &(super_version_->mutable_cf_options);
  }
  // REQUIRES: DB mutex held
  // This returns the latest MutableCFOptions, which may be not in effect yet.
-  const MutableCFOptions& GetLatestMutableCFOptions() const {
-    return mutable_cf_options_;
+  const MutableCFOptions* GetLatestMutableCFOptions() const {
+    return &mutable_cf_options_;
  }

  // REQUIRES: DB mutex held
@@ -401,11 +400,10 @@ class ColumnFamilyData {
  uint64_t OldestLogToKeep();

  // See Memtable constructor for explanation of earliest_seq param.
-  // `mutable_cf_options` might need to be a saved copy if calling this without
-  // holding the DB mutex.
  MemTable* ConstructNewMemtable(const MutableCFOptions& mutable_cf_options,
                                 SequenceNumber earliest_seq);
-  void CreateNewMemtable(SequenceNumber earliest_seq);
+  void CreateNewMemtable(const MutableCFOptions& mutable_cf_options,
+                         SequenceNumber earliest_seq);

  TableCache* table_cache() const { return table_cache_.get(); }
  BlobFileCache* blob_file_cache() const { return blob_file_cache_.get(); }
@@ -3,6 +3,7 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

+
 #include <mutex>
 #include <string>
 #include <thread>
@@ -332,11 +332,11 @@ Compaction::Compaction(
              : (_blob_garbage_collection_policy ==
                         BlobGarbageCollectionPolicy::kDisable
                     ? false
-                     : mutable_cf_options().enable_blob_garbage_collection)),
+                     : mutable_cf_options()->enable_blob_garbage_collection)),
      blob_garbage_collection_age_cutoff_(
          _blob_garbage_collection_age_cutoff < 0 ||
                  _blob_garbage_collection_age_cutoff > 1
-              ? mutable_cf_options().blob_garbage_collection_age_cutoff
+              ? mutable_cf_options()->blob_garbage_collection_age_cutoff
              : _blob_garbage_collection_age_cutoff),
      penultimate_level_(
          // For simplicity, we don't support the concept of "penultimate level"
@@ -410,7 +410,6 @@ Compaction::Compaction(

 void Compaction::PopulatePenultimateLevelOutputRange() {
  if (!SupportsPerKeyPlacement()) {
-    assert(keep_in_last_level_through_seqno_ == kMaxSequenceNumber);
    return;
  }

@@ -453,27 +452,6 @@ void Compaction::PopulatePenultimateLevelOutputRange() {
  GetBoundaryInternalKeys(input_vstorage_, inputs_,
                          &penultimate_level_smallest_,
                          &penultimate_level_largest_, exclude_level);
-
-  if (penultimate_output_range_type_ !=
-      PenultimateOutputRangeType::kFullRange) {
-    // If not full range in penultimate level, must keep everything already
-    // in the last level there, because moving it back up might cause
-    // overlap/placement issues that are difficult to resolve properly in the
-    // presence of range deletes
-    SequenceNumber max_last_level_seqno = 0;
-    for (const auto& input_lvl : inputs_) {
-      if (input_lvl.level == output_level_) {
-        for (const auto& file : input_lvl.files) {
-          max_last_level_seqno =
-              std::max(max_last_level_seqno, file->fd.largest_seqno);
-        }
-      }
-    }
-
-    keep_in_last_level_through_seqno_ = max_last_level_seqno;
-  } else {
-    keep_in_last_level_through_seqno_ = 0;
-  }
 }

 Compaction::~Compaction() {
@@ -516,29 +494,22 @@ bool Compaction::OverlapPenultimateLevelOutputRange(
 }

 // key includes timestamp if user-defined timestamp is enabled.
-void Compaction::TEST_AssertWithinPenultimateLevelOutputRange(
-    const Slice& user_key, bool expect_failure) const {
-#ifdef NDEBUG
-  (void)user_key;
-  (void)expect_failure;
-#else
-  assert(SupportsPerKeyPlacement());
+bool Compaction::WithinPenultimateLevelOutputRange(
+    const ParsedInternalKey& ikey) const {
+  if (!SupportsPerKeyPlacement()) {
+    return false;
+  }

-  assert(penultimate_level_smallest_.size() > 0);
-  assert(penultimate_level_largest_.size() > 0);
+  if (penultimate_level_smallest_.size() == 0 ||
+      penultimate_level_largest_.size() == 0) {
+    return false;
+  }

-  auto* cmp = input_vstorage_->user_comparator();
+  const InternalKeyComparator* icmp = input_vstorage_->InternalComparator();

  // op_type of a key can change during compaction, e.g. Merge -> Put.
-  if (!(cmp->Compare(user_key, penultimate_level_smallest_.user_key()) >= 0)) {
-    assert(expect_failure);
-  } else if (!(cmp->Compare(user_key, penultimate_level_largest_.user_key()) <=
-               0)) {
-    assert(expect_failure);
-  } else {
-    assert(!expect_failure);
-  }
-#endif
+  return icmp->CompareKeySeq(ikey, penultimate_level_smallest_.Encode()) >= 0 &&
+         icmp->CompareKeySeq(ikey, penultimate_level_largest_.Encode()) <= 0;
 }

 bool Compaction::InputCompressionMatchesOutput() const {
@@ -592,7 +563,7 @@ bool Compaction::IsTrivialMove() const {
  // input files are non overlapping
  if ((mutable_cf_options_.compaction_options_universal.allow_trivial_move) &&
      (output_level_ != 0) &&
-      (cfd_->ioptions().compaction_style == kCompactionStyleUniversal)) {
+      (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal)) {
    return is_trivial_move_;
  }

@@ -650,7 +621,7 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
  if (bottommost_level_) {
    return true;
  } else if (output_level_ != 0 &&
-             cfd_->ioptions().compaction_style == kCompactionStyleLevel) {
+             cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
    // Maybe use binary search to find right entry instead of linear search?
    const Comparator* user_cmp = cfd_->user_comparator();
    for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
@@ -691,7 +662,7 @@ bool Compaction::KeyRangeNotExistsBeyondOutputLevel(
  if (bottommost_level_) {
    return true /* does not overlap */;
  } else if (output_level_ != 0 &&
-             cfd_->ioptions().compaction_style == kCompactionStyleLevel) {
+             cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
    const Comparator* user_cmp = cfd_->user_comparator();
    for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
      const std::vector<FileMetaData*>& files =
@@ -867,12 +838,12 @@ uint64_t Compaction::OutputFilePreallocationSize() const {
 }

 std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
-  if (!cfd_->ioptions().compaction_filter_factory) {
+  if (!cfd_->ioptions()->compaction_filter_factory) {
    return nullptr;
  }

  if (!cfd_->ioptions()
-           .compaction_filter_factory->ShouldFilterTableFileCreation(
+           ->compaction_filter_factory->ShouldFilterTableFileCreation(
               TableFileCreationReason::kCompaction)) {
    return nullptr;
  }
@@ -891,7 +862,7 @@ std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
        "for compaction.");
  }

-  return cfd_->ioptions().compaction_filter_factory->CreateCompactionFilter(
+  return cfd_->ioptions()->compaction_filter_factory->CreateCompactionFilter(
      context);
 }

@@ -925,8 +896,8 @@ bool Compaction::ShouldFormSubcompactions() const {

  // Round-Robin pri under leveled compaction allows subcompactions by default
  // and the number of subcompactions can be larger than max_subcompactions_
-  if (cfd_->ioptions().compaction_pri == kRoundRobin &&
-      cfd_->ioptions().compaction_style == kCompactionStyleLevel) {
+  if (cfd_->ioptions()->compaction_pri == kRoundRobin &&
+      cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
    return output_level_ > 0;
  }

@@ -934,9 +905,9 @@ bool Compaction::ShouldFormSubcompactions() const {
    return false;
  }

-  if (cfd_->ioptions().compaction_style == kCompactionStyleLevel) {
+  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
    return (start_level_ == 0 || is_manual_compaction_) && output_level_ > 0;
-  } else if (cfd_->ioptions().compaction_style == kCompactionStyleUniversal) {
+  } else if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
    return number_levels_ > 1 && output_level_ > 0;
  } else {
    return false;
@@ -288,14 +288,14 @@ class Compaction {

  // Return the ImmutableOptions that should be used throughout the compaction
  // procedure
-  const ImmutableOptions& immutable_options() const {
-    return immutable_options_;
+  const ImmutableOptions* immutable_options() const {
+    return &immutable_options_;
  }

  // Return the MutableCFOptions that should be used throughout the compaction
  // procedure
-  const MutableCFOptions& mutable_cf_options() const {
-    return mutable_cf_options_;
+  const MutableCFOptions* mutable_cf_options() const {
+    return &mutable_cf_options_;
  }

  // Returns the size in bytes that the output file should be preallocated to.
@@ -388,12 +388,12 @@ class Compaction {
  bool OverlapPenultimateLevelOutputRange(const Slice& smallest_key,
                                          const Slice& largest_key) const;

-  // For testing purposes, check that a key is within penultimate level
-  // output range for per_key_placement feature, which is safe to place the key
-  // to the penultimate level. Different compaction strategies have different
-  // rules. `user_key` includes timestamp if user-defined timestamp is enabled.
-  void TEST_AssertWithinPenultimateLevelOutputRange(
-      const Slice& user_key, bool expect_failure = false) const;
+  // Return true if the key is within penultimate level output range for
+  // per_key_placement feature, which is safe to place the key to the
+  // penultimate level. different compaction strategy has different rules.
+  // If per_key_placement is not supported, always return false.
+  //  key includes timestamp if user-defined timestamp is enabled.
+  bool WithinPenultimateLevelOutputRange(const ParsedInternalKey& ikey) const;

  CompactionReason compaction_reason() const { return compaction_reason_; }

@@ -456,17 +456,11 @@ class Compaction {
      const ImmutableOptions& immutable_options, const int start_level,
      const int output_level);

-  // If some data cannot be safely migrated "up" the LSM tree due to a change
-  // in the preclude_last_level_data_seconds setting, this indicates a sequence
-  // number for the newest data that must be kept in the last level.
-  SequenceNumber GetKeepInLastLevelThroughSeqno() const {
-    return keep_in_last_level_through_seqno_;
-  }
-
  // mark (or clear) all files that are being compacted
  void MarkFilesBeingCompacted(bool being_compacted) const;

 private:
+
  Status InitInputTableProperties();

  // get the smallest and largest key present in files to be compacted
@@ -612,8 +606,6 @@ class Compaction {
  // Blob garbage collection age cutoff.
  double blob_garbage_collection_age_cutoff_;

-  SequenceNumber keep_in_last_level_through_seqno_ = kMaxSequenceNumber;
-
  // only set when per_key_placement feature is enabled, -1 (kInvalidLevel)
  // means not supported.
  const int penultimate_level_;
@@ -38,16 +38,19 @@ CompactionIterator::CompactionIterator(
    const std::atomic<bool>* shutting_down,
    const std::shared_ptr<Logger> info_log,
    const std::string* full_history_ts_low,
-    std::optional<SequenceNumber> preserve_seqno_min)
+    const SequenceNumber preserve_time_min_seqno,
+    const SequenceNumber preclude_last_level_min_seqno)
    : CompactionIterator(
          input, cmp, merge_helper, last_sequence, snapshots, earliest_snapshot,
          earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env,
          report_detailed_time, expect_valid_internal_key, range_del_agg,
          blob_file_builder, allow_data_in_errors, enforce_single_del_contracts,
          manual_compaction_canceled,
-          compaction ? std::make_unique<RealCompaction>(compaction) : nullptr,
+          std::unique_ptr<CompactionProxy>(
+              compaction ? new RealCompaction(compaction) : nullptr),
          must_count_input_entries, compaction_filter, shutting_down, info_log,
-          full_history_ts_low, preserve_seqno_min) {}
+          full_history_ts_low, preserve_time_min_seqno,
+          preclude_last_level_min_seqno) {}

 CompactionIterator::CompactionIterator(
    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
@@ -65,7 +68,8 @@ CompactionIterator::CompactionIterator(
    const std::atomic<bool>* shutting_down,
    const std::shared_ptr<Logger> info_log,
    const std::string* full_history_ts_low,
-    std::optional<SequenceNumber> preserve_seqno_min)
+    const SequenceNumber preserve_time_min_seqno,
+    const SequenceNumber preclude_last_level_min_seqno)
    : input_(input, cmp, must_count_input_entries),
      cmp_(cmp),
      merge_helper_(merge_helper),
@@ -106,9 +110,10 @@ CompactionIterator::CompactionIterator(
      current_key_committed_(false),
      cmp_with_history_ts_low_(0),
      level_(compaction_ == nullptr ? 0 : compaction_->level()),
-      preserve_seqno_after_(preserve_seqno_min.value_or(earliest_snapshot)) {
+      preserve_time_min_seqno_(preserve_time_min_seqno),
+      preclude_last_level_min_seqno_(preclude_last_level_min_seqno) {
  assert(snapshots_ != nullptr);
-  assert(preserve_seqno_after_ <= earliest_snapshot_);
+  assert(preserve_time_min_seqno_ <= preclude_last_level_min_seqno_);

  if (compaction_ != nullptr) {
    level_ptrs_ = std::vector<size_t>(compaction_->number_levels(), 0);
@@ -1013,6 +1018,7 @@ void CompactionIterator::NextFromInput() {
        } else {
          if (ikey_.sequence != 0) {
            iter_stats_.num_timed_put_swap_preferred_seqno++;
+            saved_seq_for_penul_check_ = ikey_.sequence;
            ikey_.sequence = preferred_seqno;
          }
          ikey_.type = kTypeValue;
@@ -1253,6 +1259,71 @@ void CompactionIterator::GarbageCollectBlobIfNeeded() {
  }
 }

+void CompactionIterator::DecideOutputLevel() {
+  assert(compaction_->SupportsPerKeyPlacement());
+  output_to_penultimate_level_ = false;
+  // if the key is newer than the cutoff sequence or within the earliest
+  // snapshot, it should output to the penultimate level.
+  if (ikey_.sequence > preclude_last_level_min_seqno_ ||
+      ikey_.sequence > earliest_snapshot_) {
+    output_to_penultimate_level_ = true;
+  }
+
+#ifndef NDEBUG
+  // Could be overridden by unittest
+  PerKeyPlacementContext context(level_, ikey_.user_key, value_, ikey_.sequence,
+                                 output_to_penultimate_level_);
+  TEST_SYNC_POINT_CALLBACK("CompactionIterator::PrepareOutput.context",
+                           &context);
+  if (ikey_.sequence > earliest_snapshot_) {
+    output_to_penultimate_level_ = true;
+  }
+#endif  // NDEBUG
+
+  // saved_seq_for_penul_check_ is populated in `NextFromInput` when the
+  // entry's sequence number is non zero and validity context for output this
+  // entry is kSwapPreferredSeqno for use in `DecideOutputLevel`. It should be
+  // cleared out here unconditionally. Otherwise, it may end up getting consumed
+  // incorrectly by a different entry.
+  SequenceNumber seq_for_range_check =
+      (saved_seq_for_penul_check_.has_value() &&
+       saved_seq_for_penul_check_.value() != kMaxSequenceNumber)
+          ? saved_seq_for_penul_check_.value()
+          : ikey_.sequence;
+  saved_seq_for_penul_check_ = std::nullopt;
+  ParsedInternalKey ikey_for_range_check = ikey_;
+  if (seq_for_range_check != ikey_.sequence) {
+    ikey_for_range_check.sequence = seq_for_range_check;
+  }
+  if (output_to_penultimate_level_) {
+    // If it's decided to output to the penultimate level, but unsafe to do so,
+    // still output to the last level. For example, moving the data from a lower
+    // level to a higher level outside of the higher-level input key range is
+    // considered unsafe, because the key may conflict with higher-level SSTs
+    // not from this compaction.
+    // TODO: add statistic for declined output_to_penultimate_level
+    bool safe_to_penultimate_level =
+        compaction_->WithinPenultimateLevelOutputRange(ikey_for_range_check);
+    if (!safe_to_penultimate_level) {
+      output_to_penultimate_level_ = false;
+      // It could happen when disable/enable `last_level_temperature` while
+      // holding a snapshot. When `last_level_temperature` is not set
+      // (==kUnknown), the data newer than any snapshot is pushed to the last
+      // level, but when the per_key_placement feature is enabled on the fly,
+      // the data later than the snapshot has to be moved to the penultimate
+      // level, which may or may not be safe. So the user needs to make sure all
+      // snapshot is released before enabling `last_level_temperature` feature
+      // We will migrate the feature to `last_level_temperature` and maybe make
+      // it not dynamically changeable.
+      if (seq_for_range_check > earliest_snapshot_) {
+        status_ = Status::Corruption(
+            "Unsafe to store Seq later than snapshot in the last level if "
+            "per_key_placement is enabled");
+      }
+    }
+  }
+}
+
 void CompactionIterator::PrepareOutput() {
  if (Valid()) {
    if (LIKELY(!is_range_del_)) {
@@ -1261,6 +1332,13 @@ void CompactionIterator::PrepareOutput() {
      } else if (ikey_.type == kTypeBlobIndex) {
        GarbageCollectBlobIfNeeded();
      }
+
+      // For range del sentinel, we don't use it to cut files for bottommost
+      // compaction. So it should not make a difference which output level we
+      // decide.
+      if (compaction_ != nullptr && compaction_->SupportsPerKeyPlacement()) {
+        DecideOutputLevel();
+      }
    }

    // Zeroing out the sequence number leads to better compression.
@@ -1278,7 +1356,8 @@ void CompactionIterator::PrepareOutput() {
        !compaction_->allow_ingest_behind() && bottommost_level_ &&
        DefinitelyInSnapshot(ikey_.sequence, earliest_snapshot_) &&
        ikey_.type != kTypeMerge && current_key_committed_ &&
-        ikey_.sequence <= preserve_seqno_after_ && !is_range_del_) {
+        !output_to_penultimate_level_ &&
+        ikey_.sequence < preserve_time_min_seqno_ && !is_range_del_) {
      if (ikey_.type == kTypeDeletion ||
          (ikey_.type == kTypeSingleDeletion && timestamp_size_ == 0)) {
        ROCKS_LOG_FATAL(
@@ -117,6 +117,10 @@ class CompactionIterator {
    virtual const Compaction* real_compaction() const = 0;

    virtual bool SupportsPerKeyPlacement() const = 0;
+
+    // `key` includes timestamp if user-defined timestamp is enabled.
+    virtual bool WithinPenultimateLevelOutputRange(
+        const ParsedInternalKey&) const = 0;
  };

  class RealCompaction : public CompactionProxy {
@@ -124,6 +128,8 @@ class CompactionIterator {
    explicit RealCompaction(const Compaction* compaction)
        : compaction_(compaction) {
      assert(compaction_);
+      assert(compaction_->immutable_options());
+      assert(compaction_->mutable_cf_options());
    }

    int level() const override { return compaction_->level(); }
@@ -145,11 +151,11 @@ class CompactionIterator {
    }

    bool allow_ingest_behind() const override {
-      return compaction_->immutable_options().allow_ingest_behind;
+      return compaction_->immutable_options()->allow_ingest_behind;
    }

    bool allow_mmap_reads() const override {
-      return compaction_->immutable_options().allow_mmap_reads;
+      return compaction_->immutable_options()->allow_mmap_reads;
    }

    bool enable_blob_garbage_collection() const override {
@@ -161,7 +167,7 @@ class CompactionIterator {
    }

    uint64_t blob_compaction_readahead_size() const override {
-      return compaction_->mutable_cf_options().blob_compaction_readahead_size;
+      return compaction_->mutable_cf_options()->blob_compaction_readahead_size;
    }

    const Version* input_version() const override {
@@ -178,6 +184,14 @@ class CompactionIterator {
      return compaction_->SupportsPerKeyPlacement();
    }

+    // Check if key is within penultimate level output range, to see if it's
+    // safe to output to the penultimate level for per_key_placement feature.
+    // `key` includes timestamp if user-defined timestamp is enabled.
+    bool WithinPenultimateLevelOutputRange(
+        const ParsedInternalKey& ikey) const override {
+      return compaction_->WithinPenultimateLevelOutputRange(ikey);
+    }
+
   private:
    const Compaction* compaction_;
  };
@@ -202,29 +216,29 @@ class CompactionIterator {
      const std::atomic<bool>* shutting_down = nullptr,
      const std::shared_ptr<Logger> info_log = nullptr,
      const std::string* full_history_ts_low = nullptr,
-      std::optional<SequenceNumber> preserve_seqno_min = {});
+      const SequenceNumber preserve_time_min_seqno = kMaxSequenceNumber,
+      const SequenceNumber preclude_last_level_min_seqno = kMaxSequenceNumber);

  // Constructor with custom CompactionProxy, used for tests.
-  CompactionIterator(InternalIterator* input, const Comparator* cmp,
-                     MergeHelper* merge_helper, SequenceNumber last_sequence,
-                     std::vector<SequenceNumber>* snapshots,
-                     SequenceNumber earliest_snapshot,
-                     SequenceNumber earliest_write_conflict_snapshot,
-                     SequenceNumber job_snapshot,
-                     const SnapshotChecker* snapshot_checker, Env* env,
-                     bool report_detailed_time, bool expect_valid_internal_key,
-                     CompactionRangeDelAggregator* range_del_agg,
-                     BlobFileBuilder* blob_file_builder,
-                     bool allow_data_in_errors,
-                     bool enforce_single_del_contracts,
-                     const std::atomic<bool>& manual_compaction_canceled,
-                     std::unique_ptr<CompactionProxy> compaction,
-                     bool must_count_input_entries,
-                     const CompactionFilter* compaction_filter = nullptr,
-                     const std::atomic<bool>* shutting_down = nullptr,
-                     const std::shared_ptr<Logger> info_log = nullptr,
-                     const std::string* full_history_ts_low = nullptr,
-                     std::optional<SequenceNumber> preserve_seqno_min = {});
+  CompactionIterator(
+      InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
+      SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+      SequenceNumber earliest_snapshot,
+      SequenceNumber earliest_write_conflict_snapshot,
+      SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker,
+      Env* env, bool report_detailed_time, bool expect_valid_internal_key,
+      CompactionRangeDelAggregator* range_del_agg,
+      BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
+      bool enforce_single_del_contracts,
+      const std::atomic<bool>& manual_compaction_canceled,
+      std::unique_ptr<CompactionProxy> compaction,
+      bool must_count_input_entries,
+      const CompactionFilter* compaction_filter = nullptr,
+      const std::atomic<bool>* shutting_down = nullptr,
+      const std::shared_ptr<Logger> info_log = nullptr,
+      const std::string* full_history_ts_low = nullptr,
+      const SequenceNumber preserve_time_min_seqno = kMaxSequenceNumber,
+      const SequenceNumber preclude_last_level_min_seqno = kMaxSequenceNumber);

  ~CompactionIterator();

@@ -255,6 +269,11 @@ class CompactionIterator {
  const CompactionIterationStats& iter_stats() const { return iter_stats_; }
  bool HasNumInputEntryScanned() const { return input_.HasNumItered(); }
  uint64_t NumInputEntryScanned() const { return input_.NumItered(); }
+  // If the current key should be placed on penultimate level, only valid if
+  // per_key_placement is supported
+  bool output_to_penultimate_level() const {
+    return output_to_penultimate_level_;
+  }
  Status InputStatus() const { return input_.status(); }

  bool IsDeleteRangeSentinelKey() const { return is_range_del_; }
@@ -266,6 +285,10 @@ class CompactionIterator {
  // Do final preparations before presenting the output to the callee.
  void PrepareOutput();

+  // Decide the current key should be output to the last level or penultimate
+  // level, only call for compaction supports per key placement
+  void DecideOutputLevel();
+
  // Passes the output value to the blob file builder (if any), and replaces it
  // with the corresponding blob reference if it has been actually written to a
  // blob file (i.e. if it passed the value size check). Returns true if the
@@ -417,6 +440,16 @@ class CompactionIterator {
  // NextFromInput()).
  ParsedInternalKey ikey_;

+  // When a kTypeValuePreferredSeqno entry's preferred seqno is safely swapped
+  // in in this compaction, this field saves its original sequence number for
+  // range checking whether it's safe to be placed on the penultimate level.
+  // This is to ensure when such an entry happens to be the right boundary of
+  // penultimate safe range, it won't get excluded because with the preferred
+  // seqno swapped in, it's now larger than the right boundary (itself before
+  // the swap). This is safe to do, because preferred seqno is swapped in only
+  // when no entries with the same user key exist on lower levels and this entry
+  // is already visible in the earliest snapshot.
+  std::optional<SequenceNumber> saved_seq_for_penul_check_ = kMaxSequenceNumber;
  // Stores whether ikey_.user_key is valid. If set to false, the user key is
  // not compared against the current key in the underlying iterator.
  bool has_current_user_key_ = false;
@@ -474,8 +507,17 @@ class CompactionIterator {
  // just been zeroed out during bottommost compaction.
  bool last_key_seq_zeroed_{false};

-  // Max seqno that can be zeroed out at last level (various reasons)
-  const SequenceNumber preserve_seqno_after_ = kMaxSequenceNumber;
+  // True if the current key should be output to the penultimate level if
+  // possible, compaction logic makes the final decision on which level to
+  // output to.
+  bool output_to_penultimate_level_{false};
+
+  // min seqno for preserving the time information.
+  const SequenceNumber preserve_time_min_seqno_ = kMaxSequenceNumber;
+
+  // min seqno to preclude the data from the last level, if the key seqno larger
+  // than this, it will be output to penultimate level
+  const SequenceNumber preclude_last_level_min_seqno_ = kMaxSequenceNumber;

  void AdvanceInputIter() { input_.Next(); }

@@ -192,6 +192,11 @@ class FakeCompaction : public CompactionIterator::CompactionProxy {
    return supports_per_key_placement;
  }

+  bool WithinPenultimateLevelOutputRange(
+      const ParsedInternalKey& key) const override {
+    return (!key.user_key.starts_with("unsafe_pb"));
+  }
+
  bool key_not_exists_beyond_output_level = false;

  bool is_bottommost_level = false;
@@ -839,6 +844,123 @@ TEST_P(CompactionIteratorTest, ZeroSeqOfKeyAndSnapshot) {
 INSTANTIATE_TEST_CASE_P(CompactionIteratorTestInstance, CompactionIteratorTest,
                        testing::Values(true, false));

+class PerKeyPlacementCompIteratorTest : public CompactionIteratorTest {
+ public:
+  bool SupportsPerKeyPlacement() const override { return true; }
+};
+
+TEST_P(PerKeyPlacementCompIteratorTest, SplitLastLevelData) {
+  std::atomic_uint64_t latest_cold_seq = 0;
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
+        auto context = static_cast<PerKeyPlacementContext*>(arg);
+        context->output_to_penultimate_level =
+            context->seq_num > latest_cold_seq;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  latest_cold_seq = 5;
+
+  InitIterators(
+      {test::KeyStr("a", 7, kTypeValue), test::KeyStr("b", 6, kTypeValue),
+       test::KeyStr("c", 5, kTypeValue)},
+      {"vala", "valb", "valc"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber,
+      nullptr, nullptr, true);
+  c_iter_->SeekToFirst();
+  ASSERT_TRUE(c_iter_->Valid());
+
+  // the first 2 keys are hot, which should has
+  // `output_to_penultimate_level()==true` and seq num not zeroed out
+  ASSERT_EQ(test::KeyStr("a", 7, kTypeValue), c_iter_->key().ToString());
+  ASSERT_TRUE(c_iter_->output_to_penultimate_level());
+  c_iter_->Next();
+  ASSERT_TRUE(c_iter_->Valid());
+  ASSERT_EQ(test::KeyStr("b", 6, kTypeValue), c_iter_->key().ToString());
+  ASSERT_TRUE(c_iter_->output_to_penultimate_level());
+  c_iter_->Next();
+  ASSERT_TRUE(c_iter_->Valid());
+  // `a` is cold data, which should be output to bottommost
+  ASSERT_EQ(test::KeyStr("c", 0, kTypeValue), c_iter_->key().ToString());
+  ASSERT_FALSE(c_iter_->output_to_penultimate_level());
+  c_iter_->Next();
+  ASSERT_OK(c_iter_->status());
+  ASSERT_FALSE(c_iter_->Valid());
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
+TEST_P(PerKeyPlacementCompIteratorTest, SnapshotData) {
+  AddSnapshot(5);
+
+  InitIterators(
+      {test::KeyStr("a", 7, kTypeValue), test::KeyStr("b", 6, kTypeDeletion),
+       test::KeyStr("b", 5, kTypeValue)},
+      {"vala", "", "valb"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber,
+      nullptr, nullptr, true);
+  c_iter_->SeekToFirst();
+  ASSERT_TRUE(c_iter_->Valid());
+
+  // The first key and the tombstone are within snapshot, which should output
+  // to the penultimate level (and seq num cannot be zeroed out).
+  ASSERT_EQ(test::KeyStr("a", 7, kTypeValue), c_iter_->key().ToString());
+  ASSERT_TRUE(c_iter_->output_to_penultimate_level());
+  c_iter_->Next();
+  ASSERT_TRUE(c_iter_->Valid());
+  ASSERT_EQ(test::KeyStr("b", 6, kTypeDeletion), c_iter_->key().ToString());
+  ASSERT_TRUE(c_iter_->output_to_penultimate_level());
+  c_iter_->Next();
+  ASSERT_TRUE(c_iter_->Valid());
+  // `a` is not protected by the snapshot, the sequence number is zero out and
+  // should output bottommost
+  ASSERT_EQ(test::KeyStr("b", 0, kTypeValue), c_iter_->key().ToString());
+  ASSERT_FALSE(c_iter_->output_to_penultimate_level());
+  c_iter_->Next();
+  ASSERT_OK(c_iter_->status());
+  ASSERT_FALSE(c_iter_->Valid());
+}
+
+TEST_P(PerKeyPlacementCompIteratorTest, ConflictWithSnapshot) {
+  std::atomic_uint64_t latest_cold_seq = 0;
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
+        auto context = static_cast<PerKeyPlacementContext*>(arg);
+        context->output_to_penultimate_level =
+            context->seq_num > latest_cold_seq;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  latest_cold_seq = 6;
+
+  AddSnapshot(5);
+
+  InitIterators({test::KeyStr("a", 7, kTypeValue),
+                 test::KeyStr("unsafe_pb", 6, kTypeValue),
+                 test::KeyStr("c", 5, kTypeValue)},
+                {"vala", "valb", "valc"}, {}, {}, kMaxSequenceNumber,
+                kMaxSequenceNumber, nullptr, nullptr, true);
+  c_iter_->SeekToFirst();
+  ASSERT_TRUE(c_iter_->Valid());
+
+  ASSERT_EQ(test::KeyStr("a", 7, kTypeValue), c_iter_->key().ToString());
+  ASSERT_TRUE(c_iter_->output_to_penultimate_level());
+  // the 2nd key is unsafe to output_to_penultimate_level, but it's within
+  // snapshot so for per_key_placement feature it has to be outputted to the
+  // penultimate level. which is a corruption. We should never see
+  // such case as the data with seq num (within snapshot) should always come
+  // from higher compaction input level, which makes it safe to
+  // output_to_penultimate_level.
+  c_iter_->Next();
+  ASSERT_TRUE(c_iter_->status().IsCorruption());
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
+INSTANTIATE_TEST_CASE_P(PerKeyPlacementCompIteratorTest,
+                        PerKeyPlacementCompIteratorTest,
+                        testing::Values(true, false));
+
 // Tests how CompactionIterator work together with SnapshotChecker.
 class CompactionIteratorWithSnapshotCheckerTest
    : public CompactionIteratorTest {
@@ -245,10 +245,7 @@ void CompactionJob::ReportStartedCompaction(Compaction* compaction) {
  compaction_job_stats_->is_full_compaction = compaction->is_full_compaction();
 }

-void CompactionJob::Prepare(
-    std::optional<std::pair<std::optional<Slice>, std::optional<Slice>>>
-        known_single_subcompact) {
-  db_mutex_->AssertHeld();
+void CompactionJob::Prepare() {
  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_COMPACTION_PREPARE);

@@ -263,12 +260,11 @@ void CompactionJob::Prepare(
  write_hint_ = storage_info->CalculateSSTWriteHint(c->output_level());
  bottommost_level_ = c->bottommost_level();

-  if (!known_single_subcompact.has_value() && c->ShouldFormSubcompactions()) {
+  if (c->ShouldFormSubcompactions()) {
    StopWatch sw(db_options_.clock, stats_, SUBCOMPACTION_SETUP_TIME);
    GenSubcompactionBoundaries();
  }
  if (boundaries_.size() >= 1) {
-    assert(!known_single_subcompact.has_value());
    for (size_t i = 0; i <= boundaries_.size(); i++) {
      compact_->sub_compact_states.emplace_back(
          c, (i != 0) ? std::optional<Slice>(boundaries_[i - 1]) : std::nullopt,
@@ -284,25 +280,16 @@ void CompactionJob::Prepare(
    RecordInHistogram(stats_, NUM_SUBCOMPACTIONS_SCHEDULED,
                      compact_->sub_compact_states.size());
  } else {
-    std::optional<Slice> start_key;
-    std::optional<Slice> end_key;
-    if (known_single_subcompact.has_value()) {
-      start_key = known_single_subcompact.value().first;
-      end_key = known_single_subcompact.value().second;
-    } else {
-      assert(!start_key.has_value() && !end_key.has_value());
-    }
-    compact_->sub_compact_states.emplace_back(c, start_key, end_key,
+    compact_->sub_compact_states.emplace_back(c, std::nullopt, std::nullopt,
                                              /*sub_job_id*/ 0);
  }

  // collect all seqno->time information from the input files which will be used
  // to encode seqno->time to the output files.
-  SequenceNumber preserve_time_min_seqno = kMaxSequenceNumber;
-  SequenceNumber preclude_last_level_min_seqno = kMaxSequenceNumber;
+
  uint64_t preserve_time_duration =
-      std::max(c->mutable_cf_options().preserve_internal_time_seconds,
-               c->mutable_cf_options().preclude_last_level_data_seconds);
+      std::max(c->mutable_cf_options()->preserve_internal_time_seconds,
+               c->mutable_cf_options()->preclude_last_level_data_seconds);

  if (preserve_time_duration > 0) {
    const ReadOptions read_options(Env::IOActivity::kCompaction);
@@ -332,16 +319,16 @@ void CompactionJob::Prepare(
                     "Failed to get current time in compaction: Status: %s",
                     s.ToString().c_str());
      // preserve all time information
-      preserve_time_min_seqno = 0;
-      preclude_last_level_min_seqno = 0;
+      preserve_time_min_seqno_ = 0;
+      preclude_last_level_min_seqno_ = 0;
      seqno_to_time_mapping_.Enforce();
    } else {
      seqno_to_time_mapping_.Enforce(_current_time);
      seqno_to_time_mapping_.GetCurrentTieringCutoffSeqnos(
          static_cast<uint64_t>(_current_time),
-          c->mutable_cf_options().preserve_internal_time_seconds,
-          c->mutable_cf_options().preclude_last_level_data_seconds,
-          &preserve_time_min_seqno, &preclude_last_level_min_seqno);
+          c->mutable_cf_options()->preserve_internal_time_seconds,
+          c->mutable_cf_options()->preclude_last_level_data_seconds,
+          &preserve_time_min_seqno_, &preclude_last_level_min_seqno_);
    }
    // For accuracy of the GetProximalSeqnoBeforeTime queries above, we only
    // limit the capacity after them.
@@ -354,43 +341,6 @@ void CompactionJob::Prepare(
    // larger than kMaxSeqnoTimePairsPerSST.
    seqno_to_time_mapping_.SetCapacity(kMaxSeqnoToTimeEntries);
  }
-#ifndef NDEBUG
-  assert(preserve_time_min_seqno <= preclude_last_level_min_seqno);
-  TEST_SYNC_POINT_CALLBACK(
-      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
-      static_cast<void*>(&preclude_last_level_min_seqno));
-  // Restore the invariant asserted above, in case it was broken under the
-  // callback
-  preserve_time_min_seqno =
-      std::min(preclude_last_level_min_seqno, preserve_time_min_seqno);
-#endif
-
-  // Preserve sequence numbers for preserved write times and snapshots, though
-  // the specific sequence number of the earliest snapshot can be zeroed.
-  preserve_seqno_after_ =
-      std::max(preserve_time_min_seqno, SequenceNumber{1}) - 1;
-  preserve_seqno_after_ = std::min(preserve_seqno_after_, earliest_snapshot_);
-  // If using preclude feature, also preclude snapshots from last level, just
-  // because they are heuristically more likely to be accessed than non-snapshot
-  // data.
-  if (preclude_last_level_min_seqno < kMaxSequenceNumber &&
-      earliest_snapshot_ < preclude_last_level_min_seqno) {
-    preclude_last_level_min_seqno = earliest_snapshot_;
-  }
-  // Now combine what we would like to preclude from last level with what we
-  // can safely support without dangerously moving data back up the LSM tree,
-  // to get the final seqno threshold for penultimate vs. last. In particular,
-  // when the reserved output key range for the penultimate level does not
-  // include the entire last level input key range, we need to keep entries
-  // already in the last level there. (Even allowing within-range entries to
-  // move back up could cause problems with range tombstones. Perhaps it
-  // would be better in some rare cases to keep entries in the last level
-  // one-by-one rather than based on sequence number, but that would add extra
-  // tracking and complexity to CompactionIterator that is probably not
-  // worthwhile overall. Correctness is also more clear when splitting by
-  // seqno threshold.)
-  penultimate_after_seqno_ = std::max(preclude_last_level_min_seqno,
-                                      c->GetKeepInLastLevelThroughSeqno());
 }

 uint64_t CompactionJob::GetSubcompactionsLimit() {
@@ -519,14 +469,14 @@ void CompactionJob::GenSubcompactionBoundaries() {
  ReadOptions read_options(Env::IOActivity::kCompaction);
  read_options.rate_limiter_priority = GetRateLimiterPriority();
  auto* c = compact_->compaction;
-  if (c->mutable_cf_options().table_factory->Name() ==
+  if (c->mutable_cf_options()->table_factory->Name() ==
      TableFactory::kPlainTableName()) {
    return;
  }

  if (c->max_subcompactions() <= 1 &&
-      !(c->immutable_options().compaction_pri == kRoundRobin &&
-        c->immutable_options().compaction_style == kCompactionStyleLevel)) {
+      !(c->immutable_options()->compaction_pri == kRoundRobin &&
+        c->immutable_options()->compaction_style == kCompactionStyleLevel)) {
    return;
  }
  auto* cfd = c->column_family_data();
@@ -556,7 +506,7 @@ void CompactionJob::GenSubcompactionBoundaries() {
        FileMetaData* f = flevel->files[i].file_metadata;
        std::vector<TableReader::Anchor> my_anchors;
        Status s = cfd->table_cache()->ApproximateKeyAnchors(
-            read_options, icomp, *f, c->mutable_cf_options(), my_anchors);
+            read_options, icomp, *f, *c->mutable_cf_options(), my_anchors);
        if (!s.ok() || my_anchors.empty()) {
          my_anchors.emplace_back(f->largest.user_key(), f->fd.GetFileSize());
        }
@@ -595,8 +545,8 @@ void CompactionJob::GenSubcompactionBoundaries() {
  // Get the number of planned subcompactions, may update reserve threads
  // and update extra_num_subcompaction_threads_reserved_ for round-robin
  uint64_t num_planned_subcompactions;
-  if (c->immutable_options().compaction_pri == kRoundRobin &&
-      c->immutable_options().compaction_style == kCompactionStyleLevel) {
+  if (c->immutable_options()->compaction_pri == kRoundRobin &&
+      c->immutable_options()->compaction_style == kCompactionStyleLevel) {
    // For round-robin compaction prioity, we need to employ more
    // subcompactions (may exceed the max_subcompaction limit). The extra
    // subcompactions will be executed using reserved threads and taken into
@@ -635,9 +585,9 @@ void CompactionJob::GenSubcompactionBoundaries() {
  uint64_t target_range_size = std::max(
      total_size / num_planned_subcompactions,
      MaxFileSizeForLevel(
-          c->mutable_cf_options(), out_lvl,
-          c->immutable_options().compaction_style, base_level,
-          c->immutable_options().level_compaction_dynamic_level_bytes));
+          *(c->mutable_cf_options()), out_lvl,
+          c->immutable_options()->compaction_style, base_level,
+          c->immutable_options()->level_compaction_dynamic_level_bytes));

  if (target_range_size >= total_size) {
    return;
@@ -780,13 +730,14 @@ Status CompactionJob::Run() {
            verify_table_read_options, file_options_,
            cfd->internal_comparator(), files_output[file_idx]->meta,
            /*range_del_agg=*/nullptr,
-            compact_->compaction->mutable_cf_options(),
+            *compact_->compaction->mutable_cf_options(),
            /*table_reader_ptr=*/nullptr,
            cfd->internal_stats()->GetFileReadHist(
                compact_->compaction->output_level()),
            TableReaderCaller::kCompactionRefill, /*arena=*/nullptr,
            /*skip_filters=*/false, compact_->compaction->output_level(),
-            MaxFileSizeForL0MetaPin(compact_->compaction->mutable_cf_options()),
+            MaxFileSizeForL0MetaPin(
+                *compact_->compaction->mutable_cf_options()),
            /*smallest_compaction_key=*/nullptr,
            /*largest_compaction_key=*/nullptr,
            /*allow_unprepared_value=*/false);
@@ -843,7 +794,7 @@ Status CompactionJob::Run() {
  for (const auto& state : compact_->sub_compact_states) {
    for (const auto& output : state.GetOutputs()) {
      auto fn =
-          TableFileName(state.compaction->immutable_options().cf_paths,
+          TableFileName(state.compaction->immutable_options()->cf_paths,
                        output.meta.fd.GetNumber(), output.meta.fd.GetPathId());
      compact_->compaction->SetOutputTableProperties(fn,
                                                     output.table_properties);
@@ -902,7 +853,8 @@ Status CompactionJob::Run() {
  return status;
 }

-Status CompactionJob::Install(bool* compaction_released) {
+Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
+                              bool* compaction_released) {
  assert(compact_);

  AutoThreadOperationStageUpdater stage_updater(
@@ -918,7 +870,7 @@ Status CompactionJob::Install(bool* compaction_released) {
                                            compaction_stats_);

  if (status.ok()) {
-    status = InstallCompactionResults(compaction_released);
+    status = InstallCompactionResults(mutable_cf_options, compaction_released);
  }
  if (!versions_->io_status().ok()) {
    io_status_ = versions_->io_status();
@@ -1007,7 +959,8 @@ Status CompactionJob::Install(bool* compaction_released) {
  UpdateCompactionJobStats(stats);

  auto stream = event_logger_->LogToBuffer(log_buffer_, 8192);
-  stream << "job" << job_id_ << "event" << "compaction_finished"
+  stream << "job" << job_id_ << "event"
+         << "compaction_finished"
         << "compaction_time_micros" << stats.micros
         << "compaction_time_cpu_micros" << stats.cpu_micros << "output_level"
         << compact_->compaction->output_level() << "num_output_files"
@@ -1097,10 +1050,12 @@ void CompactionJob::NotifyOnSubcompactionBegin(
    listener->OnSubcompactionBegin(info);
  }
  info.status.PermitUncheckedError();
+
 }

 void CompactionJob::NotifyOnSubcompactionCompleted(
    SubcompactionState* sub_compact) {
+
  if (db_options_.listeners.empty()) {
    return;
  }
@@ -1143,7 +1098,8 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {

  // Create compaction filter and fail the compaction if
  // IgnoreSnapshots() = false because it is not supported anymore
-  const CompactionFilter* compaction_filter = cfd->ioptions().compaction_filter;
+  const CompactionFilter* compaction_filter =
+      cfd->ioptions()->compaction_filter;
  std::unique_ptr<CompactionFilter> compaction_filter_from_factory = nullptr;
  if (compaction_filter == nullptr) {
    compaction_filter_from_factory =
@@ -1159,12 +1115,9 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {

  NotifyOnSubcompactionBegin(sub_compact);

-  // This is assigned after creation of SubcompactionState to simplify that
-  // creation across both CompactionJob and CompactionServiceCompactionJob
-  sub_compact->AssignRangeDelAggregator(
-      std::make_unique<CompactionRangeDelAggregator>(
-          &cfd->internal_comparator(), existing_snapshots_,
-          &full_history_ts_low_, &trim_ts_));
+  auto range_del_agg = std::make_unique<CompactionRangeDelAggregator>(
+      &cfd->internal_comparator(), existing_snapshots_, &full_history_ts_low_,
+      &trim_ts_);

  // TODO: since we already use C++17, should use
  // std::optional<const Slice> instead.
@@ -1209,7 +1162,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  // Although the v2 aggregator is what the level iterator(s) know about,
  // the AddTombstones calls will be propagated down to the v1 aggregator.
  std::unique_ptr<InternalIterator> raw_input(versions_->MakeInputIterator(
-      read_options, sub_compact->compaction, sub_compact->RangeDelAgg(),
+      read_options, sub_compact->compaction, range_del_agg.get(),
      file_options_for_read_, start, end));
  InternalIterator* input = raw_input.get();

@@ -1301,27 +1254,28 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  }

  MergeHelper merge(
-      env_, cfd->user_comparator(), cfd->ioptions().merge_operator.get(),
+      env_, cfd->user_comparator(), cfd->ioptions()->merge_operator.get(),
      compaction_filter, db_options_.info_log.get(),
      false /* internal key corruption is expected */,
      existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
      snapshot_checker_, compact_->compaction->level(), db_options_.stats);

-  const auto& mutable_cf_options =
+  const MutableCFOptions* mutable_cf_options =
      sub_compact->compaction->mutable_cf_options();
+  assert(mutable_cf_options);

  std::vector<std::string> blob_file_paths;

  // TODO: BlobDB to support output_to_penultimate_level compaction, which needs
  //  2 builders, so may need to move to `CompactionOutputs`
  std::unique_ptr<BlobFileBuilder> blob_file_builder(
-      (mutable_cf_options.enable_blob_files &&
+      (mutable_cf_options->enable_blob_files &&
       sub_compact->compaction->output_level() >=
-           mutable_cf_options.blob_file_starting_level)
+           mutable_cf_options->blob_file_starting_level)
          ? new BlobFileBuilder(
                versions_, fs_.get(),
-                &sub_compact->compaction->immutable_options(),
-                &mutable_cf_options, &file_options_, &write_options, db_id_,
+                sub_compact->compaction->immutable_options(),
+                mutable_cf_options, &file_options_, &write_options, db_id_,
                db_session_id_, job_id_, cfd->GetID(), cfd->GetName(),
                write_hint_, io_tracer_, blob_callback_,
                BlobFileCreationReason::kCompaction, &blob_file_paths,
@@ -1344,15 +1298,20 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
      &existing_snapshots_, earliest_snapshot_,
      earliest_write_conflict_snapshot_, job_snapshot_seq, snapshot_checker_,
      env_, ShouldReportDetailedTime(env_, stats_),
-      /*expect_valid_internal_key=*/true, sub_compact->RangeDelAgg(),
+      /*expect_valid_internal_key=*/true, range_del_agg.get(),
      blob_file_builder.get(), db_options_.allow_data_in_errors,
      db_options_.enforce_single_del_contracts, manual_compaction_canceled_,
      sub_compact->compaction
          ->DoesInputReferenceBlobFiles() /* must_count_input_entries */,
      sub_compact->compaction, compaction_filter, shutting_down_,
-      db_options_.info_log, full_history_ts_low, preserve_seqno_after_);
+      db_options_.info_log, full_history_ts_low, preserve_time_min_seqno_,
+      preclude_last_level_min_seqno_);
  c_iter->SeekToFirst();

+  // Assign range delete aggregator to the target output level, which makes sure
+  // it only output to single level
+  sub_compact->AssignRangeDelAggregator(std::move(range_del_agg));
+
  const auto& c_iter_stats = c_iter->iter_stats();

  // define the open and close functions for the compaction files, which will be
@@ -1396,41 +1355,12 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
      last_cpu_micros = cur_cpu_micros;
    }

-    const auto& ikey = c_iter->ikey();
-    bool use_penultimate_output = ikey.sequence > penultimate_after_seqno_;
-#ifndef NDEBUG
-    if (sub_compact->compaction->SupportsPerKeyPlacement()) {
-      // Could be overridden by unittest
-      PerKeyPlacementContext context(sub_compact->compaction->output_level(),
-                                     ikey.user_key, c_iter->value(),
-                                     ikey.sequence, use_penultimate_output);
-      TEST_SYNC_POINT_CALLBACK("CompactionIterator::PrepareOutput.context",
-                               &context);
-      if (use_penultimate_output) {
-        // Verify that entries sent to the penultimate level are within the
-        // allowed range (because the input key range of the last level could
-        // be larger than the allowed output key range of the penultimate
-        // level). This check uses user keys (ignores sequence numbers) because
-        // compaction boundaries are a "clean cut" between user keys (see
-        // CompactionPicker::ExpandInputsToCleanCut()), which is especially
-        // important when preferred sequence numbers has been swapped in for
-        // kTypeValuePreferredSeqno / TimedPut.
-        sub_compact->compaction->TEST_AssertWithinPenultimateLevelOutputRange(
-            c_iter->user_key());
-      }
-    } else {
-      assert(penultimate_after_seqno_ == kMaxSequenceNumber);
-      assert(!use_penultimate_output);
-    }
-#endif  // NDEBUG
-
    // Add current compaction_iterator key to target compaction output, if the
    // output file needs to be close or open, it will call the `open_file_func`
    // and `close_file_func`.
    // TODO: it would be better to have the compaction file open/close moved
    // into `CompactionOutputs` which has the output file information.
-    status = sub_compact->AddToOutput(*c_iter, use_penultimate_output,
-                                      open_file_func, close_file_func);
+    status = sub_compact->AddToOutput(*c_iter, open_file_func, close_file_func);
    if (!status.ok()) {
      break;
    }
@@ -1637,31 +1567,16 @@ Status CompactionJob::FinishCompactionOutputFile(
    earliest_snapshot = existing_snapshots_[0];
  }
  if (s.ok()) {
-    // Inclusive lower bound, exclusive upper bound
-    std::pair<SequenceNumber, SequenceNumber> keep_seqno_range{
-        0, kMaxSequenceNumber};
-    if (sub_compact->compaction->SupportsPerKeyPlacement()) {
-      if (outputs.IsPenultimateLevel()) {
-        keep_seqno_range.first = penultimate_after_seqno_;
-      } else {
-        keep_seqno_range.second = penultimate_after_seqno_;
-      }
-    }
    CompactionIterationStats range_del_out_stats;
-    // NOTE1: Use `bottommost_level_ = true` for both bottommost and
+    // if the compaction supports per_key_placement, only output range dels to
+    // the penultimate level.
+    // Note: Use `bottommost_level_ = true` for both bottommost and
    // output_to_penultimate_level compaction here, as it's only used to decide
-    // if range dels could be dropped. (Logically, we are taking a single sorted
-    // run returned from CompactionIterator and physically splitting it between
-    // two output levels.)
-    // NOTE2: with per-key placement, range tombstones will be filtered on
-    // each output level based on sequence number (traversed twice). This is
-    // CPU-inefficient for a large number of range tombstones, but that would
-    // be an unusual work load.
-    if (sub_compact->HasRangeDel()) {
-      s = outputs.AddRangeDels(*sub_compact->RangeDelAgg(), comp_start_user_key,
-                               comp_end_user_key, range_del_out_stats,
-                               bottommost_level_, cfd->internal_comparator(),
-                               earliest_snapshot, keep_seqno_range,
+    // if range dels could be dropped.
+    if (outputs.HasRangeDel()) {
+      s = outputs.AddRangeDels(comp_start_user_key, comp_end_user_key,
+                               range_del_out_stats, bottommost_level_,
+                               cfd->internal_comparator(), earliest_snapshot,
                               next_table_min_key, full_history_ts_low_);
    }
    RecordDroppedKeys(range_del_out_stats, &sub_compact->compaction_job_stats);
@@ -1724,7 +1639,7 @@ Status CompactionJob::FinishCompactionOutputFile(
    // This happens when the output level is bottom level, at the same time
    // the sub_compact output nothing.
    std::string fname =
-        TableFileName(sub_compact->compaction->immutable_options().cf_paths,
+        TableFileName(sub_compact->compaction->immutable_options()->cf_paths,
                      meta->fd.GetNumber(), meta->fd.GetPathId());

    // TODO(AR) it is not clear if there are any larger implications if
@@ -1771,7 +1686,7 @@ Status CompactionJob::FinishCompactionOutputFile(
    }
  }
  EventHelpers::LogAndNotifyTableFileCreationFinished(
-      event_logger_, cfd->ioptions().listeners, dbname_, cfd->GetName(), fname,
+      event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname,
      job_id_, output_fd, oldest_blob_file_number, tp,
      TableFileCreationReason::kCompaction, status_for_listener, file_checksum,
      file_checksum_func_name);
@@ -1799,7 +1714,8 @@ Status CompactionJob::FinishCompactionOutputFile(
  return s;
 }

-Status CompactionJob::InstallCompactionResults(bool* compaction_released) {
+Status CompactionJob::InstallCompactionResults(
+    const MutableCFOptions& mutable_cf_options, bool* compaction_released) {
  assert(compact_);

  db_mutex_->AssertHeld();
@@ -1873,7 +1789,7 @@ Status CompactionJob::InstallCompactionResults(bool* compaction_released) {
  if ((compaction->compaction_reason() ==
           CompactionReason::kLevelMaxLevelSize ||
       compaction->compaction_reason() == CompactionReason::kRoundRobinTtl) &&
-      compaction->immutable_options().compaction_pri == kRoundRobin) {
+      compaction->immutable_options()->compaction_pri == kRoundRobin) {
    int start_level = compaction->start_level();
    if (start_level > 0) {
      auto vstorage = compaction->input_version()->storage_info();
@@ -1888,11 +1804,11 @@ Status CompactionJob::InstallCompactionResults(bool* compaction_released) {
    *compaction_released = true;
  };

-  return versions_->LogAndApply(compaction->column_family_data(), read_options,
-                                write_options, edit, db_mutex_, db_directory_,
-                                /*new_descriptor_log=*/false,
-                                /*column_family_options=*/nullptr,
-                                manifest_wcb);
+  return versions_->LogAndApply(
+      compaction->column_family_data(), mutable_cf_options, read_options,
+      write_options, edit, db_mutex_, db_directory_,
+      /*new_descriptor_log=*/false,
+      /*column_family_options=*/nullptr, manifest_wcb);
 }

 void CompactionJob::RecordCompactionIOStats() {
@@ -1928,7 +1844,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
  // Fire events.
  ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
  EventHelpers::NotifyTableFileCreationStarted(
-      cfd->ioptions().listeners, dbname_, cfd->GetName(), fname, job_id_,
+      cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname, job_id_,
      TableFileCreationReason::kCompaction);
  // Make the output file
  std::unique_ptr<FSWritableFile> writable_file;
@@ -1942,12 +1858,12 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
  FileOptions fo_copy = file_options_;
  Temperature temperature = sub_compact->compaction->output_temperature();
  Temperature last_level_temp =
-      sub_compact->compaction->mutable_cf_options().last_level_temperature;
+      sub_compact->compaction->mutable_cf_options()->last_level_temperature;
  // Here last_level_temperature supersedes default_write_temperature, when
  // enabled and applicable
  if (last_level_temp != Temperature::kUnknown &&
      sub_compact->compaction->is_last_level() &&
-      !outputs.IsPenultimateLevel()) {
+      !sub_compact->IsCurrentPenultimateLevel()) {
    temperature = last_level_temp;
  }
  fo_copy.temperature = temperature;
@@ -1970,7 +1886,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
        job_id_, file_number, s.ToString().c_str());
    LogFlush(db_options_.info_log);
    EventHelpers::LogAndNotifyTableFileCreationFinished(
-        event_logger_, cfd->ioptions().listeners, dbname_, cfd->GetName(),
+        event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(),
        fname, job_id_, FileDescriptor(), kInvalidBlobFileNumber,
        TableProperties(), TableFileCreationReason::kCompaction, s,
        kUnknownFileChecksum, kUnknownFileChecksumFuncName);
@@ -2041,7 +1957,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
  writable_file->SetPreallocationBlockSize(static_cast<size_t>(
      sub_compact->compaction->OutputFilePreallocationSize()));
  const auto& listeners =
-      sub_compact->compaction->immutable_options().listeners;
+      sub_compact->compaction->immutable_options()->listeners;
  outputs.AssignFileWriter(new WritableFileWriter(
      std::move(writable_file), fname, fo_copy, db_options_.clock, io_tracer_,
      db_options_.stats, Histograms::SST_WRITE_MICROS, listeners,
@@ -2052,7 +1968,7 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
  const ReadOptions read_options(Env::IOActivity::kCompaction);
  const WriteOptions write_options(Env::IOActivity::kCompaction);
  TableBuilderOptions tboptions(
-      cfd->ioptions(), sub_compact->compaction->mutable_cf_options(),
+      *cfd->ioptions(), *(sub_compact->compaction->mutable_cf_options()),
      read_options, write_options, cfd->internal_comparator(),
      cfd->internal_tbl_prop_coll_factories(),
      sub_compact->compaction->output_compression(),
@@ -2061,7 +1977,9 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
      bottommost_level_, TableFileCreationReason::kCompaction,
      0 /* oldest_key_time */, current_time, db_id_, db_session_id_,
      sub_compact->compaction->max_output_file_size(), file_number,
-      penultimate_after_seqno_ /*last_level_inclusive_max_seqno_threshold*/);
+      preclude_last_level_min_seqno_ == kMaxSequenceNumber
+          ? preclude_last_level_min_seqno_
+          : std::min(earliest_snapshot_, preclude_last_level_min_seqno_));

  outputs.NewBuilder(tboptions);

@@ -2118,8 +2036,9 @@ bool CompactionJob::UpdateCompactionStats(uint64_t* num_input_range_del) {
      if (file_input_entries == 0) {
        uint64_t file_number = file_meta->fd.GetNumber();
        // Try getting info from table property
-        std::string fn = TableFileName(compaction->immutable_options().cf_paths,
-                                       file_number, file_meta->fd.GetPathId());
+        std::string fn =
+            TableFileName(compaction->immutable_options()->cf_paths,
+                          file_number, file_meta->fd.GetPathId());
        const auto& tp = input_table_properties.find(fn);
        if (tp != input_table_properties.end()) {
          file_input_entries = tp->second->num_entries;
@@ -2201,6 +2120,7 @@ void CompactionJob::UpdateCompactionJobStats(
 void CompactionJob::LogCompaction() {
  Compaction* compaction = compact_->compaction;
  ColumnFamilyData* cfd = compaction->column_family_data();
+
  // Let's check if anything will get logged. Don't prepare all the info if
  // we're not logging
  if (db_options_.info_log_level <= InfoLogLevel::INFO_LEVEL) {
@@ -2215,7 +2135,8 @@ void CompactionJob::LogCompaction() {
                   cfd->GetName().c_str(), scratch);
    // build event logger report
    auto stream = event_logger_->Log();
-    stream << "job" << job_id_ << "event" << "compaction_started"
+    stream << "job" << job_id_ << "event"
+           << "compaction_started"
           << "compaction_reason"
           << GetCompactionReasonString(compaction->compaction_reason());
    for (size_t i = 0; i < compaction->num_input_levels(); ++i) {
@@ -2232,8 +2153,8 @@ void CompactionJob::LogCompaction() {
                   ? int64_t{-1}  // Use -1 for "none"
                   : static_cast<int64_t>(existing_snapshots_[0]));
    if (compaction->SupportsPerKeyPlacement()) {
-      stream << "prenultimate_after_seqno" << penultimate_after_seqno_;
-      stream << "preserve_seqno_after" << preserve_seqno_after_;
+      stream << "preclude_last_level_min_seqno"
+             << preclude_last_level_min_seqno_;
      stream << "penultimate_output_level" << compaction->GetPenultimateLevel();
      stream << "penultimate_output_range"
             << GetCompactionPenultimateOutputRangeTypeString(
@@ -2252,7 +2173,7 @@ void CompactionJob::LogCompaction() {
 }

 std::string CompactionJob::GetTableFileName(uint64_t file_number) {
-  return TableFileName(compact_->compaction->immutable_options().cf_paths,
+  return TableFileName(compact_->compaction->immutable_options()->cf_paths,
                       file_number, compact_->compaction->output_path_id());
 }

@@ -176,13 +176,7 @@ class CompactionJob {

  // REQUIRED: mutex held
  // Prepare for the compaction by setting up boundaries for each subcompaction
-  // and organizing seqno <-> time info. `known_single_subcompact` is non-null
-  // if we already have a known single subcompaction, with optional key bounds
-  // (currently for executing a remote compaction).
-  void Prepare(
-      std::optional<std::pair<std::optional<Slice>, std::optional<Slice>>>
-          known_single_subcompact);
-
+  void Prepare();
  // REQUIRED mutex not held
  // Launch threads for each subcompaction and wait for them to finish. After
  // that, verify table is usable and finally do bookkeeping to unify
@@ -193,7 +187,8 @@ class CompactionJob {
  // Add compaction input/output to the current version
  // Releases compaction file through Compaction::ReleaseCompactionFiles().
  // Sets *compaction_released to true if compaction is released.
-  Status Install(bool* compaction_released);
+  Status Install(const MutableCFOptions& mutable_cf_options,
+                 bool* compaction_released);

  // Return the IO status
  IOStatus io_status() const { return io_status_; }
@@ -281,7 +276,8 @@ class CompactionJob {
                                    const Slice& next_table_min_key,
                                    const Slice* comp_start_user_key,
                                    const Slice* comp_end_user_key);
-  Status InstallCompactionResults(bool* compaction_released);
+  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options,
+                                  bool* compaction_released);
  Status OpenCompactionOutputFile(SubcompactionState* sub_compact,
                                  CompactionOutputs& outputs);

@@ -357,14 +353,15 @@ class CompactionJob {
  // it also collects the smallest_seqno -> oldest_ancester_time from the SST.
  SeqnoToTimeMapping seqno_to_time_mapping_;

-  // Max seqno that can be zeroed out in last level, including for preserving
-  // write times.
-  SequenceNumber preserve_seqno_after_ = kMaxSequenceNumber;
+  // Minimal sequence number for preserving the time information. The time info
+  // older than this sequence number won't be preserved after the compaction and
+  // if it's bottommost compaction, the seq num will be zeroed out.
+  SequenceNumber preserve_time_min_seqno_ = kMaxSequenceNumber;

  // Minimal sequence number to preclude the data from the last level. If the
  // key has bigger (newer) sequence number than this, it will be precluded from
  // the last level (output to penultimate level).
-  SequenceNumber penultimate_after_seqno_ = kMaxSequenceNumber;
+  SequenceNumber preclude_last_level_min_seqno_ = kMaxSequenceNumber;

  // Get table file name in where it's outputting to, which should also be in
  // `output_directory_`.
@@ -499,10 +496,6 @@ class CompactionServiceCompactionJob : private CompactionJob {
      const CompactionServiceInput& compaction_service_input,
      CompactionServiceResult* compaction_service_result);

-  // REQUIRED: mutex held
-  // Like CompactionJob::Prepare()
-  void Prepare();
-
  // Run the compaction in current thread and return the result
  Status Run();

@@ -959,6 +959,7 @@ int main(int argc, char** argv) {
  return RUN_ALL_TESTS();
 }

+
 #else

 int main(int /*argc*/, char** /*argv*/) { return 0; }
@@ -3,6 +3,7 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

+
 #include "db/compaction/compaction_job.h"

 #include <algorithm>
@@ -301,7 +302,7 @@ class CompactionJobTestBase : public testing::Test {
    std::unique_ptr<TableBuilder> table_builder(
        cf_options_.table_factory->NewTableBuilder(
            TableBuilderOptions(
-                cfd_->ioptions(), mutable_cf_options_, read_options,
+                *cfd_->ioptions(), mutable_cf_options_, read_options,
                write_options, cfd_->internal_comparator(),
                cfd_->internal_tbl_prop_coll_factories(),
                CompressionType::kNoCompression, CompressionOptions(),
@@ -397,8 +398,8 @@ class CompactionJobTestBase : public testing::Test {

    mutex_.Lock();
    EXPECT_OK(versions_->LogAndApply(
-        versions_->GetColumnFamilySet()->GetDefault(), read_options_,
-        write_options_, &edit, &mutex_, nullptr));
+        versions_->GetColumnFamilySet()->GetDefault(), mutable_cf_options_,
+        read_options_, write_options_, &edit, &mutex_, nullptr));
    mutex_.Unlock();
  }

@@ -460,7 +461,7 @@ class CompactionJobTestBase : public testing::Test {
      ReadOptions read_opts;
      Status s = cf_options_.table_factory->NewTableReader(
          read_opts,
-          TableReaderOptions(cfd->ioptions(), nullptr, FileOptions(),
+          TableReaderOptions(*cfd->ioptions(), nullptr, FileOptions(),
                             cfd_->internal_comparator(),
                             0 /* block_protection_bytes_per_key */),
          std::move(freader), file_size, &table_reader, false);
@@ -645,12 +646,12 @@ class CompactionJobTestBase : public testing::Test {
    }

    Compaction compaction(
-        cfd->current()->storage_info(), cfd->ioptions(),
-        cfd->GetLatestMutableCFOptions(), mutable_db_options_,
+        cfd->current()->storage_info(), *cfd->ioptions(),
+        *cfd->GetLatestMutableCFOptions(), mutable_db_options_,
        compaction_input_files, output_level,
        mutable_cf_options_.target_file_size_base,
        mutable_cf_options_.max_compaction_bytes, 0, kNoCompression,
-        cfd->GetLatestMutableCFOptions().compression_opts,
+        cfd->GetLatestMutableCFOptions()->compression_opts,
        Temperature::kUnknown, max_subcompactions, grandparents,
        /*earliest_snapshot*/ std::nullopt, /*snapshot_checker*/ nullptr, true);
    compaction.FinalizeInputInfo(cfd->current());
@@ -677,14 +678,15 @@ class CompactionJobTestBase : public testing::Test {
        full_history_ts_low_);
    VerifyInitializationOfCompactionJobStats(compaction_job_stats_);

-    compaction_job.Prepare(std::nullopt /*subcompact to be computed*/);
+    compaction_job.Prepare();
    mutex_.Unlock();
    Status s = compaction_job.Run();
    ASSERT_OK(s);
    ASSERT_OK(compaction_job.io_status());
    mutex_.Lock();
    bool compaction_released = false;
-    ASSERT_OK(compaction_job.Install(&compaction_released));
+    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions(),
+                                     &compaction_released));
    ASSERT_OK(compaction_job.io_status());
    mutex_.Unlock();
    log_buffer.FlushBufferToLog();
@@ -1534,11 +1536,14 @@ TEST_F(CompactionJobTest, VerifyPenultimateLevelOutput) {
      /*verify_func=*/[&](Compaction& comp) {
        for (char c = 'a'; c <= 'z'; c++) {
          if (c == 'a') {
-            comp.TEST_AssertWithinPenultimateLevelOutputRange(
-                "a", true /*expect_failure*/);
+            ParsedInternalKey pik("a", 0U, kTypeValue);
+            ASSERT_FALSE(comp.WithinPenultimateLevelOutputRange(pik));
          } else {
            std::string c_str{c};
-            comp.TEST_AssertWithinPenultimateLevelOutputRange(c_str);
+            // WithinPenultimateLevelOutputRange checks internal key range.
+            // 'z' is the last key, so set seqno properly.
+            ParsedInternalKey pik(c_str, c == 'z' ? 12U : 0U, kTypeValue);
+            ASSERT_TRUE(comp.WithinPenultimateLevelOutputRange(pik));
          }
        }
      });
@@ -1895,8 +1900,8 @@ TEST_F(CompactionJobTest, CutToSkipGrandparentFile) {
  const std::vector<int> input_levels = {0, 1};
  auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
  auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1);
-  RunCompaction({lvl0_files, lvl1_files}, input_levels,
-                {expected_file1, expected_file2});
+    RunCompaction({lvl0_files, lvl1_files}, input_levels,
+                  {expected_file1, expected_file2});
 }

 TEST_F(CompactionJobTest, CutToAlignGrandparentBoundary) {
@@ -1970,8 +1975,8 @@ TEST_F(CompactionJobTest, CutToAlignGrandparentBoundary) {
  const std::vector<int> input_levels = {0, 1};
  auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
  auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1);
-  RunCompaction({lvl0_files, lvl1_files}, input_levels,
-                {expected_file1, expected_file2});
+    RunCompaction({lvl0_files, lvl1_files}, input_levels,
+                  {expected_file1, expected_file2});
 }

 TEST_F(CompactionJobTest, CutToAlignGrandparentBoundarySameKey) {
@@ -2032,8 +2037,8 @@ TEST_F(CompactionJobTest, CutToAlignGrandparentBoundarySameKey) {
  for (int i = 80; i <= 100; i++) {
    snapshots.emplace_back(i);
  }
-  RunCompaction({lvl0_files, lvl1_files}, input_levels,
-                {expected_file1, expected_file2}, snapshots);
+    RunCompaction({lvl0_files, lvl1_files}, input_levels,
+                  {expected_file1, expected_file2}, snapshots);
 }

 TEST_F(CompactionJobTest, CutForMaxCompactionBytesSameKey) {
@@ -320,7 +320,7 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) {
    // More details, check PR #1963
    const size_t num_skippable_boundaries_crossed =
        being_grandparent_gap_ ? 2 : 3;
-    if (compaction_->immutable_options().compaction_style ==
+    if (compaction_->immutable_options()->compaction_style ==
            kCompactionStyleLevel &&
        num_grandparent_boundaries_crossed >=
            num_skippable_boundaries_crossed &&
@@ -341,7 +341,7 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) {
    // target file size. The test shows it can generate larger files than a
    // static threshold like 75% and has a similar write amplification
    // improvement.
-    if (compaction_->immutable_options().compaction_style ==
+    if (compaction_->immutable_options()->compaction_style ==
            kCompactionStyleLevel &&
        current_output_file_size_ >=
            ((compaction_->target_output_file_size() + 99) / 100) *
@@ -455,11 +455,9 @@ void SetMaxSeqAndTs(InternalKey& internal_key, const Slice& user_key,
 }  // namespace

 Status CompactionOutputs::AddRangeDels(
-    CompactionRangeDelAggregator& range_del_agg,
    const Slice* comp_start_user_key, const Slice* comp_end_user_key,
    CompactionIterationStats& range_del_out_stats, bool bottommost_level,
    const InternalKeyComparator& icmp, SequenceNumber earliest_snapshot,
-    std::pair<SequenceNumber, SequenceNumber> keep_seqno_range,
    const Slice& next_table_min_key, const std::string& full_history_ts_low) {
  // The following example does not happen since
  // CompactionOutput::ShouldStopBefore() always return false for the first
@@ -472,7 +470,7 @@ Status CompactionOutputs::AddRangeDels(
  // Then meta.smallest will be set to comp_start_user_key@seqno
  // and meta.largest will be set to comp_start_user_key@kMaxSequenceNumber
  // which violates the assumption that meta.smallest should be <= meta.largest.
-  assert(!range_del_agg.IsEmpty());
+  assert(HasRangeDel());
  FileMetaData& meta = current_output().meta;
  const Comparator* ucmp = icmp.user_comparator();
  InternalKey lower_bound_buf, upper_bound_buf;
@@ -581,19 +579,13 @@ Status CompactionOutputs::AddRangeDels(
  assert(comp_end_user_key == nullptr || upper_bound == nullptr ||
         ucmp->CompareWithoutTimestamp(ExtractUserKey(*upper_bound),
                                       *comp_end_user_key) <= 0);
-  auto it = range_del_agg.NewIterator(lower_bound, upper_bound);
+  auto it = range_del_agg_->NewIterator(lower_bound, upper_bound);
  Slice last_tombstone_start_user_key{};
  bool reached_lower_bound = false;
  const ReadOptions read_options(Env::IOActivity::kCompaction);
  for (it->SeekToFirst(); it->Valid(); it->Next()) {
    auto tombstone = it->Tombstone();
    auto kv = tombstone.Serialize();
-    // Filter out by seqno for per-key placement
-    if (tombstone.seq_ < keep_seqno_range.first ||
-        tombstone.seq_ >= keep_seqno_range.second) {
-      continue;
-    }
-
    InternalKey tombstone_end = tombstone.SerializeEndKey();
    // TODO: the underlying iterator should support clamping the bounds.
    // tombstone_end.Encode is of form user_key@kMaxSeqno
@@ -752,10 +744,11 @@ Status CompactionOutputs::AddRangeDels(
 }

 void CompactionOutputs::FillFilesToCutForTtl() {
-  if (compaction_->immutable_options().compaction_style !=
+  if (compaction_->immutable_options()->compaction_style !=
          kCompactionStyleLevel ||
-      compaction_->immutable_options().compaction_pri != kMinOverlappingRatio ||
-      compaction_->mutable_cf_options().ttl == 0 ||
+      compaction_->immutable_options()->compaction_pri !=
+          kMinOverlappingRatio ||
+      compaction_->mutable_cf_options()->ttl == 0 ||
      compaction_->num_input_levels() < 2 || compaction_->bottommost_level()) {
    return;
  }
@@ -763,19 +756,20 @@ void CompactionOutputs::FillFilesToCutForTtl() {
  // We define new file with the oldest ancestor time to be younger than 1/4
  // TTL, and an old one to be older than 1/2 TTL time.
  int64_t temp_current_time;
-  auto get_time_status = compaction_->immutable_options().clock->GetCurrentTime(
-      &temp_current_time);
+  auto get_time_status =
+      compaction_->immutable_options()->clock->GetCurrentTime(
+          &temp_current_time);
  if (!get_time_status.ok()) {
    return;
  }

  auto current_time = static_cast<uint64_t>(temp_current_time);
-  if (current_time < compaction_->mutable_cf_options().ttl) {
+  if (current_time < compaction_->mutable_cf_options()->ttl) {
    return;
  }

  uint64_t old_age_thres =
-      current_time - compaction_->mutable_cf_options().ttl / 2;
+      current_time - compaction_->mutable_cf_options()->ttl / 2;
  const std::vector<FileMetaData*>& olevel =
      *(compaction_->inputs(compaction_->num_input_levels() - 1));
  for (FileMetaData* file : olevel) {
@@ -785,7 +779,7 @@ void CompactionOutputs::FillFilesToCutForTtl() {
    // of small files.
    if (oldest_ancester_time < old_age_thres &&
        file->fd.GetFileSize() >
-            compaction_->mutable_cf_options().target_file_size_base / 2) {
+            compaction_->mutable_cf_options()->target_file_size_base / 2) {
      files_to_cut_for_ttl_.push_back(file);
    }
  }
@@ -45,8 +45,6 @@ class CompactionOutputs {
  explicit CompactionOutputs(const Compaction* compaction,
                             const bool is_penultimate_level);

-  bool IsPenultimateLevel() const { return is_penultimate_level_; }
-
  // Add generated output to the list
  void AddOutput(FileMetaData&& meta, const InternalKeyComparator& icmp,
                 bool enable_hash, bool finished = false,
@@ -184,13 +182,18 @@ class CompactionOutputs {
  // @param next_table_min_key internal key lower bound for the next compaction
  // output.
  // @param full_history_ts_low used for range tombstone garbage collection.
-  Status AddRangeDels(
-      CompactionRangeDelAggregator& range_del_agg,
-      const Slice* comp_start_user_key, const Slice* comp_end_user_key,
-      CompactionIterationStats& range_del_out_stats, bool bottommost_level,
-      const InternalKeyComparator& icmp, SequenceNumber earliest_snapshot,
-      std::pair<SequenceNumber, SequenceNumber> keep_seqno_range,
-      const Slice& next_table_min_key, const std::string& full_history_ts_low);
+  Status AddRangeDels(const Slice* comp_start_user_key,
+                      const Slice* comp_end_user_key,
+                      CompactionIterationStats& range_del_out_stats,
+                      bool bottommost_level, const InternalKeyComparator& icmp,
+                      SequenceNumber earliest_snapshot,
+                      const Slice& next_table_min_key,
+                      const std::string& full_history_ts_low);
+
+  // if the outputs have range delete, range delete is also data
+  bool HasRangeDel() const {
+    return range_del_agg_ && !range_del_agg_->IsEmpty();
+  }

 private:
  friend class SubcompactionState;
@@ -256,15 +259,11 @@ class CompactionOutputs {
  // Close the current output. `open_file_func` is needed for creating new file
  // for range-dels only output file.
  Status CloseOutput(const Status& curr_status,
-                     CompactionRangeDelAggregator* range_del_agg,
                     const CompactionFileOpenFunc& open_file_func,
                     const CompactionFileCloseFunc& close_file_func) {
    Status status = curr_status;
-    // Handle subcompaction containing only range deletions. They could
-    // be dropped or sent to another output level, so this is only an
-    // over-approximate check for whether opening is needed.
-    if (status.ok() && !HasBuilder() && !HasOutput() && range_del_agg &&
-        !range_del_agg->IsEmpty()) {
+    // handle subcompaction containing only range deletions
+    if (status.ok() && !HasBuilder() && !HasOutput() && HasRangeDel()) {
      status = open_file_func(*this);
    }
    if (HasBuilder()) {
@@ -289,6 +288,16 @@ class CompactionOutputs {
    return outputs_.back();
  }

+  // Assign the range_del_agg to the target output level. There's only one
+  // range-del-aggregator per compaction outputs, for
+  // output_to_penultimate_level compaction it is only assigned to the
+  // penultimate level.
+  void AssignRangeDelAggregator(
+      std::unique_ptr<CompactionRangeDelAggregator>&& range_del_agg) {
+    assert(range_del_agg_ == nullptr);
+    range_del_agg_ = std::move(range_del_agg);
+  }
+
  const Compaction* compaction_;

  // current output builder and writer
@@ -310,6 +319,7 @@ class CompactionOutputs {
  // indicate if this CompactionOutputs obj for penultimate_level, should always
  // be false if per_key_placement feature is not enabled.
  const bool is_penultimate_level_;
+  std::unique_ptr<CompactionRangeDelAggregator> range_del_agg_ = nullptr;

  // partitioner information
  std::string last_key_for_partitioner_;
@@ -721,14 +721,14 @@ Compaction* CompactionPicker::CompactRange(
  // two files overlap.
  if (input_level > 0) {
    const uint64_t limit = mutable_cf_options.max_compaction_bytes;
+    uint64_t input_level_total = 0;
    int hint_index = -1;
-    assert(!inputs.empty());
-    // Always include first file for progress.
-    uint64_t input_level_total = inputs[0]->fd.GetFileSize();
-    InternalKey* smallest = &(inputs[0]->smallest);
+    InternalKey* smallest = nullptr;
    InternalKey* largest = nullptr;
-    for (size_t i = 1; i < inputs.size(); ++i) {
-      // Consider whether to include inputs[i]
+    for (size_t i = 0; i + 1 < inputs.size(); ++i) {
+      if (!smallest) {
+        smallest = &inputs[i]->smallest;
+      }
      largest = &inputs[i]->largest;

      uint64_t input_file_size = inputs[i]->fd.GetFileSize();
@@ -744,11 +744,13 @@ Compaction* CompactionPicker::CompactRange(

      input_level_total += input_file_size;

-      if (input_level_total + output_level_total > limit) {
-        // To ensure compaction size is <= limit, leave out inputs from
-        // index i onwards.
+      if (input_level_total + output_level_total >= limit) {
        covering_the_whole_range = false;
-        inputs.files.resize(i);
+        // still include the current file, so the compaction could be larger
+        // than max_compaction_bytes, which is also to make sure the compaction
+        // can make progress even `max_compaction_bytes` is small (e.g. smaller
+        // than an SST file).
+        inputs.files.resize(i + 1);
        break;
      }
    }
@@ -817,9 +819,6 @@ Compaction* CompactionPicker::CompactRange(
    output_level = vstorage->base_level();
    assert(output_level > 0);
  }
-  for (int i = input_level + 1; i < output_level; i++) {
-    assert(vstorage->NumLevelFiles(i) == 0);
-  }
  output_level_inputs.level = output_level;
  if (input_level != output_level) {
    int parent_index = -1;
@@ -73,8 +73,6 @@ class CompactionPicker {
  // compaction_end will be set to nullptr.
  // Client is responsible for compaction_end storage -- when called,
  // *compaction_end should point to valid InternalKey!
-  // REQUIRES: If not compacting all levels (input_level == kCompactAllLevels),
-  // then levels between input_level and output_level should be empty.
  virtual Compaction* CompactRange(
      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
      const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
@@ -2656,37 +2656,6 @@ TEST_F(CompactionPickerTest, HitCompactionLimitWhenAddFileFromInputLevel) {
  ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
 }

-TEST_F(CompactionPickerTest, CompactRangeMaxCompactionBytes) {
-  mutable_cf_options_.max_compaction_bytes = 800000U;
-  NewVersionStorage(6, kCompactionStyleLevel);
-  // We will first pick file 1 and 2 and then stop before file 3.
-  // Since picking file 3 will pull in file 4 and 5 from L2 and
-  // exceed max_compaction_bytes.
-  Add(1, 1U, "100", "110", 10000U);
-  Add(1, 2U, "200", "210", 10000U, 0, 0);
-  Add(1, 3U, "400", "410", 10000U, 0, 0);
-  Add(2, 4U, "300", "310", 400000U);
-  Add(2, 5U, "320", "330", 400000U);
-  UpdateVersionStorageInfo();
-
-  bool manual_conflict = false;
-  InternalKey manual_end;
-  InternalKey* manual_end_ptr = &manual_end;
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.CompactRange(
-      cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
-      /*input_level=*/1, /*output_level=*/2,
-      /*compact_range_options*/ {}, /*begin=*/nullptr, /*end=*/nullptr,
-      &manual_end_ptr, &manual_conflict,
-      /*max_file_num_to_ignore=*/std::numeric_limits<uint64_t>::max(),
-      /*trim_ts=*/""));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_levels());
-  ASSERT_EQ(2, compaction->output_level());
-  ASSERT_EQ(2U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
-}
-
 TEST_F(CompactionPickerTest, IsTrivialMoveOn) {
  mutable_cf_options_.max_bytes_for_level_base = 10000u;
  mutable_cf_options_.max_compaction_bytes = 10001u;
@@ -177,7 +177,7 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(
  for (const auto& file : compaction_result.output_files) {
    uint64_t file_num = versions_->NewFileNumber();
    auto src_file = compaction_result.output_path + "/" + file.file_name;
-    auto tgt_file = TableFileName(compaction->immutable_options().cf_paths,
+    auto tgt_file = TableFileName(compaction->immutable_options()->cf_paths,
                                  file_num, compaction->output_path_id());
    s = fs_->RenameFile(src_file, tgt_file, IOOptions(), nullptr);
    if (!s.ok()) {
@@ -189,10 +189,6 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(

    FileMetaData meta;
    uint64_t file_size;
-    // FIXME: file_size should be part of CompactionServiceOutputFile so that
-    // we don't get DB corruption if the full file size has not been propagated
-    // back to the caller through the file system (which could have metadata
-    // lag or caching bugs).
    s = fs_->GetFileSize(tgt_file, IOOptions(), &file_size, nullptr);
    if (!s.ok()) {
      sub_compact->status = s;
@@ -269,38 +265,43 @@ CompactionServiceCompactionJob::CompactionServiceCompactionJob(
    std::string output_path,
    const CompactionServiceInput& compaction_service_input,
    CompactionServiceResult* compaction_service_result)
-    : CompactionJob(job_id, compaction, db_options, mutable_db_options,
-                    file_options, versions, shutting_down, log_buffer, nullptr,
-                    output_directory, nullptr, stats, db_mutex,
-                    db_error_handler, std::move(existing_snapshots),
-                    kMaxSequenceNumber, nullptr, nullptr,
-                    std::move(table_cache), event_logger,
-                    compaction->mutable_cf_options().paranoid_file_checks,
-                    compaction->mutable_cf_options().report_bg_io_stats, dbname,
-                    &(compaction_service_result->stats), Env::Priority::USER,
-                    io_tracer, manual_compaction_canceled, db_id, db_session_id,
-                    compaction->column_family_data()->GetFullHistoryTsLow()),
+    : CompactionJob(
+          job_id, compaction, db_options, mutable_db_options, file_options,
+          versions, shutting_down, log_buffer, nullptr, output_directory,
+          nullptr, stats, db_mutex, db_error_handler,
+          std::move(existing_snapshots), kMaxSequenceNumber, nullptr, nullptr,
+          std::move(table_cache), event_logger,
+          compaction->mutable_cf_options()->paranoid_file_checks,
+          compaction->mutable_cf_options()->report_bg_io_stats, dbname,
+          &(compaction_service_result->stats), Env::Priority::USER, io_tracer,
+          manual_compaction_canceled, db_id, db_session_id,
+          compaction->column_family_data()->GetFullHistoryTsLow()),
      output_path_(std::move(output_path)),
      compaction_input_(compaction_service_input),
      compaction_result_(compaction_service_result) {}

-void CompactionServiceCompactionJob::Prepare() {
-  std::optional<Slice> begin;
-  if (compaction_input_.has_begin) {
-    begin = compaction_input_.begin;
-  }
-  std::optional<Slice> end;
-  if (compaction_input_.has_end) {
-    end = compaction_input_.end;
-  }
-  CompactionJob::Prepare(std::make_pair(begin, end));
-}
-
 Status CompactionServiceCompactionJob::Run() {
  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_COMPACTION_RUN);

  auto* c = compact_->compaction;
+  assert(c->column_family_data() != nullptr);
+  const VersionStorageInfo* storage_info = c->input_version()->storage_info();
+  assert(storage_info);
+  assert(storage_info->NumLevelFiles(compact_->compaction->level()) > 0);
+  write_hint_ = storage_info->CalculateSSTWriteHint(c->output_level());
+
+  bottommost_level_ = c->bottommost_level();
+
+  Slice begin = compaction_input_.begin;
+  Slice end = compaction_input_.end;
+  compact_->sub_compact_states.emplace_back(
+      c,
+      compaction_input_.has_begin ? std::optional<Slice>(begin)
+                                  : std::optional<Slice>(),
+      compaction_input_.has_end ? std::optional<Slice>(end)
+                                : std::optional<Slice>(),
+      /*sub_job_id*/ 0);

  log_buffer_->FlushBufferToLog();
  LogCompaction();
@@ -114,8 +114,6 @@ class MyTestCompactionService : public CompactionService {
    }
  }

-  void CancelAwaitingJobs() override { canceled_ = true; }
-
  void OnInstallation(const std::string& /*scheduled_job_id*/,
                      CompactionServiceJobStatus status) override {
    final_updated_status_ = status;
@@ -148,7 +146,6 @@ class MyTestCompactionService : public CompactionService {
  }

  void SetCanceled(bool canceled) { canceled_ = canceled; }
-  bool GetCanceled() { return canceled_; }

  void GetResult(CompactionServiceResult* deserialized) {
    CompactionServiceResult::Read(result_, deserialized).PermitUncheckedError();
@@ -197,15 +194,12 @@ class CompactionServiceTest : public DBTestBase {
    options->statistics = primary_statistics_;
    compactor_statistics_ = CreateDBStatistics();

-    auto my_cs = std::make_shared<MyTestCompactionService>(
+    compaction_service_ = std::make_shared<MyTestCompactionService>(
        dbname_, *options, compactor_statistics_, remote_listeners,
        remote_table_properties_collector_factories);
-
-    compaction_service_ = my_cs;
    options->compaction_service = compaction_service_;
    DestroyAndReopen(*options);
    CreateAndReopenWithCF({"cf_1", "cf_2", "cf_3"}, *options);
-    my_cs->SetCanceled(false);
  }

  Statistics* GetCompactorStatistics() { return compactor_statistics_.get(); }
@@ -345,8 +339,6 @@ TEST_F(CompactionServiceTest, BasicCompactions) {
        assert(*id != kNullUniqueId64x2);
        verify_passed++;
      });
-  Close();
-  my_cs->SetCanceled(false);
  ReopenWithColumnFamilies({kDefaultColumnFamilyName, "cf_1", "cf_2", "cf_3"},
                           options);
  ASSERT_GT(verify_passed, 0);
@@ -495,7 +487,7 @@ TEST_F(CompactionServiceTest, PreservedOptionsLocalCompaction) {
        ASSERT_TRUE(s.IsNotFound());
        // Should be old value
        ASSERT_EQ(2, compaction->mutable_cf_options()
-                         .level0_file_num_compaction_trigger);
+                         ->level0_file_num_compaction_trigger);
        ASSERT_TRUE(dbfull()->min_options_file_numbers_.empty());
      });

@@ -581,7 +573,7 @@ TEST_F(CompactionServiceTest, PreservedOptionsRemoteCompaction) {
      "CompactionJob::ProcessKeyValueCompaction()::Processing", [&](void* arg) {
        auto compaction = static_cast<Compaction*>(arg);
        ASSERT_EQ(2, compaction->mutable_cf_options()
-                         .level0_file_num_compaction_trigger);
+                         ->level0_file_num_compaction_trigger);
      });

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
@@ -1004,41 +996,6 @@ TEST_F(CompactionServiceTest, CancelCompactionOnRemoteSide) {
  VerifyTestData();
 }

-TEST_F(CompactionServiceTest, CancelCompactionOnPrimarySide) {
-  Options options = CurrentOptions();
-  options.disable_auto_compactions = true;
-  ReopenWithCompactionService(&options);
-  GenerateTestData();
-
-  auto my_cs = GetCompactionService();
-
-  std::string start_str = Key(15);
-  std::string end_str = Key(45);
-  Slice start(start_str);
-  Slice end(end_str);
-  uint64_t comp_num = my_cs->GetCompactionNum();
-
-  ReopenWithCompactionService(&options);
-  GenerateTestData();
-  my_cs = GetCompactionService();
-
-  // Primary DB calls CancelAllBackgroundWork() while the compaction is running
-  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionJob::Run():Inprogress",
-      [&](void* /*arg*/) { CancelAllBackgroundWork(db_, false /*wait*/); });
-
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  Status s = db_->CompactRange(CompactRangeOptions(), &start, &end);
-  ASSERT_TRUE(s.IsIncomplete());
-
-  // Check canceled_ was set to true by CancelAwaitingJobs()
-  ASSERT_TRUE(my_cs->GetCanceled());
-
-  // compaction number is not increased
-  ASSERT_GE(my_cs->GetCompactionNum(), comp_num);
-}
-
 TEST_F(CompactionServiceTest, FailedToStart) {
  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
@@ -1161,63 +1118,6 @@ TEST_F(CompactionServiceTest, Snapshot) {
  db_->ReleaseSnapshot(s1);
 }

-TEST_F(CompactionServiceTest, PrecludeLastLevel) {
-  const int kNumTrigger = 4;
-  const int kNumLevels = 7;
-  const int kNumKeys = 100;
-
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleUniversal;
-  options.last_level_temperature = Temperature::kCold;
-  options.level0_file_num_compaction_trigger = 4;
-  options.max_subcompactions = 10;
-  options.num_levels = kNumLevels;
-
-  ReopenWithCompactionService(&options);
-  // Alternate for comparison: DestroyAndReopen(options);
-
-  // This is simpler than setting up mock time to make the user option work,
-  // but is not as direct as testing with preclude option itself.
-  SyncPoint::GetInstance()->SetCallBack(
-      "Compaction::SupportsPerKeyPlacement:Enabled",
-      [&](void* arg) { *static_cast<bool*>(arg) = true; });
-  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
-      [&](void* arg) { *static_cast<SequenceNumber*>(arg) = 100; });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  for (int i = 0; i < kNumTrigger; i++) {
-    for (int j = 0; j < kNumKeys; j++) {
-      // FIXME: need to assign outputs to levels to allow overlapping ranges:
-      // ASSERT_OK(Put(Key(j * kNumTrigger + i), "v" + std::to_string(i)));
-      // instead of this (too easy):
-      ASSERT_OK(Put(Key(i * kNumKeys + j), "v" + std::to_string(i)));
-    }
-    ASSERT_OK(Flush());
-  }
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-
-  // Data split between penultimate (kUnknown) and last (kCold) levels
-  // FIXME: need to assign outputs to levels to get this:
-  // ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
-  // ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
-  // ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
-  // instead of this (WRONG but currently expected):
-  ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel());
-  // Check manifest temperatures
-  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
-  // TODO: Check FileSystem temperatures with FileTemperatureTestFS
-
-  for (int i = 0; i < kNumTrigger; i++) {
-    for (int j = 0; j < kNumKeys; j++) {
-      // FIXME
-      // ASSERT_EQ(Get(Key(j * kNumTrigger + i)), "v" + std::to_string(i));
-      ASSERT_EQ(Get(Key(i * kNumKeys + j)), "v" + std::to_string(i));
-    }
-  }
-}
-
 TEST_F(CompactionServiceTest, ConcurrentCompaction) {
  Options options = CurrentOptions();
  options.level0_file_num_compaction_trigger = 100;
@@ -15,19 +15,8 @@
 namespace ROCKSDB_NAMESPACE {
 void SubcompactionState::AggregateCompactionOutputStats(
    InternalStats::CompactionStatsFull& compaction_stats) const {
-  // Outputs should be closed. By extension, any files created just for
-  // range deletes have already been written also.
-  assert(compaction_outputs_.HasBuilder() == false);
-  assert(penultimate_level_outputs_.HasBuilder() == false);
-
-  // FIXME: These stats currently include abandonned output files
-  // assert(compaction_outputs_.stats_.num_output_files ==
-  //        compaction_outputs_.outputs_.size());
-  // assert(penultimate_level_outputs_.stats_.num_output_files ==
-  //        penultimate_level_outputs_.outputs_.size());
-
  compaction_stats.stats.Add(compaction_outputs_.stats_);
-  if (penultimate_level_outputs_.HasOutput()) {
+  if (HasPenultimateLevelOutputs()) {
    compaction_stats.has_penultimate_level_output = true;
    compaction_stats.penultimate_level_stats.Add(
        penultimate_level_outputs_.stats_);
@@ -54,7 +43,7 @@ void SubcompactionState::Cleanup(Cache* cache) {
      // be picked up by scanning the DB directory.
      TableCache::ReleaseObsolete(
          cache, out.meta.fd.GetNumber(), nullptr /*handle*/,
-          compaction->mutable_cf_options().uncache_aggressiveness);
+          compaction->mutable_cf_options()->uncache_aggressiveness);
    }
  }
  // TODO: sub_compact.io_status is not checked like status. Not sure if thats
@@ -63,7 +52,7 @@ void SubcompactionState::Cleanup(Cache* cache) {
 }

 Slice SubcompactionState::SmallestUserKey() const {
-  if (penultimate_level_outputs_.HasOutput()) {
+  if (has_penultimate_level_outputs_) {
    Slice a = compaction_outputs_.SmallestUserKey();
    Slice b = penultimate_level_outputs_.SmallestUserKey();
    if (a.empty()) {
@@ -85,7 +74,7 @@ Slice SubcompactionState::SmallestUserKey() const {
 }

 Slice SubcompactionState::LargestUserKey() const {
-  if (penultimate_level_outputs_.HasOutput()) {
+  if (has_penultimate_level_outputs_) {
    Slice a = compaction_outputs_.LargestUserKey();
    Slice b = penultimate_level_outputs_.LargestUserKey();
    if (a.empty()) {
@@ -107,13 +96,18 @@ Slice SubcompactionState::LargestUserKey() const {
 }

 Status SubcompactionState::AddToOutput(
-    const CompactionIterator& iter, bool use_penultimate_output,
+    const CompactionIterator& iter,
    const CompactionFileOpenFunc& open_file_func,
    const CompactionFileCloseFunc& close_file_func) {
-  // update target output
-  current_outputs_ = use_penultimate_output ? &penultimate_level_outputs_
-                                            : &compaction_outputs_;
-  return current_outputs_->AddToOutput(iter, open_file_func, close_file_func);
+  // update target output first
+  is_current_penultimate_level_ = iter.output_to_penultimate_level();
+  current_outputs_ = is_current_penultimate_level_ ? &penultimate_level_outputs_
+                                                   : &compaction_outputs_;
+  if (is_current_penultimate_level_) {
+    has_penultimate_level_outputs_ = true;
+  }
+
+  return Current().AddToOutput(iter, open_file_func, close_file_func);
 }

 }  // namespace ROCKSDB_NAMESPACE
@@ -81,13 +81,22 @@ class SubcompactionState {
  // it returns both the last level outputs and penultimate level outputs.
  OutputIterator GetOutputs() const;

-  // Assign range dels aggregator. The various tombstones will potentially
-  // be filtered to different outputs.
+  // Assign range dels aggregator, for each range_del, it can only be assigned
+  // to one output level, for per_key_placement, it's going to be the
+  // penultimate level.
+  // TODO: This does not work for per_key_placement + user-defined timestamp +
+  //  DeleteRange() combo. If user-defined timestamp is enabled,
+  //  it is possible for a range tombstone to belong to bottommost level (
+  //  seqno < earliest snapshot) without being dropped (garbage collection
+  //  for user-defined timestamp).
  void AssignRangeDelAggregator(
      std::unique_ptr<CompactionRangeDelAggregator>&& range_del_agg) {
-    assert(range_del_agg_ == nullptr);
-    assert(range_del_agg);
-    range_del_agg_ = std::move(range_del_agg);
+    if (compaction->SupportsPerKeyPlacement()) {
+      penultimate_level_outputs_.AssignRangeDelAggregator(
+          std::move(range_del_agg));
+    } else {
+      compaction_outputs_.AssignRangeDelAggregator(std::move(range_del_agg));
+    }
  }

  void RemoveLastEmptyOutput() {
@@ -142,11 +151,20 @@ class SubcompactionState {
        sub_job_id(state.sub_job_id),
        compaction_outputs_(std::move(state.compaction_outputs_)),
        penultimate_level_outputs_(std::move(state.penultimate_level_outputs_)),
-        range_del_agg_(std::move(state.range_del_agg_)) {
-    current_outputs_ =
-        state.current_outputs_ == &state.penultimate_level_outputs_
-            ? &penultimate_level_outputs_
-            : &compaction_outputs_;
+        is_current_penultimate_level_(state.is_current_penultimate_level_),
+        has_penultimate_level_outputs_(state.has_penultimate_level_outputs_) {
+    current_outputs_ = is_current_penultimate_level_
+                           ? &penultimate_level_outputs_
+                           : &compaction_outputs_;
+  }
+
+  bool HasPenultimateLevelOutputs() const {
+    return has_penultimate_level_outputs_ ||
+           penultimate_level_outputs_.HasRangeDel();
+  }
+
+  bool IsCurrentPenultimateLevel() const {
+    return is_current_penultimate_level_;
  }

  // Add all the new files from this compaction to version_edit
@@ -169,18 +187,8 @@ class SubcompactionState {
    return *current_outputs_;
  }

-  CompactionRangeDelAggregator* RangeDelAgg() const {
-    return range_del_agg_.get();
-  }
-
-  // if the outputs have range delete, range delete is also data
-  bool HasRangeDel() const {
-    return range_del_agg_ && !range_del_agg_->IsEmpty();
-  }
-
  // Add compaction_iterator key/value to the `Current` output group.
  Status AddToOutput(const CompactionIterator& iter,
-                     bool use_penultimate_output,
                     const CompactionFileOpenFunc& open_file_func,
                     const CompactionFileCloseFunc& close_file_func);

@@ -189,20 +197,14 @@ class SubcompactionState {
  Status CloseCompactionFiles(const Status& curr_status,
                              const CompactionFileOpenFunc& open_file_func,
                              const CompactionFileCloseFunc& close_file_func) {
-    auto per_key = compaction->SupportsPerKeyPlacement();
    // Call FinishCompactionOutputFile() even if status is not ok: it needs to
    // close the output file.
    // CloseOutput() may open new compaction output files.
-    Status s = curr_status;
-    if (per_key) {
-      s = penultimate_level_outputs_.CloseOutput(
-          s, range_del_agg_.get(), open_file_func, close_file_func);
-    } else {
-      assert(penultimate_level_outputs_.HasBuilder() == false);
-      assert(penultimate_level_outputs_.HasOutput() == false);
-    }
-    s = compaction_outputs_.CloseOutput(s, range_del_agg_.get(), open_file_func,
-                                        close_file_func);
+    is_current_penultimate_level_ = true;
+    Status s = penultimate_level_outputs_.CloseOutput(
+        curr_status, open_file_func, close_file_func);
+    is_current_penultimate_level_ = false;
+    s = compaction_outputs_.CloseOutput(s, open_file_func, close_file_func);
    return s;
  }

@@ -211,7 +213,8 @@ class SubcompactionState {
  CompactionOutputs compaction_outputs_;
  CompactionOutputs penultimate_level_outputs_;
  CompactionOutputs* current_outputs_ = &compaction_outputs_;
-  std::unique_ptr<CompactionRangeDelAggregator> range_del_agg_;
+  bool is_current_penultimate_level_ = false;
+  bool has_penultimate_level_outputs_ = false;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -195,9 +195,10 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
  std::vector<SequenceNumber> seq_history;

  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
-      [&](void* arg) {
-        *static_cast<SequenceNumber*>(arg) = latest_cold_seq.load();
+      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
+        auto context = static_cast<PerKeyPlacementContext*>(arg);
+        context->output_to_penultimate_level =
+            context->seq_num > latest_cold_seq;
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -338,10 +339,7 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }

-// This test was essentially for a hacked-up version on future functionality.
-// It can be resurrected if/when a form of range-based tiering is properly
-// implemented.
-TEST_F(TieredCompactionTest, DISABLED_RangeBasedTieredStorageUniversal) {
+TEST_F(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -442,12 +440,12 @@ TEST_F(TieredCompactionTest, DISABLED_RangeBasedTieredStorageUniversal) {
  {
    MutexLock l(&mutex);
    hot_start = Key(1);
-    hot_end = Key(300);
+    hot_end = Key(1000);
  }

  // generate files just enough to trigger compaction
  for (int i = 0; i < kNumTrigger - 1; i++) {
-    for (int j = 0; j < 300; j++) {
+    for (int j = 0; j < 1000; j++) {
      ASSERT_OK(Put(Key(j), "value" + std::to_string(j)));
    }
    ASSERT_OK(Flush());
@@ -496,7 +494,6 @@ TEST_F(TieredCompactionTest, DISABLED_RangeBasedTieredStorageUniversal) {
  // verify data
  std::string value;
  for (int i = 0; i < kNumKeys; i++) {
-    SCOPED_TRACE(Key(i));
    if (i < 70) {
      ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound());
    } else {
@@ -530,13 +527,13 @@ TEST_F(TieredCompactionTest, LevelColdRangeDelete) {
  options.max_subcompactions = 10;
  DestroyAndReopen(options);

-  // Initially let everything into cold
-  std::atomic_uint64_t latest_cold_seq = kMaxSequenceNumber;
+  std::atomic_uint64_t latest_cold_seq = 0;

  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
-      [&](void* arg) {
-        *static_cast<SequenceNumber*>(arg) = latest_cold_seq.load();
+      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
+        auto context = static_cast<PerKeyPlacementContext*>(arg);
+        context->output_to_penultimate_level =
+            context->seq_num > latest_cold_seq;
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -637,13 +634,13 @@ TEST_F(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
  options.max_subcompactions = 10;
  DestroyAndReopen(options);

-  // Initially let everything into cold
-  std::atomic_uint64_t latest_cold_seq = kMaxSequenceNumber;
+  std::atomic_uint64_t latest_cold_seq = 0;

  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
-      [&](void* arg) {
-        *static_cast<SequenceNumber*>(arg) = latest_cold_seq.load();
+      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
+        auto context = static_cast<PerKeyPlacementContext*>(arg);
+        context->output_to_penultimate_level =
+            context->seq_num > latest_cold_seq;
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -657,8 +654,6 @@ TEST_F(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
  ASSERT_EQ("0,0,10", FilesPerLevel());

-  // Stop admitting to cold tier
-  latest_cold_seq = dbfull()->GetLatestSequenceNumber();
  auto snap = db_->GetSnapshot();

  // only range delete
@@ -772,9 +767,10 @@ TEST_F(TieredCompactionTest, UniversalRangeDelete) {
  std::atomic_uint64_t latest_cold_seq = 0;

  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
-      [&](void* arg) {
-        *static_cast<SequenceNumber*>(arg) = latest_cold_seq.load();
+      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
+        auto context = static_cast<PerKeyPlacementContext*>(arg);
+        context->output_to_penultimate_level =
+            context->seq_num > latest_cold_seq;
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -898,13 +894,14 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
  options.max_subcompactions = 10;
  DestroyAndReopen(options);

-  std::atomic_uint64_t latest_cold_seq = kMaxSequenceNumber;
+  std::atomic_uint64_t latest_cold_seq = 0;
  std::vector<SequenceNumber> seq_history;

  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionJob::PrepareTimes():preclude_last_level_min_seqno",
-      [&](void* arg) {
-        *static_cast<SequenceNumber*>(arg) = latest_cold_seq.load();
+      "CompactionIterator::PrepareOutput.context", [&](void* arg) {
+        auto context = static_cast<PerKeyPlacementContext*>(arg);
+        context->output_to_penultimate_level =
+            context->seq_num > latest_cold_seq;
      });
  SyncPoint::GetInstance()->EnableProcessing();

@@ -955,7 +952,6 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
  VerifyCompactionStats(expect_stats, expect_pl_stats);

  // Add new data, which is all hot and overriding all existing data
-  latest_cold_seq = dbfull()->GetLatestSequenceNumber();
  for (int i = 0; i < kNumTrigger; i++) {
    for (int j = 0; j < kNumKeys; j++) {
      ASSERT_OK(Put(Key(i * 10 + j), "value" + std::to_string(i)));
@@ -1108,13 +1104,7 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }

-// This test was essentially for a hacked-up version on future functionality.
-// It can be resurrected if/when a form of range-based tiering is properly
-// implemented.
-// FIXME: aside from that, this test reproduces a near-endless compaction
-// cycle that needs to be reproduced independently and fixed before
-// leveled compaction can be used with the preclude feature in production.
-TEST_F(TieredCompactionTest, DISABLED_RangeBasedTieredStorageLevel) {
+TEST_F(TieredCompactionTest, RangeBasedTieredStorageLevel) {
  const int kNumTrigger = 4;
  const int kNumLevels = 7;
  const int kNumKeys = 100;
@@ -1223,9 +1213,6 @@ TEST_F(TieredCompactionTest, DISABLED_RangeBasedTieredStorageLevel) {

  // Tests that we only compact keys up to penultimate level
  // that are within penultimate level input's internal key range.
-  // UPDATE: this functionality has changed. With penultimate-enabled
-  // compaction, the expanded potential output range in the penultimate
-  // level is reserved so should be safe to use.
  {
    MutexLock l(&mutex);
    hot_start = Key(0);
@@ -1234,6 +1221,7 @@ TEST_F(TieredCompactionTest, DISABLED_RangeBasedTieredStorageLevel) {
  const Snapshot* temp_snap = db_->GetSnapshot();
  // Key(0) and Key(1) here are inserted with higher sequence number
  // than Key(0) and Key(1) inserted above.
+  // Only Key(0) in last level will be compacted up, not Key(1).
  ASSERT_OK(Put(Key(0), "value" + std::to_string(0)));
  ASSERT_OK(Put(Key(1), "value" + std::to_string(100)));
  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
@@ -1243,17 +1231,14 @@ TEST_F(TieredCompactionTest, DISABLED_RangeBasedTieredStorageLevel) {
    db_->GetLiveFilesMetaData(&metas);
    for (const auto& f : metas) {
      if (f.temperature == Temperature::kUnknown) {
-        // UPDATED (was 3 entries, Key0..Key1)
-        ASSERT_EQ(f.num_entries, 52);
+        // Expect Key(0), Key(0), Key(1)
+        ASSERT_EQ(f.num_entries, 3);
        ASSERT_EQ(f.smallestkey, Key(0));
-        ASSERT_EQ(f.largestkey, Key(49));
+        ASSERT_EQ(f.largestkey, Key(1));
      } else {
        ASSERT_EQ(f.temperature, Temperature::kCold);
-        // UPDATED (was 50 entries, Key0..Key49)
-        // Key(100) is outside the hot range
-        ASSERT_EQ(f.num_entries, 1);
-        ASSERT_EQ(f.smallestkey, Key(100));
-        ASSERT_EQ(f.largestkey, Key(100));
+        // Key(2)-Key(49) and Key(100).
+        ASSERT_EQ(f.num_entries, 50);
      }
    }
  }
@@ -1294,6 +1279,8 @@ class PrecludeLastLevelTestBase : public DBTestBase {
      const std::unordered_map<std::string, std::string>& db_config_change) {
    if (dynamic) {
      if (config_change.size() > 0) {
+        // FIXME: temporary while preserve/preclude options are not user mutable
+        SaveAndRestore<bool> m(&TEST_allowSetOptionsImmutableInMutable, true);
        ASSERT_OK(db_->SetOptions(config_change));
      }
      if (db_config_change.size() > 0) {
@@ -1629,8 +1616,7 @@ TEST_P(PrecludeLastLevelTest, CheckInternalKeyRange) {

  ASSERT_EQ("0,0,0,0,0,2,2", FilesPerLevel());

-  auto VerifyLogicalState = [&](int line) {
-    SCOPED_TRACE("Called from line " + std::to_string(line));
+  auto VerifyLogicalState = [&]() {
    // First with snapshot
    ASSERT_EQ("val2", Get(Key(2), snapshot));
    ASSERT_EQ("val3", Get(Key(3), snapshot));
@@ -1644,7 +1630,7 @@ TEST_P(PrecludeLastLevelTest, CheckInternalKeyRange) {
    ASSERT_EQ("val100", Get(Key(100)));
  };

-  VerifyLogicalState(__LINE__);
+  VerifyLogicalState();

  // Try to compact keys up
  CompactRangeOptions cro;
@@ -1655,40 +1641,24 @@ TEST_P(PrecludeLastLevelTest, CheckInternalKeyRange) {
  // type:1 vs. file #15 smallest key: '6B6579303030303033' seq:104, type:1
  ASSERT_OK(CompactRange(cro, Key(1), Key(2)));

-  VerifyLogicalState(__LINE__);
+  VerifyLogicalState();

  db_->ReleaseSnapshot(snapshot);
  Close();
 }

-INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTest, PrecludeLastLevelTest,
-                        ::testing::Bool());
-
-class PrecludeWithCompactStyleTest : public PrecludeLastLevelTestBase,
-                                     public testing::WithParamInterface<bool> {
- public:
-  void ApplyConfigChange(
-      Options* options,
-      const std::unordered_map<std::string, std::string>& config_change,
-      const std::unordered_map<std::string, std::string>& db_config_change =
-          {}) {
-    // Depends on dynamic config change while holding a snapshot
-    ApplyConfigChangeImpl(true /*dynamic*/, options, config_change,
-                          db_config_change);
-  }
-};
-
-TEST_P(PrecludeWithCompactStyleTest, RangeTombstoneSnapshotMigrateFromLast) {
+TEST_P(PrecludeLastLevelTest, RangeTombstoneSnapshotMigrateFromLast) {
  // Reproducer for issue originally described in
  // https://github.com/facebook/rocksdb/pull/9964/files#r1024449523
-  const bool universal = GetParam();
+  if (!UseDynamicConfig()) {
+    // Depends on config change while holding a snapshot
+    return;
+  }
  const int kNumLevels = 7;
  auto options = CurrentOptions();
  options.env = mock_env_.get();
  options.last_level_temperature = Temperature::kCold;
-  options.compaction_style =
-      universal ? kCompactionStyleUniversal : kCompactionStyleLevel;
-  options.compaction_options_universal.allow_trivial_move = true;
+  options.level_compaction_dynamic_level_bytes = true;
  options.num_levels = kNumLevels;
  options.statistics = CreateDBStatistics();
  options.max_subcompactions = 10;
@@ -1708,13 +1678,7 @@ TEST_P(PrecludeWithCompactStyleTest, RangeTombstoneSnapshotMigrateFromLast) {
  ASSERT_OK(db_->DeleteRange({}, db_->DefaultColumnFamily(), Key(1), Key(5)));
  ASSERT_OK(Put(Key(1), "val1"));
  ASSERT_OK(Flush());
-
-  // Send to last level
-  if (universal) {
-    ASSERT_OK(CompactRange({}, {}, {}));
-  } else {
-    MoveFilesToLevel(6);
-  }
+  MoveFilesToLevel(6);
  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());

  ApplyConfigChange(&options, {{"preclude_last_level_data_seconds", "10000"}});
@@ -1733,18 +1697,10 @@ TEST_P(PrecludeWithCompactStyleTest, RangeTombstoneSnapshotMigrateFromLast) {
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(1000)); });

  ASSERT_OK(Flush());
-
-  // Send three files to next-to-last level (if explicitly needed)
-  if (universal) {
-    ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  } else {
-    MoveFilesToLevel(5);
-  }
-
+  MoveFilesToLevel(5);
  ASSERT_EQ("0,0,0,0,0,3,1", FilesPerLevel());

-  auto VerifyLogicalState = [&](int line) {
-    SCOPED_TRACE("Called from line " + std::to_string(line));
+  auto VerifyLogicalState = [&]() {
    // First with snapshot
    if (snapshot) {
      ASSERT_EQ("NOT_FOUND", Get(Key(0), snapshot));
@@ -1764,31 +1720,24 @@ TEST_P(PrecludeWithCompactStyleTest, RangeTombstoneSnapshotMigrateFromLast) {
    ASSERT_EQ("val7", Get(Key(7)));
  };

-  VerifyLogicalState(__LINE__);
+  VerifyLogicalState();

  // Try a limited range compaction
-  // (These would previously hit "Unsafe to store Seq later than snapshot")
-  if (universal) {
-    uint64_t middle_l5 = GetLevelFileMetadatas(5)[1]->fd.GetNumber();
-    ASSERT_OK(db_->CompactFiles({}, {MakeTableFileName(middle_l5)}, 6));
-  } else {
-    ASSERT_OK(CompactRange({}, Key(3), Key(4)));
-  }
+  // FIXME: this currently hits the "Unsafe to store Seq later than snapshot"
+  // error. Needs to work safely for preclude option to be user mutable.
+  // ASSERT_OK(CompactRange({}, Key(3), Key(4)));
  EXPECT_EQ("0,0,0,0,0,3,1", FilesPerLevel());
-  VerifyLogicalState(__LINE__);
+  VerifyLogicalState();

  // Compact everything, but some data still goes to both penultimate and last
  // levels. A full-range compaction should be safe to "migrate" data from the
  // last level to penultimate (because of preclude setting change).
  ASSERT_OK(CompactRange({}, {}, {}));
  EXPECT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
-  VerifyLogicalState(__LINE__);
+  VerifyLogicalState();
  // Key1 should have been migrated out of the last level
-  // FIXME: doesn't yet work with leveled compaction
-  if (universal) {
-    auto& meta = *GetLevelFileMetadatas(6)[0];
-    ASSERT_LT(Key(1), meta.smallest.user_key().ToString());
-  }
+  auto& meta = *GetLevelFileMetadatas(6)[0];
+  ASSERT_LT(Key(1), meta.smallest.user_key().ToString());

  // Make data eligible for last level
  db_->ReleaseSnapshot(snapshot);
@@ -1797,13 +1746,13 @@ TEST_P(PrecludeWithCompactStyleTest, RangeTombstoneSnapshotMigrateFromLast) {

  ASSERT_OK(CompactRange({}, {}, {}));
  EXPECT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
-  VerifyLogicalState(__LINE__);
+  VerifyLogicalState();

  Close();
 }

-INSTANTIATE_TEST_CASE_P(PrecludeWithCompactStyleTest,
-                        PrecludeWithCompactStyleTest, ::testing::Bool());
+INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTest, PrecludeLastLevelTest,
+                        ::testing::Bool());

 class TimedPutPrecludeLastLevelTest
    : public PrecludeLastLevelTestBase,
@@ -4,6 +4,7 @@
 //  (found in the LICENSE.Apache file in the root directory).
 //

+
 #include "rocksdb/convenience.h"

 #include "db/convenience_impl.h"
@@ -7,6 +7,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

+#include "rocksdb/options.h"
+
 #include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -23,7 +25,6 @@
 #include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
-#include "rocksdb/options.h"
 #include "rocksdb/table.h"
 #include "rocksdb/utilities/transaction_db.h"
 #include "rocksdb/write_batch.h"
@@ -3,6 +3,7 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

+
 #include "db/db_impl/db_impl.h"
 #include "db/db_test_util.h"
 #include "rocksdb/db.h"
@@ -348,3 +349,4 @@ int main(int argc, char** argv) {
    return 0;
  }
 }
+
@@ -878,6 +878,7 @@ TEST_F(DBBasicTest, Snapshot) {
  } while (ChangeOptions());
 }

+
 class DBBasicMultiConfigs : public DBBasicTest,
                            public ::testing::WithParamInterface<int> {
 public:
@@ -2603,7 +2604,8 @@ class DBMultiGetAsyncIOTest : public DBBasicTest,
    // Warm up the block cache so we don't need to use the IO uring
    Iterator* iter = dbfull()->NewIterator(ReadOptions());
    for (iter->SeekToFirst(); iter->Valid() && iter->status().ok();
-         iter->Next());
+         iter->Next())
+      ;
    EXPECT_OK(iter->status());
    delete iter;
 #endif  // ROCKSDB_IOURING_PRESENT
@@ -305,6 +305,7 @@ class ReadOnlyCacheWrapper : public CacheWrapper {
 }  // anonymous namespace
 #endif  // SNAPPY

+
 // Make sure that when options.block_cache is set, after a new table is
 // created its index/filter blocks are added to block cache.
 TEST_F(DBBlockCacheTest, IndexAndFilterBlocksOfNewTableAddedToCache) {
@@ -1757,8 +1758,8 @@ class CacheKeyTest : public testing::Test {
    tp_.db_id = std::to_string(db_id_);
    tp_.orig_file_number = file_number;
    bool is_stable;
-    std::string cur_session_id;     // ignored
-    uint64_t cur_file_number = 42;  // ignored
+    std::string cur_session_id;       // ignored
+    uint64_t cur_file_number = 42;    // ignored
    OffsetableCacheKey rv;
    BlockBasedTable::SetupBaseCacheKey(&tp_, cur_session_id, cur_file_number,
                                       &rv, &is_stable);
@@ -6417,13 +6417,10 @@ TEST_P(RoundRobinSubcompactionsAgainstPressureToken, PressureTokenTest) {
  options.level0_file_num_compaction_trigger = 4;
  options.target_file_size_base = kKeysPerBuffer * 1024;
  options.compaction_pri = CompactionPri::kRoundRobin;
-  // Pick a small target level size so that round-robin compaction will pick
-  // more files to compact and trigger subcompactions. Actual number of
-  // subcompactions will be limited by number of bg threads available.
-  // Cannot be too small to cause compaction pressure. See
-  // GetPendingCompactionBytesForCompactionSpeedup().
+  // Target size is chosen so that filling the level with `kFilesPerLevel` files
+  // will make it oversized by `kNumSubcompactions` files.
  options.max_bytes_for_level_base =
-      (kFilesPerLevel - 10) * kKeysPerBuffer * 1024;
+      (kFilesPerLevel - kNumSubcompactions) * kKeysPerBuffer * 1024;
  options.disable_auto_compactions = true;
  // Setup `kNumSubcompactions` threads but limited subcompactions so
  // that RoundRobin requires extra compactions from reserved threads
@@ -6449,34 +6446,21 @@ TEST_P(RoundRobinSubcompactionsAgainstPressureToken, PressureTokenTest) {
    ASSERT_EQ(kFilesPerLevel, NumTableFilesAtLevel(lvl, 0));
  }

-  bool compaction_num_input_file_verified = false;
-  SyncPoint::GetInstance()->SetCallBack(
-      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        if (!compaction_num_input_file_verified) {
-          Compaction* compaction = static_cast<Compaction*>(arg);
-          // In Round-Robin, # of subcompactions needed is the # of input files
-          ASSERT_GT(compaction->num_input_files(0), 1);
-          compaction_num_input_file_verified = true;
-        }
-      });
-
  // This is a variable for making sure the following callback is called
  // and the assertions in it are indeed excuted.
  bool num_planned_subcompactions_verified = false;
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "CompactionJob::GenSubcompactionBoundaries:0", [&](void* arg) {
-        if (!num_planned_subcompactions_verified) {
-          uint64_t num_planned_subcompactions = *(static_cast<uint64_t*>(arg));
-          if (grab_pressure_token_) {
-            // `kNumSubcompactions` files are selected for round-robin under
-            // auto compaction. The number of planned subcompaction is
-            // restricted by the limited number of max_background_compactions
-            ASSERT_EQ(num_planned_subcompactions, kNumSubcompactions);
-          } else {
-            ASSERT_EQ(num_planned_subcompactions, 1);
-          }
-          num_planned_subcompactions_verified = true;
+        uint64_t num_planned_subcompactions = *(static_cast<uint64_t*>(arg));
+        if (grab_pressure_token_) {
+          // `kNumSubcompactions` files are selected for round-robin under auto
+          // compaction. The number of planned subcompaction is restricted by
+          // the limited number of max_background_compactions
+          ASSERT_EQ(num_planned_subcompactions, kNumSubcompactions);
+        } else {
+          ASSERT_EQ(num_planned_subcompactions, 1);
        }
+        num_planned_subcompactions_verified = true;
      });

  // The following 3 dependencies have to be added to ensure the auto
@@ -6927,8 +6911,7 @@ TEST_F(DBCompactionTest, ManualCompactionMax) {
  DestroyAndReopen(opts);
  generate_sst_func();
  uint64_t total_size = (l1_avg_size * 10) + (l2_avg_size * 100);
-  // Slightly inflate max_compaction_bytes since it's usually a strict bound.
-  opts.max_compaction_bytes = total_size / num_split + l2_avg_size * 10;
+  opts.max_compaction_bytes = total_size / num_split;
  opts.target_file_size_base = total_size / num_split;
  Reopen(opts);
  num_compactions.store(0);
@@ -6950,10 +6933,8 @@ TEST_F(DBCompactionTest, ManualCompactionMax) {
  DestroyAndReopen(opts);
  generate_sst_func();
  total_size = (l1_avg_size * 10) + (l2_avg_size * 100);
-  // Slightly inflate max_compaction_bytes since it's usually a strict bound.
  Status s = db_->SetOptions(
-      {{"max_compaction_bytes",
-        std::to_string(total_size / num_split + 10 * l2_avg_size)},
+      {{"max_compaction_bytes", std::to_string(total_size / num_split)},
       {"target_file_size_base", std::to_string(total_size / num_split)}});
  ASSERT_OK(s);

@@ -491,6 +491,7 @@ TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
 }
 }  // namespace ROCKSDB_NAMESPACE

+
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -3,13 +3,12 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 //
-#include <iostream>
-#include <string>
-
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
 #include "rocksdb/perf_context.h"
 #include "test_util/sync_point.h"
+#include <iostream>
+#include <string>

 namespace ROCKSDB_NAMESPACE {

@@ -26,6 +25,7 @@ class DBEncryptionTest : public DBTestBase {
  }
 };

+
 TEST_F(DBEncryptionTest, CheckEncrypted) {
  ASSERT_OK(Put("foo567", "v1.fetdq"));
  ASSERT_OK(Put("bar123", "v2.dfgkjdfghsd"));
@@ -116,6 +116,7 @@ TEST_F(DBEncryptionTest, ReadEmptyFile) {
  ASSERT_TRUE(data.empty());
 }

+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -4,6 +4,7 @@
 //  (found in the LICENSE.Apache file in the root directory).
 //

+
 #include <algorithm>
 #include <cstdint>
 #include <memory>
@@ -265,7 +266,7 @@ Status DBImpl::GetLiveFilesStorageInfo(
      continue;
    }
    VersionStorageInfo& vsi = *cfd->current()->storage_info();
-    auto& cf_paths = cfd->ioptions().cf_paths;
+    auto& cf_paths = cfd->ioptions()->cf_paths;

    auto GetDir = [&](size_t path_id) {
      // Matching TableFileName() behavior
@@ -248,7 +248,7 @@ Status CompactedDBImpl::Init(const Options& options) {
 }

 Status CompactedDBImpl::Open(const Options& options, const std::string& dbname,
-                             std::unique_ptr<DB>* dbptr) {
+                             DB** dbptr) {
  *dbptr = nullptr;

  if (options.max_open_files != -1) {
@@ -267,7 +267,7 @@ Status CompactedDBImpl::Open(const Options& options, const std::string& dbname,
    ROCKS_LOG_INFO(db->immutable_db_options_.info_log,
                   "Opened the db as fully compacted mode");
    LogFlush(db->immutable_db_options_.info_log);
-    *dbptr = std::move(db);
+    *dbptr = db.release();
  }
  return s;
 }
@@ -22,7 +22,7 @@ class CompactedDBImpl : public DBImpl {
  ~CompactedDBImpl() override;

  static Status Open(const Options& options, const std::string& dbname,
-                     std::unique_ptr<DB>* dbptr);
+                     DB** dbptr);

  // Implementations of the DB interface
  using DB::Get;
@@ -386,8 +386,9 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) {
          static_cast_with_check<ColumnFamilyHandleImpl>(default_cf_handle_);
      assert(cfh);
      ColumnFamilyData* cfd = cfh->cfd();
-      s = versions_->LogAndApply(cfd, read_options, write_options, &edit,
-                                 &mutex_, directories_.GetDbDir());
+      const MutableCFOptions& cf_opts = *cfd->GetLatestMutableCFOptions();
+      s = versions_->LogAndApply(cfd, cf_opts, read_options, write_options,
+                                 &edit, &mutex_, directories_.GetDbDir());
      if (!s.ok()) {
        io_s = versions_->io_status();
        if (!io_s.ok()) {
@@ -417,19 +418,10 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) {
    }
  }

-  JobContext job_context(0);
-  FindObsoleteFiles(&job_context, true);
-  mutex_.Unlock();
-  // If DB shutdown initiated here, it will wait for this ongoing recovery.
-  job_context.manifest_file_number = 1;
-  if (job_context.HaveSomethingToDelete()) {
-    PurgeObsoleteFiles(job_context);
-  }
-  job_context.Clean();
-  mutex_.Lock();
-
  if (s.ok()) {
-    // Will notify and unblock threads waiting for error recovery to finish.
+    // This will notify and unblock threads waiting for error recovery to
+    // finish. Those previouly waiting threads can now proceed, which may
+    // include closing the db.
    s = error_handler_.ClearBGError();
  } else {
    // NOTE: this is needed to pass ASSERT_STATUS_CHECKED
@@ -438,6 +430,15 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) {
    error_handler_.GetRecoveryError().PermitUncheckedError();
  }

+  JobContext job_context(0);
+  FindObsoleteFiles(&job_context, true);
+  mutex_.Unlock();
+  job_context.manifest_file_number = 1;
+  if (job_context.HaveSomethingToDelete()) {
+    PurgeObsoleteFiles(job_context);
+  }
+  job_context.Clean();
+
  if (s.ok()) {
    ROCKS_LOG_INFO(immutable_db_options_.info_log, "Successfully resumed DB");
  } else {
@@ -445,6 +446,7 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) {
                   s.ToString().c_str());
  }

+  mutex_.Lock();
  // Check for shutdown again before scheduling further compactions,
  // since we released and re-acquired the lock above
  if (shutdown_initiated_) {
@@ -507,11 +509,6 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
    s.PermitUncheckedError();  //**TODO: What to do on error?
  }

-  // Cancel awaiting remote compactions
-  if (immutable_db_options_.compaction_service) {
-    immutable_db_options_.compaction_service->CancelAwaitingJobs();
-  }
-
  shutting_down_.store(true, std::memory_order_release);
  bg_cv_.SignalAll();
  if (!wait) {
@@ -543,8 +540,8 @@ Status DBImpl::CloseHelper() {
  // continuing with the shutdown
  mutex_.Lock();
  shutdown_initiated_ = true;
-  error_handler_.CancelErrorRecoveryForShutDown();
-  while (!error_handler_.ReadyForShutdown()) {
+  error_handler_.CancelErrorRecovery();
+  while (error_handler_.IsRecoveryInProgress()) {
    bg_cv_.Wait();
  }
  mutex_.Unlock();
@@ -855,7 +852,7 @@ Status DBImpl::RegisterRecordSeqnoTimeWorker(const ReadOptions& read_options,
    InstrumentedMutexLock l(&mutex_);

    for (auto cfd : *versions_->GetColumnFamilySet()) {
-      auto& mopts = cfd->GetLatestMutableCFOptions();
+      auto& mopts = *cfd->GetLatestMutableCFOptions();
      // preserve time is the max of 2 options.
      uint64_t preserve_seconds =
          std::max(mopts.preserve_internal_time_seconds,
@@ -1170,7 +1167,7 @@ void DBImpl::DumpStats() {
      }

      auto* table_factory =
-          cfd->GetCurrentMutableCFOptions().table_factory.get();
+          cfd->GetCurrentMutableCFOptions()->table_factory.get();
      assert(table_factory != nullptr);
      // FIXME: need to a shared_ptr if/when block_cache is going to be mutable
      Cache* cache =
@@ -1289,7 +1286,7 @@ Status DBImpl::SetOptions(
  }

  InstrumentedMutexLock ol(&options_mutex_);
-  MutableCFOptions new_options_copy;  // For logging outside of DB mutex
+  MutableCFOptions new_options;
  Status s;
  Status persist_options_status;
  SuperVersionContext sv_context(/* create_superversion */ true);
@@ -1298,20 +1295,16 @@ Status DBImpl::SetOptions(
    InstrumentedMutexLock l(&mutex_);
    s = cfd->SetOptions(db_options, options_map);
    if (s.ok()) {
-      new_options_copy = cfd->GetLatestMutableCFOptions();
+      new_options = *cfd->GetLatestMutableCFOptions();
      // Append new version to recompute compaction score.
      VersionEdit dummy_edit;
-      s = versions_->LogAndApply(cfd, read_options, write_options, &dummy_edit,
-                                 &mutex_, directories_.GetDbDir());
-      if (!versions_->io_status().ok()) {
-        assert(!s.ok());
-        error_handler_.SetBGError(versions_->io_status(),
-                                  BackgroundErrorReason::kManifestWrite);
-      }
+      s = versions_->LogAndApply(cfd, new_options, read_options, write_options,
+                                 &dummy_edit, &mutex_, directories_.GetDbDir());
      // Trigger possible flush/compactions. This has to be before we persist
      // options to file, otherwise there will be a deadlock with writer
      // thread.
-      InstallSuperVersionAndScheduleWork(cfd, &sv_context);
+      InstallSuperVersionAndScheduleWork(cfd, &sv_context, new_options);
+
      persist_options_status =
          WriteOptionsFile(write_options, true /*db_mutex_already_held*/);
      bg_cv_.SignalAll();
@@ -1335,7 +1328,7 @@ Status DBImpl::SetOptions(
  if (s.ok()) {
    ROCKS_LOG_INFO(immutable_db_options_.info_log,
                   "[%s] SetOptions() succeeded", cfd->GetName().c_str());
-    new_options_copy.Dump(immutable_db_options_.info_log.get());
+    new_options.Dump(immutable_db_options_.info_log.get());
    if (!persist_options_status.ok()) {
      // NOTE: WriteOptionsFile already logs on failure
      s = persist_options_status;
@@ -2117,7 +2110,7 @@ InternalIterator* DBImpl::NewInternalIterator(
    } else {
      mem_tombstone_iter = std::make_unique<TruncatedRangeDelIterator>(
          std::unique_ptr<FragmentedRangeTombstoneIterator>(range_del_iter),
-          &cfd->ioptions().internal_comparator, nullptr /* smallest */,
+          &cfd->ioptions()->internal_comparator, nullptr /* smallest */,
          nullptr /* largest */);
    }
    merge_iter_builder.AddPointAndTombstoneIterator(
@@ -3650,9 +3643,9 @@ Status DBImpl::CreateColumnFamilyImpl(const ReadOptions& read_options,
      write_thread_.EnterUnbatched(&w, &mutex_);
      // LogAndApply will both write the creation in MANIFEST and create
      // ColumnFamilyData object
-      s = versions_->LogAndApply(nullptr, read_options, write_options, &edit,
-                                 &mutex_, directories_.GetDbDir(), false,
-                                 &cf_options);
+      s = versions_->LogAndApply(nullptr, MutableCFOptions(cf_options),
+                                 read_options, write_options, &edit, &mutex_,
+                                 directories_.GetDbDir(), false, &cf_options);
      write_thread_.ExitUnbatched(&w);
    }
    if (s.ok()) {
@@ -3666,7 +3659,8 @@ Status DBImpl::CreateColumnFamilyImpl(const ReadOptions& read_options,
      auto* cfd =
          versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name);
      assert(cfd != nullptr);
-      InstallSuperVersionAndScheduleWork(cfd, &sv_context);
+      InstallSuperVersionAndScheduleWork(cfd, &sv_context,
+                                         *cfd->GetLatestMutableCFOptions());

      if (!cfd->mem()->IsSnapshotSupported()) {
        is_snapshot_supported_ = false;
@@ -3760,12 +3754,13 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
      // we drop column family from a single write thread
      WriteThread::Writer w;
      write_thread_.EnterUnbatched(&w, &mutex_);
-      s = versions_->LogAndApply(cfd, read_options, write_options, &edit,
-                                 &mutex_, directories_.GetDbDir());
+      s = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
+                                 read_options, write_options, &edit, &mutex_,
+                                 directories_.GetDbDir());
      write_thread_.ExitUnbatched(&w);
    }
    if (s.ok()) {
-      auto& moptions = cfd->GetLatestMutableCFOptions();
+      auto& moptions = *cfd->GetLatestMutableCFOptions();
      max_total_in_memory_state_ -=
          moptions.write_buffer_size * moptions.max_write_buffer_number;
      used_preserve_preclude = moptions.preserve_internal_time_seconds > 0 ||
@@ -3896,7 +3891,7 @@ Iterator* DBImpl::NewIterator(const ReadOptions& _read_options,
    auto iter = new ForwardIterator(this, read_options, cfd, sv,
                                    /* allow_unprepared_value */ true);
    result = NewDBIterator(
-        env_, read_options, cfd->ioptions(), sv->mutable_cf_options,
+        env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
        cfd->user_comparator(), iter, sv->current, kMaxSequenceNumber,
        sv->mutable_cf_options.max_sequential_skip_in_iterations,
        nullptr /* read_callback */, cfh);
@@ -3978,7 +3973,7 @@ ArenaWrappedDBIter* DBImpl::NewIteratorImpl(
  // likely that any iterator pointer is close to the iterator it points to so
  // that they are likely to be in the same cache line and/or page.
  ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, cfh->cfd()->ioptions(), sv->mutable_cf_options,
+      env_, read_options, *cfh->cfd()->ioptions(), sv->mutable_cf_options,
      sv->current, snapshot,
      sv->mutable_cf_options.max_sequential_skip_in_iterations,
      sv->version_number, read_callback, cfh, expose_blob_index, allow_refresh);
@@ -4113,7 +4108,7 @@ Status DBImpl::NewIterators(
                                      cf_sv_pair.super_version,
                                      /* allow_unprepared_value */ true);
      iterators->push_back(
-          NewDBIterator(env_, read_options, cf_sv_pair.cfd->ioptions(),
+          NewDBIterator(env_, read_options, *cf_sv_pair.cfd->ioptions(),
                        cf_sv_pair.super_version->mutable_cf_options,
                        cf_sv_pair.cfd->user_comparator(), iter,
                        cf_sv_pair.super_version->current, kMaxSequenceNumber,
@@ -4351,7 +4346,7 @@ void DBImpl::ReleaseSnapshot(const Snapshot* s) {
    CfdList cf_scheduled;
    if (oldest_snapshot > bottommost_files_mark_threshold_) {
      for (auto* cfd : *versions_->GetColumnFamilySet()) {
-        if (!cfd->ioptions().allow_ingest_behind) {
+        if (!cfd->ioptions()->allow_ingest_behind) {
          cfd->current()->storage_info()->UpdateOldestSnapshot(
              oldest_snapshot, /*allow_ingest_behind=*/false);
          if (!cfd->current()
@@ -4372,7 +4367,7 @@ void DBImpl::ReleaseSnapshot(const Snapshot* s) {
      SequenceNumber new_bottommost_files_mark_threshold = kMaxSequenceNumber;
      for (auto* cfd : *versions_->GetColumnFamilySet()) {
        if (CfdListContains(cf_scheduled, cfd) ||
-            cfd->ioptions().allow_ingest_behind) {
+            cfd->ioptions()->allow_ingest_behind) {
          continue;
        }
        new_bottommost_files_mark_threshold = std::min(
@@ -4883,7 +4878,7 @@ Status DBImpl::GetUpdatesSince(
  return wal_manager_.GetUpdatesSince(seq, iter, read_options, versions_.get());
 }

-Status DBImpl::DEPRECATED_DeleteFile(std::string name) {
+Status DBImpl::DeleteFile(std::string name) {
  // TODO: plumb Env::IOActivity, Env::IOPriority
  const ReadOptions read_options;
  const WriteOptions write_options;
@@ -4966,11 +4961,13 @@ Status DBImpl::DEPRECATED_DeleteFile(std::string name) {
    }
    edit.SetColumnFamily(cfd->GetID());
    edit.DeleteFile(level, number);
-    status = versions_->LogAndApply(cfd, read_options, write_options, &edit,
-                                    &mutex_, directories_.GetDbDir());
+    status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
+                                    read_options, write_options, &edit, &mutex_,
+                                    directories_.GetDbDir());
    if (status.ok()) {
      InstallSuperVersionAndScheduleWork(
-          cfd, job_context.superversion_contexts.data());
+          cfd, job_context.superversion_contexts.data(),
+          *cfd->GetLatestMutableCFOptions());
    }
    FindObsoleteFiles(&job_context, false);
  }  // lock released here
@@ -5069,19 +5066,21 @@ Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family,
      }
    }
    if (!deleted_files.empty()) {
-      vstorage->ComputeCompactionScore(cfd->ioptions(),
-                                       cfd->GetLatestMutableCFOptions());
+      vstorage->ComputeCompactionScore(*cfd->ioptions(),
+                                       *cfd->GetLatestMutableCFOptions());
    }
    if (edit.GetDeletedFiles().empty()) {
      job_context.Clean();
      return status;
    }
    input_version->Ref();
-    status = versions_->LogAndApply(cfd, read_options, write_options, &edit,
-                                    &mutex_, directories_.GetDbDir());
+    status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
+                                    read_options, write_options, &edit, &mutex_,
+                                    directories_.GetDbDir());
    if (status.ok()) {
      InstallSuperVersionAndScheduleWork(
-          cfd, job_context.superversion_contexts.data());
+          cfd, job_context.superversion_contexts.data(),
+          *cfd->GetLatestMutableCFOptions());
    }
    for (auto* deleted_file : deleted_files) {
      deleted_file->being_compacted = false;
@@ -5680,7 +5679,7 @@ Status DBImpl::RenameTempFileToOptionsFile(const std::string& file_name,
 void DBImpl::NewThreadStatusCfInfo(ColumnFamilyData* cfd) const {
  if (immutable_db_options_.enable_thread_tracking) {
    ThreadStatusUtil::NewColumnFamilyInfo(this, cfd, cfd->GetName(),
-                                          cfd->ioptions().env);
+                                          cfd->ioptions()->env);
  }
 }

@@ -5898,7 +5897,6 @@ Status DBImpl::IngestExternalFile(

 Status DBImpl::IngestExternalFiles(
    const std::vector<IngestExternalFileArg>& args) {
-  PERF_TIMER_GUARD(file_ingestion_nanos);
  // TODO: plumb Env::IOActivity, Env::IOPriority
  const WriteOptions write_options;

@@ -6043,7 +6041,6 @@ Status DBImpl::IngestExternalFiles(
    if (two_write_queues_) {
      nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
    }
-    PERF_TIMER_GUARD(file_ingestion_blocking_live_writes_nanos);

    // When unordered_write is enabled, the keys are writing to memtable in an
    // unordered way. If the ingestion job checks memtable key range before the
@@ -6124,12 +6121,14 @@ Status DBImpl::IngestExternalFiles(
      ReadOptions read_options;
      read_options.fill_cache = args[0].options.fill_cache;
      autovector<ColumnFamilyData*> cfds_to_commit;
+      autovector<const MutableCFOptions*> mutable_cf_options_list;
      autovector<autovector<VersionEdit*>> edit_lists;
      uint32_t num_entries = 0;
      for (size_t i = 0; i != num_cfs; ++i) {
        auto* cfd = ingestion_jobs[i].GetColumnFamilyData();
        assert(!cfd->IsDropped());
        cfds_to_commit.push_back(cfd);
+        mutable_cf_options_list.push_back(cfd->GetLatestMutableCFOptions());
        autovector<VersionEdit*> edit_list;
        edit_list.push_back(ingestion_jobs[i].edit());
        edit_lists.push_back(edit_list);
@@ -6144,10 +6143,10 @@ Status DBImpl::IngestExternalFiles(
        }
        assert(0 == num_entries);
      }
-      status =
-          versions_->LogAndApply(cfds_to_commit, read_options, write_options,
+      status = versions_->LogAndApply(
+          cfds_to_commit, mutable_cf_options_list, read_options, write_options,

-                                 edit_lists, &mutex_, directories_.GetDbDir());
+          edit_lists, &mutex_, directories_.GetDbDir());
      // It is safe to update VersionSet last seqno here after LogAndApply since
      // LogAndApply persists last sequence number from VersionEdits,
      // which are from file's largest seqno and not from VersionSet.
@@ -6179,7 +6178,8 @@ Status DBImpl::IngestExternalFiles(
      for (size_t i = 0; i != num_cfs; ++i) {
        auto* cfd = ingestion_jobs[i].GetColumnFamilyData();
        assert(!cfd->IsDropped());
-        InstallSuperVersionAndScheduleWork(cfd, &sv_ctxs[i]);
+        InstallSuperVersionAndScheduleWork(cfd, &sv_ctxs[i],
+                                           *cfd->GetLatestMutableCFOptions());
 #ifndef NDEBUG
        if (0 == i && num_cfs > 1) {
          TEST_SYNC_POINT("DBImpl::IngestExternalFiles:InstallSVForFirstCF:0");
@@ -6203,7 +6203,6 @@ Status DBImpl::IngestExternalFiles(
      nonmem_write_thread_.ExitUnbatched(&nonmem_w);
    }
    write_thread_.ExitUnbatched(&w);
-    PERF_TIMER_STOP(file_ingestion_blocking_live_writes_nanos);

    if (status.ok()) {
      for (auto& job : ingestion_jobs) {
@@ -6298,11 +6297,12 @@ Status DBImpl::CreateColumnFamilyWithImport(
      // and this will overwrite the external file. To protect the external
      // file, we have to make sure the file number will never being reused.
      next_file_number = versions_->FetchAddFileNumber(total_file_num);
+      auto cf_options = cfd->GetLatestMutableCFOptions();
      status =
-          versions_->LogAndApply(cfd, read_options, write_options, &dummy_edit,
-                                 &mutex_, directories_.GetDbDir());
+          versions_->LogAndApply(cfd, *cf_options, read_options, write_options,
+                                 &dummy_edit, &mutex_, directories_.GetDbDir());
      if (status.ok()) {
-        InstallSuperVersionAndScheduleWork(cfd, &dummy_sv_ctx);
+        InstallSuperVersionAndScheduleWork(cfd, &dummy_sv_ctx, *cf_options);
      }
    }
  }
@@ -6335,11 +6335,12 @@ Status DBImpl::CreateColumnFamilyWithImport(

      // Install job edit [Mutex will be unlocked here]
      if (status.ok()) {
-        status = versions_->LogAndApply(cfd, read_options, write_options,
-                                        import_job.edit(), &mutex_,
-                                        directories_.GetDbDir());
+        auto cf_options = cfd->GetLatestMutableCFOptions();
+        status = versions_->LogAndApply(cfd, *cf_options, read_options,
+                                        write_options, import_job.edit(),
+                                        &mutex_, directories_.GetDbDir());
        if (status.ok()) {
-          InstallSuperVersionAndScheduleWork(cfd, &sv_context);
+          InstallSuperVersionAndScheduleWork(cfd, &sv_context, *cf_options);
        }
      }

@@ -6540,7 +6541,7 @@ Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options,
        const auto& fd = fd_with_krange.fd;
        const FileMetaData* fmeta = fd_with_krange.file_metadata;
        assert(fmeta);
-        std::string fname = TableFileName(cfd->ioptions().cf_paths,
+        std::string fname = TableFileName(cfd->ioptions()->cf_paths,
                                          fd.GetNumber(), fd.GetPathId());
        if (use_file_checksum) {
          s = VerifyFullFileChecksum(fmeta->file_checksum,
@@ -6564,7 +6565,7 @@ Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options,
        const uint64_t blob_file_number = meta->GetBlobFileNumber();

        const std::string blob_file_name = BlobFileName(
-            cfd->ioptions().cf_paths.front().path, blob_file_number);
+            cfd->ioptions()->cf_paths.front().path, blob_file_number);
        s = VerifyFullFileChecksum(meta->GetChecksumValue(),
                                   meta->GetChecksumMethod(), blob_file_name,
                                   read_options);
@@ -6757,15 +6758,16 @@ Status DBImpl::ReserveFileNumbersBeforeIngestion(
  pending_output_elem.reset(new std::list<uint64_t>::iterator(
      CaptureCurrentFileNumberInPendingOutputs()));
  *next_file_number = versions_->FetchAddFileNumber(static_cast<uint64_t>(num));
+  auto cf_options = cfd->GetLatestMutableCFOptions();
  VersionEdit dummy_edit;
  // If crash happen after a hard link established, Recover function may
  // reuse the file number that has already assigned to the internal file,
  // and this will overwrite the external file. To protect the external
  // file, we have to make sure the file number will never being reused.
-  s = versions_->LogAndApply(cfd, read_options, write_options, &dummy_edit,
-                             &mutex_, directories_.GetDbDir());
+  s = versions_->LogAndApply(cfd, *cf_options, read_options, write_options,
+                             &dummy_edit, &mutex_, directories_.GetDbDir());
  if (s.ok()) {
-    InstallSuperVersionAndScheduleWork(cfd, &dummy_sv_ctx);
+    InstallSuperVersionAndScheduleWork(cfd, &dummy_sv_ctx, *cf_options);
  }
  dummy_sv_ctx.Clean();
  return s;
@@ -6804,24 +6806,17 @@ void DBImpl::RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds) {
  // for SeqnoToTimeMapping. We don't know how long it has been since the last
  // sequence number was written, so we at least have a one-sided bound by
  // sampling in this order.
-  // ALSO, to avoid out-of-order mappings, we need to get the seqno and times
-  // while holding the DB mutex. (This is really to make testing happy because
-  // it's fine to throw out extra close-but-not-quite-consistent mappings in
-  // production.)
+  SequenceNumber seqno = GetLatestSequenceNumber();
+  int64_t unix_time_signed = 0;
+  immutable_db_options_.clock->GetCurrentTime(&unix_time_signed)
+      .PermitUncheckedError();  // Ignore error
+  uint64_t unix_time = static_cast<uint64_t>(unix_time_signed);
+
  std::vector<SuperVersionContext> sv_contexts;
-  bool success = true;
-  SequenceNumber seqno;
-  uint64_t unix_time;
-  {
-    InstrumentedMutexLock l(&mutex_);
-
-    seqno = GetLatestSequenceNumber();
-    int64_t unix_time_signed = 0;
-    immutable_db_options_.clock->GetCurrentTime(&unix_time_signed)
-        .PermitUncheckedError();  // Ignore error
-    unix_time = static_cast<uint64_t>(unix_time_signed);
-
-    if (populate_historical_seconds > 0) {
+  if (populate_historical_seconds > 0) {
+    bool success = true;
+    {
+      InstrumentedMutexLock l(&mutex_);
      if (seqno > 1 && unix_time > populate_historical_seconds) {
        // seqno=0 is reserved
        SequenceNumber from_seqno = 1;
@@ -6835,20 +6830,7 @@ void DBImpl::RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds) {
        assert(unix_time > populate_historical_seconds);
        success = false;
      }
-    } else {
-      // FIXME: assert(seqno > 0);
-      // Always successful assuming seqno never go backwards
-      seqno_to_time_mapping_.Append(seqno, unix_time);
-      InstallSeqnoToTimeMappingInSV(&sv_contexts);
    }
-  }
-
-  // clean up & report outside db mutex
-  for (SuperVersionContext& sv_context : sv_contexts) {
-    sv_context.Clean();
-  }
-
-  if (populate_historical_seconds > 0) {
    if (success) {
      ROCKS_LOG_INFO(
          immutable_db_options_.info_log,
@@ -6863,7 +6845,16 @@ void DBImpl::RecordSeqnoToTimeMapping(uint64_t populate_historical_seconds) {
          seqno, unix_time - populate_historical_seconds, unix_time);
    }
  } else {
-    assert(success);
+    InstrumentedMutexLock l(&mutex_);
+    // FIXME: assert(seqno > 0);
+    // Always successful assuming seqno never go backwards
+    seqno_to_time_mapping_.Append(seqno, unix_time);
+    InstallSeqnoToTimeMappingInSV(&sv_contexts);
+  }
+
+  // clean up outside db mutex
+  for (SuperVersionContext& sv_context : sv_contexts) {
+    sv_context.Clean();
  }
 }

@@ -6936,7 +6927,7 @@ void DBImpl::InstallSeqnoToTimeMappingInSV(
    sv_contexts->emplace_back(/*create_superversion=*/true);
    sv_contexts->back().new_seqno_to_time_mapping = new_seqno_to_time_mapping;
    cfd->InstallSuperVersion(&sv_contexts->back(),
-                             cfd->GetLatestMutableCFOptions());
+                             *(cfd->GetLatestMutableCFOptions()));
  }
  bg_cv_.SignalAll();
 }
@@ -162,17 +162,6 @@ class Directories {
  std::unique_ptr<FSDirectory> wal_dir_;
 };

-struct DBOpenLogReporter : public log::Reader::Reporter {
-  Env* env;
-  Logger* info_log;
-  const char* fname;
-  Status* status;  // nullptr if immutable_db_options_.paranoid_checks==false
-  bool* old_log_record;
-  void Corruption(size_t bytes, const Status& s) override;
-
-  void OldLogRecord(size_t bytes) override;
-};
-
 // While DB is the public interface of RocksDB, and DBImpl is the actual
 // class implementing it. It's the entrance of the core RocksdB engine.
 // All other DB implementations, e.g. TransactionDB, BlobDB, etc, wrap a
@@ -535,7 +524,7 @@ class DBImpl : public DB {
      SequenceNumber seq_number, std::unique_ptr<TransactionLogIterator>* iter,
      const TransactionLogIterator::ReadOptions& read_options =
          TransactionLogIterator::ReadOptions()) override;
-  Status DEPRECATED_DeleteFile(std::string name) override;
+  Status DeleteFile(std::string name) override;
  Status DeleteFilesInRanges(ColumnFamilyHandle* column_family,
                             const RangePtr* ranges, size_t n,
                             bool include_end = true);
@@ -1092,10 +1081,9 @@ class DBImpl : public DB {
  // This is to be used only by internal rocksdb classes.
  static Status Open(const DBOptions& db_options, const std::string& name,
                     const std::vector<ColumnFamilyDescriptor>& column_families,
-                     std::vector<ColumnFamilyHandle*>* handles,
-                     std::unique_ptr<DB>* dbptr, const bool seq_per_batch,
-                     const bool batch_per_txn, const bool is_retry,
-                     bool* can_retry);
+                     std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
+                     const bool seq_per_batch, const bool batch_per_txn,
+                     const bool is_retry, bool* can_retry);

  static IOStatus CreateAndNewDirectory(
      FileSystem* fs, const std::string& dirname,
@@ -1452,6 +1440,7 @@ class DBImpl : public DB {
        uint32_t size = static_cast<uint32_t>(map_.size());
        map_.emplace(cfd->GetID(), size);
        cfds_.emplace_back(cfd);
+        mutable_cf_opts_.emplace_back(cfd->GetLatestMutableCFOptions());
        edit_lists_.emplace_back(autovector<VersionEdit*>());
      }
      uint32_t i = map_[cfd->GetID()];
@@ -1460,6 +1449,7 @@ class DBImpl : public DB {

    std::unordered_map<uint32_t, uint32_t> map_;  // cf_id to index;
    autovector<ColumnFamilyData*> cfds_;
+    autovector<const MutableCFOptions*> mutable_cf_opts_;
    autovector<autovector<VersionEdit*>> edit_lists_;
    // All existing data files (SST files and Blob files) found during DB::Open.
    std::vector<std::string> existing_data_files_;
@@ -1920,8 +1910,8 @@ class DBImpl : public DB {
    const InternalKey* begin = nullptr;  // nullptr means beginning of key range
    const InternalKey* end = nullptr;    // nullptr means end of key range
    InternalKey* manual_end = nullptr;   // how far we are compacting
-    InternalKey tmp_storage;   // Used to keep track of compaction progress
-    InternalKey tmp_storage1;  // Used to keep track of compaction progress
+    InternalKey tmp_storage;      // Used to keep track of compaction progress
+    InternalKey tmp_storage1;     // Used to keep track of compaction progress

    // When the user provides a canceled pointer in CompactRangeOptions, the
    // above varaibe is the reference of the user-provided
@@ -2053,90 +2043,6 @@ class DBImpl : public DB {
                         bool is_retry, bool* corrupted_log_found,
                         RecoveryContext* recovery_ctx);

-  void SetupLogFilesRecovery(
-      const std::vector<uint64_t>& wal_numbers,
-      std::unordered_map<int, VersionEdit>* version_edits, int* job_id,
-      uint64_t* min_wal_number);
-
-  Status ProcessLogFiles(const std::vector<uint64_t>& wal_numbers,
-                         bool read_only, bool is_retry, uint64_t min_wal_number,
-                         int job_id, SequenceNumber* next_sequence,
-                         std::unordered_map<int, VersionEdit>* version_edits,
-                         bool* corrupted_wal_found,
-                         RecoveryContext* recovery_ctx);
-
-  Status ProcessLogFile(
-      uint64_t wal_number, uint64_t min_wal_number, bool is_retry,
-      bool read_only, int job_id, SequenceNumber* next_sequence,
-      bool* stop_replay_for_corruption, bool* stop_replay_by_wal_filter,
-      uint64_t* corrupted_wal_number, bool* corrupted_wal_found,
-      std::unordered_map<int, VersionEdit>* version_edits, bool* flushed,
-      PredecessorWALInfo& predecessor_wal_info);
-
-  void SetupLogFileProcessing(uint64_t wal_number);
-
-  Status InitializeLogReader(
-      uint64_t wal_number, bool is_retry, std::string& fname,
-
-      bool stop_replay_for_corruption, uint64_t min_wal_number,
-      const PredecessorWALInfo& predecessor_wal_info,
-      bool* const old_log_record, Status* const reporter_status,
-      DBOpenLogReporter* reporter, std::unique_ptr<log::Reader>& reader);
-  Status ProcessLogRecord(
-      Slice record, const std::unique_ptr<log::Reader>& reader,
-      const UnorderedMap<uint32_t, size_t>& running_ts_sz, uint64_t wal_number,
-      const std::string& fname, bool read_only, int job_id,
-      std::function<void()> logFileDropped, DBOpenLogReporter* reporter,
-      uint64_t* record_checksum, SequenceNumber* last_seqno_observed,
-      SequenceNumber* next_sequence, bool* stop_replay_for_corruption,
-      Status* status, bool* stop_replay_by_wal_filter,
-      std::unordered_map<int, VersionEdit>* version_edits, bool* flushed);
-
-  Status InitializeWriteBatchForLogRecord(
-      Slice record, const std::unique_ptr<log::Reader>& reader,
-      const UnorderedMap<uint32_t, size_t>& running_ts_sz, WriteBatch* batch,
-      std::unique_ptr<WriteBatch>& new_batch, WriteBatch*& batch_to_use,
-      uint64_t* record_checksum);
-
-  void MaybeReviseStopReplayForCorruption(
-      SequenceNumber sequence, SequenceNumber const* const next_sequence,
-      bool* stop_replay_for_corruption);
-
-  Status InsertLogRecordToMemtable(WriteBatch* batch_to_use,
-                                   uint64_t wal_number,
-                                   SequenceNumber* next_sequence,
-                                   bool* has_valid_writes);
-
-  Status MaybeWriteLevel0TableForRecovery(
-      bool has_valid_writes, bool read_only, uint64_t wal_number, int job_id,
-      SequenceNumber const* const next_sequence,
-      std::unordered_map<int, VersionEdit>* version_edits, bool* flushed);
-
-  Status HandleNonOkStatusOrOldLogRecord(
-      uint64_t wal_number, SequenceNumber const* const next_sequence,
-      Status log_read_status, bool* old_log_record,
-      bool* stop_replay_for_corruption, uint64_t* corrupted_wal_number,
-      bool* corrupted_wal_found);
-
-  Status UpdatePredecessorWALInfo(uint64_t wal_number,
-                                  const SequenceNumber last_seqno_observed,
-                                  const std::string& fname,
-                                  PredecessorWALInfo& predecessor_wal_info);
-
-  void FinishLogFileProcessing(const Status& status,
-                               const SequenceNumber* next_sequence);
-
-  // Return `Status::Corruption()` when `stop_replay_for_corruption == true` and
-  // exits inconsistency between SST and WAL data
-  Status MaybeHandleStopReplayForCorruptionForInconsistency(
-      bool stop_replay_for_corruption, uint64_t corrupted_wal_number);
-
-  Status MaybeFlushFinalMemtableOrRestoreActiveLogFiles(
-      const std::vector<uint64_t>& wal_numbers, bool read_only, int job_id,
-      bool flushed, std::unordered_map<int, VersionEdit>* version_edits,
-      RecoveryContext* recovery_ctx);
-
-  void FinishLogFilesRecovery(int job_id, const Status& status);
  // The following two methods are used to flush a memtable to
  // storage. The first one is used at database RecoveryTime (when the
  // database is opened) and is heavyweight because it holds the mutex
@@ -2316,8 +2222,7 @@ class DBImpl : public DB {
                      const WriteOptions& write_options,
                      log::Writer* log_writer, uint64_t* log_used,
                      uint64_t* log_size,
-                      LogFileNumberSize& log_file_number_size,
-                      SequenceNumber sequence);
+                      LogFileNumberSize& log_file_number_size);

  IOStatus WriteToWAL(const WriteThread::WriteGroup& write_group,
                      log::Writer* log_writer, uint64_t* log_used,
@@ -2521,8 +2426,9 @@ class DBImpl : public DB {
  // All ColumnFamily state changes go through this function. Here we analyze
  // the new state and we schedule background work if we detect that the new
  // state needs flush or compaction.
-  void InstallSuperVersionAndScheduleWork(ColumnFamilyData* cfd,
-                                          SuperVersionContext* sv_context);
+  void InstallSuperVersionAndScheduleWork(
+      ColumnFamilyData* cfd, SuperVersionContext* sv_context,
+      const MutableCFOptions& mutable_cf_options);

  bool GetIntPropertyInternal(ColumnFamilyData* cfd,
                              const DBPropertyInfo& property_info,
@@ -2561,7 +2467,6 @@ class DBImpl : public DB {

  IOStatus CreateWAL(const WriteOptions& write_options, uint64_t log_file_num,
                     uint64_t recycle_log_number, size_t preallocate_block_size,
-                     const PredecessorWALInfo& predecessor_wal_info,
                     log::Writer** new_log);

  // Validate self-consistency of DB options
@@ -66,7 +66,7 @@ bool DBImpl::RequestCompactionToken(ColumnFamilyData* cfd, bool force,
                                    LogBuffer* log_buffer) {
  assert(*token == nullptr);
  auto limiter = static_cast<ConcurrentTaskLimiterImpl*>(
-      cfd->ioptions().compaction_thread_limiter.get());
+      cfd->ioptions()->compaction_thread_limiter.get());
  if (limiter == nullptr) {
    return true;
  }
@@ -99,7 +99,7 @@ bool DBImpl::ShouldRescheduleFlushRequestToRetainUDT(
  // Check if holding on the flush will cause entering write stall mode.
  // Write stall entered because of the accumulation of write buffers can be
  // alleviated if we continue with the flush instead of postponing it.
-  const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();
+  const auto& mutable_cf_options = *cfd->GetLatestMutableCFOptions();

  // Use the same criteria as WaitUntilFlushWouldNotStallWrites does w.r.t
  // defining what a write stall is about to happen means. If this uses a
@@ -116,7 +116,7 @@ bool DBImpl::ShouldRescheduleFlushRequestToRetainUDT(
          cfd->GetUnflushedMemTableCountForWriteStallCheck(),
          /*num_l0_files=*/0,
          /*num_compaction_needed_bytes=*/0, mutable_cf_options,
-          cfd->ioptions())
+          *cfd->ioptions())
          .first;
  if (write_stall != WriteStallCondition::kNormal) {
    return false;
@@ -213,7 +213,7 @@ Status DBImpl::FlushMemTableToOutputFile(
      snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker,
      job_context, flush_reason, log_buffer, directories_.GetDbDir(),
      GetDataDir(cfd, 0U),
-      GetCompressionFlush(cfd->ioptions(), mutable_cf_options), stats_,
+      GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
      &event_logger_, mutable_cf_options.report_bg_io_stats,
      true /* sync_output_directory */, true /* write_manifest */, thread_pri,
      io_tracer_, cfd->GetSuperVersion()->ShareSeqnoToTimeMapping(), db_id_,
@@ -299,7 +299,8 @@ Status DBImpl::FlushMemTableToOutputFile(
  }

  if (s.ok()) {
-    InstallSuperVersionAndScheduleWork(cfd, superversion_context);
+    InstallSuperVersionAndScheduleWork(cfd, superversion_context,
+                                       mutable_cf_options);
    if (made_progress) {
      *made_progress = true;
    }
@@ -368,7 +369,7 @@ Status DBImpl::FlushMemTableToOutputFile(
    if (sfm) {
      // Notify sst_file_manager that a new file was added
      std::string file_path = MakeTableFileName(
-          cfd->ioptions().cf_paths[0].path, file_meta.fd.GetNumber());
+          cfd->ioptions()->cf_paths[0].path, file_meta.fd.GetNumber());
      // TODO (PR7798).  We should only add the file to the FileManager if it
      // exists. Otherwise, some tests may fail.  Ignore the error in the
      // interim.
@@ -403,8 +404,7 @@ Status DBImpl::FlushMemTablesToOutputFiles(
  const auto& bg_flush_arg = bg_flush_args[0];
  ColumnFamilyData* cfd = bg_flush_arg.cfd_;
  // intentional infrequent copy for each flush
-  const MutableCFOptions mutable_cf_options_copy =
-      cfd->GetLatestMutableCFOptions();
+  MutableCFOptions mutable_cf_options_copy = *cfd->GetLatestMutableCFOptions();
  SuperVersionContext* superversion_context =
      bg_flush_arg.superversion_context_;
  FlushReason flush_reason = bg_flush_arg.flush_reason_;
@@ -461,7 +461,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
  for (int i = 0; i < num_cfs; ++i) {
    auto cfd = cfds[i];
    FSDirectory* data_dir = GetDataDir(cfd, 0U);
-    const std::string& curr_path = cfd->ioptions().cf_paths[0].path;
+    const std::string& curr_path = cfd->ioptions()->cf_paths[0].path;

    // Add to distinct output directories if eligible. Use linear search. Since
    // the number of elements in the vector is not large, performance should be
@@ -478,7 +478,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
      distinct_output_dirs.emplace_back(data_dir);
    }

-    all_mutable_cf_options.emplace_back(cfd->GetLatestMutableCFOptions());
+    all_mutable_cf_options.emplace_back(*cfd->GetLatestMutableCFOptions());
    const MutableCFOptions& mutable_cf_options = all_mutable_cf_options.back();
    uint64_t max_memtable_id = bg_flush_args[i].max_memtable_id_;
    FlushReason flush_reason = bg_flush_args[i].flush_reason_;
@@ -488,7 +488,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
        &shutting_down_, snapshot_seqs, earliest_write_conflict_snapshot,
        snapshot_checker, job_context, flush_reason, log_buffer,
        directories_.GetDbDir(), data_dir,
-        GetCompressionFlush(cfd->ioptions(), mutable_cf_options), stats_,
+        GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
        &event_logger_, mutable_cf_options.report_bg_io_stats,
        false /* sync_output_directory */, false /* write_manifest */,
        thread_pri, io_tracer_,
@@ -754,6 +754,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
  if (s.ok()) {
    autovector<ColumnFamilyData*> tmp_cfds;
    autovector<const autovector<ReadOnlyMemTable*>*> mems_list;
+    autovector<const MutableCFOptions*> mutable_cf_options_list;
    autovector<FileMetaData*> tmp_file_meta;
    autovector<std::list<std::unique_ptr<FlushJobInfo>>*>
        committed_flush_jobs_info;
@@ -762,6 +763,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
      if (!cfds[i]->IsDropped() && !mems.empty()) {
        tmp_cfds.emplace_back(cfds[i]);
        mems_list.emplace_back(&mems);
+        mutable_cf_options_list.emplace_back(&all_mutable_cf_options[i]);
        tmp_file_meta.emplace_back(&file_meta[i]);
        committed_flush_jobs_info.emplace_back(
            jobs[i]->GetCommittedFlushJobsInfo());
@@ -769,8 +771,8 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    }

    s = InstallMemtableAtomicFlushResults(
-        nullptr /* imm_lists */, tmp_cfds, mems_list, versions_.get(),
-        &logs_with_prep_tracker_, &mutex_, tmp_file_meta,
+        nullptr /* imm_lists */, tmp_cfds, mutable_cf_options_list, mems_list,
+        versions_.get(), &logs_with_prep_tracker_, &mutex_, tmp_file_meta,
        committed_flush_jobs_info, &job_context->memtables_to_free,
        directories_.GetDbDir(), log_buffer);
  }
@@ -784,8 +786,9 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
      if (cfds[i]->IsDropped()) {
        continue;
      }
-      InstallSuperVersionAndScheduleWork(
-          cfds[i], &job_context->superversion_contexts[i]);
+      InstallSuperVersionAndScheduleWork(cfds[i],
+                                         &job_context->superversion_contexts[i],
+                                         all_mutable_cf_options[i]);

      const std::string& column_family_name = cfds[i]->GetName();

@@ -831,7 +834,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
                             jobs[i]->GetCommittedFlushJobsInfo());
      if (sfm) {
        std::string file_path = MakeTableFileName(
-            cfds[i]->ioptions().cf_paths[0].path, file_meta[i].fd.GetNumber());
+            cfds[i]->ioptions()->cf_paths[0].path, file_meta[i].fd.GetNumber());
        // TODO (PR7798).  We should only add the file to the FileManager if it
        // exists. Otherwise, some tests may fail.  Ignore the error in the
        // interim.
@@ -905,7 +908,7 @@ void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
    //                 go to L0 in the future.
    const uint64_t file_number = file_meta->fd.GetNumber();
    info.file_path =
-        MakeTableFileName(cfd->ioptions().cf_paths[0].path, file_number);
+        MakeTableFileName(cfd->ioptions()->cf_paths[0].path, file_number);
    info.file_number = file_number;
    info.thread_id = env_->GetThreadID();
    info.job_id = job_id;
@@ -1037,8 +1040,9 @@ Status DBImpl::IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd,
    return Status::InvalidArgument(oss.str());
  }

-  Status s = versions_->LogAndApply(cfd, read_options, write_options, &edit,
-                                    &mutex_, directories_.GetDbDir());
+  Status s = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
+                                    read_options, write_options, &edit, &mutex_,
+                                    directories_.GetDbDir());
  if (!s.ok()) {
    return s;
  }
@@ -1062,11 +1066,11 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
  auto cfd = cfh->cfd();

-  if (options.target_path_id >= cfd->ioptions().cf_paths.size()) {
+  if (options.target_path_id >= cfd->ioptions()->cf_paths.size()) {
    return Status::InvalidArgument("Invalid target path ID");
  }
  if (options.change_level &&
-      cfd->ioptions().compaction_style == kCompactionStyleFIFO) {
+      cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
    return Status::NotSupported(
        "FIFO compaction does not support change_level.");
  }
@@ -1118,7 +1122,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
  constexpr int kInvalidLevel = -1;
  int final_output_level = kInvalidLevel;
  bool exclusive = options.exclusive_manual_compaction;
-  if (cfd->ioptions().compaction_style == kCompactionStyleUniversal &&
+  if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal &&
      cfd->NumberLevels() > 1) {
    // Always compact all files together.
    final_output_level = cfd->NumberLevels() - 1;
@@ -1130,7 +1134,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
                            final_output_level, options, begin, end, exclusive,
                            false /* disable_trivial_move */,
                            std::numeric_limits<uint64_t>::max(), trim_ts);
-  } else if (cfd->ioptions().compaction_style == kCompactionStyleFIFO) {
+  } else if (cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
    // FIFOCompactionPicker::CompactRange() will ignore the input key range
    // [begin, end] and just try to pick compaction based on the configured
    // option `compaction_options_fifo`. So we skip checking if [begin, end]
@@ -1148,7 +1152,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,

      // Might need to query the partitioner
      SstPartitionerFactory* partitioner_factory =
-          current_version->cfd()->ioptions().sst_partitioner_factory.get();
+          current_version->cfd()->ioptions()->sst_partitioner_factory.get();
      std::unique_ptr<SstPartitioner> partitioner;
      if (partitioner_factory && begin != nullptr && end != nullptr) {
        SstPartitioner::Context context;
@@ -1224,7 +1228,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
      CleanupSuperVersion(super_version);
    }
    if (s.ok() && first_overlapped_level != kInvalidLevel) {
-      if (cfd->ioptions().compaction_style == kCompactionStyleUniversal) {
+      if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal) {
        assert(first_overlapped_level == 0);
        s = RunManualCompaction(
            cfd, first_overlapped_level, first_overlapped_level, options, begin,
@@ -1233,7 +1237,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
            trim_ts);
        final_output_level = first_overlapped_level;
      } else {
-        assert(cfd->ioptions().compaction_style == kCompactionStyleLevel);
+        assert(cfd->ioptions()->compaction_style == kCompactionStyleLevel);
        uint64_t next_file_number = versions_->current_next_file_number();
        // Start compaction from `first_overlapped_level`, one level down at a
        // time, until output level >= max_overlapped_level.
@@ -1246,9 +1250,9 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
        for (;;) {
          // Always allow L0 -> L1 compaction
          if (level > 0) {
-            if (cfd->ioptions().level_compaction_dynamic_level_bytes) {
-              assert(final_output_level < cfd->ioptions().num_levels);
-              if (final_output_level + 1 == cfd->ioptions().num_levels) {
+            if (cfd->ioptions()->level_compaction_dynamic_level_bytes) {
+              assert(final_output_level < cfd->ioptions()->num_levels);
+              if (final_output_level + 1 == cfd->ioptions()->num_levels) {
                break;
              }
            } else {
@@ -1267,7 +1271,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
            }
          }
          output_level = level + 1;
-          if (cfd->ioptions().level_compaction_dynamic_level_bytes &&
+          if (cfd->ioptions()->level_compaction_dynamic_level_bytes &&
              level == 0) {
            output_level = ColumnFamilyData::kCompactToBaseLevel;
          }
@@ -1299,8 +1303,8 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
          // bottommost level intra-level compaction
          if ((options.bottommost_level_compaction ==
                   BottommostLevelCompaction::kIfHaveCompactionFilter &&
-               (cfd->ioptions().compaction_filter != nullptr ||
-                cfd->ioptions().compaction_filter_factory != nullptr)) ||
+               (cfd->ioptions()->compaction_filter != nullptr ||
+                cfd->ioptions()->compaction_filter_factory != nullptr)) ||
              options.bottommost_level_compaction ==
                  BottommostLevelCompaction::kForceOptimized ||
              options.bottommost_level_compaction ==
@@ -1454,7 +1458,7 @@ Status DBImpl::CompactFilesImpl(
  }

  if (output_path_id < 0) {
-    if (cfd->ioptions().cf_paths.size() == 1U) {
+    if (cfd->ioptions()->cf_paths.size() == 1U) {
      output_path_id = 0;
    } else {
      return Status::NotSupported(
@@ -1463,12 +1467,12 @@ Status DBImpl::CompactFilesImpl(
    }
  }

-  if (cfd->ioptions().allow_ingest_behind &&
-      output_level >= cfd->ioptions().num_levels - 1) {
+  if (cfd->ioptions()->allow_ingest_behind &&
+      output_level >= cfd->ioptions()->num_levels - 1) {
    return Status::InvalidArgument(
        "Exceed the maximum output level defined by "
        "the current compaction algorithm with ingest_behind --- " +
-        std::to_string(cfd->ioptions().num_levels - 1));
+        std::to_string(cfd->ioptions()->num_levels - 1));
  }

  std::vector<CompactionInputFiles> input_files;
@@ -1505,7 +1509,7 @@ Status DBImpl::CompactFilesImpl(
  assert(cfd->compaction_picker());
  c.reset(cfd->compaction_picker()->CompactFiles(
      compact_options, input_files, output_level, version->storage_info(),
-      cfd->GetLatestMutableCFOptions(), mutable_db_options_, output_path_id));
+      *cfd->GetLatestMutableCFOptions(), mutable_db_options_, output_path_id));
  // we already sanitized the set of input files and checked for conflicts
  // without releasing the lock, so we're guaranteed a compaction can be formed.
  assert(c != nullptr);
@@ -1535,8 +1539,8 @@ Status DBImpl::CompactFilesImpl(
      GetDataDir(c->column_family_data(), 0), stats_, &mutex_, &error_handler_,
      snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker,
      job_context, table_cache_, &event_logger_,
-      c->mutable_cf_options().paranoid_file_checks,
-      c->mutable_cf_options().report_bg_io_stats, dbname_,
+      c->mutable_cf_options()->paranoid_file_checks,
+      c->mutable_cf_options()->report_bg_io_stats, dbname_,
      &compaction_job_stats, Env::Priority::USER, io_tracer_,
      kManualCompactionCanceledFalse_, db_id_, db_session_id_,
      c->column_family_data()->GetFullHistoryTsLow(), c->trim_ts(),
@@ -1547,10 +1551,10 @@ Status DBImpl::CompactFilesImpl(
  // takes running compactions into account (by skipping files that are already
  // being compacted). Since we just changed compaction score, we recalculate it
  // here.
-  version->storage_info()->ComputeCompactionScore(cfd->ioptions(),
-                                                  c->mutable_cf_options());
+  version->storage_info()->ComputeCompactionScore(*cfd->ioptions(),
+                                                  *c->mutable_cf_options());

-  compaction_job.Prepare(std::nullopt /*subcompact to be computed*/);
+  compaction_job.Prepare();

  std::unique_ptr<std::list<uint64_t>::iterator> min_options_file_number_elem;
  if (immutable_db_options().compaction_service != nullptr) {
@@ -1572,14 +1576,16 @@ Status DBImpl::CompactFilesImpl(
  }

  bool compaction_released = false;
-  Status status = compaction_job.Install(&compaction_released);
+  Status status =
+      compaction_job.Install(*c->mutable_cf_options(), &compaction_released);
  if (!compaction_released) {
    c->ReleaseCompactionFiles(s);
  }
  if (status.ok()) {
    assert(compaction_job.io_status().ok());
    InstallSuperVersionAndScheduleWork(
-        c->column_family_data(), job_context->superversion_contexts.data());
+        c->column_family_data(), job_context->superversion_contexts.data(),
+        *c->mutable_cf_options());
  }
  // status above captures any error during compaction_job.Install, so its ok
  // not check compaction_job.io_status() explicitly if we're not calling
@@ -1627,13 +1633,13 @@ Status DBImpl::CompactFilesImpl(
  if (output_file_names != nullptr) {
    for (const auto& newf : c->edit()->GetNewFiles()) {
      output_file_names->push_back(TableFileName(
-          c->immutable_options().cf_paths, newf.second.fd.GetNumber(),
+          c->immutable_options()->cf_paths, newf.second.fd.GetNumber(),
          newf.second.fd.GetPathId()));
    }

    for (const auto& blob_file : c->edit()->GetBlobFileAdditions()) {
      output_file_names->push_back(
-          BlobFileName(c->immutable_options().cf_paths.front().path,
+          BlobFileName(c->immutable_options()->cf_paths.front().path,
                       blob_file.GetBlobFileNumber()));
    }
  }
@@ -1768,7 +1774,7 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
  }
  refitting_level_ = true;

-  const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();
+  const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
  // move to a smaller level
  int to_level = target_level;
  if (target_level < 0) {
@@ -1839,12 +1845,12 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
                    cfd->current()->DebugString().data());

    std::unique_ptr<Compaction> c(new Compaction(
-        vstorage, cfd->ioptions(), mutable_cf_options, mutable_db_options_,
+        vstorage, *cfd->ioptions(), mutable_cf_options, mutable_db_options_,
        {input}, to_level,
        MaxFileSizeForLevel(
            mutable_cf_options, to_level,
            cfd->ioptions()
-                .compaction_style) /* output file size limit, not applicable */
+                ->compaction_style) /* output file size limit, not applicable */
        ,
        LLONG_MAX /* max compaction bytes, not applicable */,
        0 /* output path ID, not applicable */, mutable_cf_options.compression,
@@ -1877,14 +1883,14 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
                    "[%s] Apply version edit:\n%s", cfd->GetName().c_str(),
                    edit.DebugString().data());

-    Status status =
-        versions_->LogAndApply(cfd, read_options, write_options, &edit, &mutex_,
-                               directories_.GetDbDir());
+    Status status = versions_->LogAndApply(cfd, mutable_cf_options,
+                                           read_options, write_options, &edit,
+                                           &mutex_, directories_.GetDbDir());
    c->MarkFilesBeingCompacted(false);
    cfd->compaction_picker()->UnregisterCompaction(c.get());
    c.reset();

-    InstallSuperVersionAndScheduleWork(cfd, &sv_context);
+    InstallSuperVersionAndScheduleWork(cfd, &sv_context, mutable_cf_options);

    ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "[%s] LogAndApply: %s\n",
                    cfd->GetName().c_str(), status.ToString().data());
@@ -2037,16 +2043,16 @@ Status DBImpl::RunManualCompaction(
  // For universal compaction, we enforce every manual compaction to compact
  // all files.
  if (begin == nullptr ||
-      cfd->ioptions().compaction_style == kCompactionStyleUniversal ||
-      cfd->ioptions().compaction_style == kCompactionStyleFIFO) {
+      cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
+      cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
    manual.begin = nullptr;
  } else {
    begin_storage.SetMinPossibleForUserKey(*begin);
    manual.begin = &begin_storage;
  }
  if (end == nullptr ||
-      cfd->ioptions().compaction_style == kCompactionStyleUniversal ||
-      cfd->ioptions().compaction_style == kCompactionStyleFIFO) {
+      cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
+      cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
    manual.end = nullptr;
  } else {
    end_storage.SetMaxPossibleForUserKey(*end);
@@ -2127,7 +2133,7 @@ Status DBImpl::RunManualCompaction(
        scheduled ||
        (((manual.manual_end = &manual.tmp_storage1) != nullptr) &&
         ((compaction = manual.cfd->CompactRange(
-               manual.cfd->GetLatestMutableCFOptions(), mutable_db_options_,
+               *manual.cfd->GetLatestMutableCFOptions(), mutable_db_options_,
               manual.input_level, manual.output_level, compact_range_options,
               manual.begin, manual.end, &manual.manual_end, &manual_conflict,
               max_file_num_to_ignore, trim_ts)) == nullptr &&
@@ -2670,7 +2676,7 @@ Status DBImpl::WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd,
        return Status::OK();
      }

-      const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();
+      const auto& mutable_cf_options = *cfd->GetLatestMutableCFOptions();
      const auto* vstorage = cfd->current()->storage_info();

      // Skip stalling check if we're below auto-flush and auto-compaction
@@ -2678,7 +2684,7 @@ Status DBImpl::WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd,
      // triggers are so low that stalling is needed for any background work. In
      // that case we shouldn't wait since background work won't be scheduled.
      if (cfd->imm()->NumNotFlushed() <
-              cfd->ioptions().min_write_buffer_number_to_merge &&
+              cfd->ioptions()->min_write_buffer_number_to_merge &&
          vstorage->l0_delay_trigger_count() <
              mutable_cf_options.level0_file_num_compaction_trigger) {
        break;
@@ -2694,7 +2700,7 @@ Status DBImpl::WaitUntilFlushWouldNotStallWrites(ColumnFamilyData* cfd,
              cfd->GetUnflushedMemTableCountForWriteStallCheck(),
              vstorage->l0_delay_trigger_count() + 1,
              vstorage->estimated_compaction_needed_bytes(), mutable_cf_options,
-              cfd->ioptions())
+              *cfd->ioptions())
              .first;
    } while (write_stall_condition != WriteStallCondition::kNormal);
  }
@@ -3673,8 +3679,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    // Compaction makes a copy of the latest MutableCFOptions. It should be used
    // throughout the compaction procedure to make sure consistency. It will
    // eventually be installed into SuperVersion
-    const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();
-    if (!mutable_cf_options.disable_auto_compactions && !cfd->IsDropped()) {
+    auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
+    if (!mutable_cf_options->disable_auto_compactions && !cfd->IsDropped()) {
      // NOTE: try to avoid unnecessary copy of MutableCFOptions if
      // compaction is not necessary. Need to make sure mutex is held
      // until we make a copy in the following code
@@ -3683,7 +3689,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      std::vector<SequenceNumber> snapshot_seqs;
      // This info is not useful for other scenarios, so save querying existing
      // snapshots for those cases.
-      if (cfd->ioptions().compaction_style == kCompactionStyleUniversal &&
+      if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal &&
          cfd->user_comparator()->timestamp_size() == 0) {
        SequenceNumber earliest_write_conflict_snapshot;
        GetSnapshotContext(job_context, &snapshot_seqs,
@@ -3691,7 +3697,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
                           &snapshot_checker);
        assert(is_snapshot_supported_ || snapshots_.empty());
      }
-      c.reset(cfd->PickCompaction(mutable_cf_options, mutable_db_options_,
+      c.reset(cfd->PickCompaction(*mutable_cf_options, mutable_db_options_,
                                  snapshot_seqs, snapshot_checker, log_buffer));
      TEST_SYNC_POINT("DBImpl::BackgroundCompaction():AfterPickCompaction");

@@ -3705,8 +3711,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
          c->column_family_data()
              ->current()
              ->storage_info()
-              ->ComputeCompactionScore(c->immutable_options(),
-                                       c->mutable_cf_options());
+              ->ComputeCompactionScore(*(c->immutable_options()),
+                                       *(c->mutable_cf_options()));
          AddToCompactionQueue(cfd);

          c.reset();
@@ -3754,7 +3760,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:BeforeCompaction",
                             c->column_family_data());
    assert(c->num_input_files(1) == 0);
-    assert(c->column_family_data()->ioptions().compaction_style ==
+    assert(c->column_family_data()->ioptions()->compaction_style ==
           kCompactionStyleFIFO);

    compaction_job_stats.num_input_files = c->num_input_files(0);
@@ -3766,8 +3772,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      c->edit()->DeleteFile(c->level(), f->fd.GetNumber());
    }
    status = versions_->LogAndApply(
-        c->column_family_data(), read_options, write_options, c->edit(),
-        &mutex_, directories_.GetDbDir(),
+        c->column_family_data(), *c->mutable_cf_options(), read_options,
+        write_options, c->edit(), &mutex_, directories_.GetDbDir(),
        /*new_descriptor_log=*/false, /*column_family_options=*/nullptr,
        [&c, &compaction_released](const Status& s) {
          c->ReleaseCompactionFiles(s);
@@ -3775,7 +3781,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
        });
    io_s = versions_->io_status();
    InstallSuperVersionAndScheduleWork(
-        c->column_family_data(), job_context->superversion_contexts.data());
+        c->column_family_data(), job_context->superversion_contexts.data(),
+        *c->mutable_cf_options());
    ROCKS_LOG_BUFFER(log_buffer, "[%s] Deleted %d files\n",
                     c->column_family_data()->GetName().c_str(),
                     c->num_input_files(0));
@@ -3831,7 +3838,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      }
    }
    if (c->compaction_reason() == CompactionReason::kLevelMaxLevelSize &&
-        c->immutable_options().compaction_pri == kRoundRobin) {
+        c->immutable_options()->compaction_pri == kRoundRobin) {
      int start_level = c->start_level();
      if (start_level > 0) {
        auto vstorage = c->input_version()->storage_info();
@@ -3841,23 +3848,26 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      }
    }
    status = versions_->LogAndApply(
-        c->column_family_data(), read_options, write_options, c->edit(),
-        &mutex_, directories_.GetDbDir(),
+        c->column_family_data(), *c->mutable_cf_options(), read_options,
+        write_options, c->edit(), &mutex_, directories_.GetDbDir(),
        /*new_descriptor_log=*/false, /*column_family_options=*/nullptr,
        [&c, &compaction_released](const Status& s) {
          c->ReleaseCompactionFiles(s);
          compaction_released = true;
        });
    io_s = versions_->io_status();
+    // Use latest MutableCFOptions
    InstallSuperVersionAndScheduleWork(
-        c->column_family_data(), job_context->superversion_contexts.data());
+        c->column_family_data(), job_context->superversion_contexts.data(),
+        *c->mutable_cf_options());

    VersionStorageInfo::LevelSummaryStorage tmp;
    c->column_family_data()->internal_stats()->IncBytesMoved(c->output_level(),
                                                             moved_bytes);
    {
      event_logger_.LogToBuffer(log_buffer)
-          << "job" << job_context->job_id << "event" << "trivial_move"
+          << "job" << job_context->job_id << "event"
+          << "trivial_move"
          << "destination_level" << c->output_level() << "files" << moved_files
          << "total_files_size" << moved_bytes;
    }
@@ -3919,15 +3929,15 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
        GetDataDir(c->column_family_data(), 0), stats_, &mutex_,
        &error_handler_, snapshot_seqs, earliest_write_conflict_snapshot,
        snapshot_checker, job_context, table_cache_, &event_logger_,
-        c->mutable_cf_options().paranoid_file_checks,
-        c->mutable_cf_options().report_bg_io_stats, dbname_,
+        c->mutable_cf_options()->paranoid_file_checks,
+        c->mutable_cf_options()->report_bg_io_stats, dbname_,
        &compaction_job_stats, thread_pri, io_tracer_,
        is_manual ? manual_compaction->canceled
                  : kManualCompactionCanceledFalse_,
        db_id_, db_session_id_, c->column_family_data()->GetFullHistoryTsLow(),
        c->trim_ts(), &blob_callback_, &bg_compaction_scheduled_,
        &bg_bottom_compaction_scheduled_);
-    compaction_job.Prepare(std::nullopt /*subcompact to be computed*/);
+    compaction_job.Prepare();

    std::unique_ptr<std::list<uint64_t>::iterator> min_options_file_number_elem;
    if (immutable_db_options().compaction_service != nullptr) {
@@ -3949,11 +3959,13 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      ReleaseOptionsFileNumber(min_options_file_number_elem);
    }

-    status = compaction_job.Install(&compaction_released);
+    status =
+        compaction_job.Install(*c->mutable_cf_options(), &compaction_released);
    io_s = compaction_job.io_status();
    if (status.ok()) {
      InstallSuperVersionAndScheduleWork(
-          c->column_family_data(), job_context->superversion_contexts.data());
+          c->column_family_data(), job_context->superversion_contexts.data(),
+          *c->mutable_cf_options());
    }
    *made_progress = true;
    TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:AfterCompaction",
@@ -4031,8 +4043,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      c->column_family_data()
          ->current()
          ->storage_info()
-          ->ComputeCompactionScore(c->immutable_options(),
-                                   c->mutable_cf_options());
+          ->ComputeCompactionScore(*(c->immutable_options()),
+                                   *(c->mutable_cf_options()));
      if (!cfd->queued_for_compaction()) {
        AddToCompactionQueue(cfd);
      }
@@ -4067,9 +4079,10 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      // We only compacted part of the requested range.  Update *m
      // to the range that is left to be compacted.
      // Universal and FIFO compactions should always compact the whole range
-      assert(m->cfd->ioptions().compaction_style != kCompactionStyleUniversal ||
-             m->cfd->ioptions().num_levels > 1);
-      assert(m->cfd->ioptions().compaction_style != kCompactionStyleFIFO);
+      assert(m->cfd->ioptions()->compaction_style !=
+                 kCompactionStyleUniversal ||
+             m->cfd->ioptions()->num_levels > 1);
+      assert(m->cfd->ioptions()->compaction_style != kCompactionStyleFIFO);
      m->tmp_storage = *m->manual_end;
      m->begin = &m->tmp_storage;
      m->incomplete = true;
@@ -4216,7 +4229,7 @@ void DBImpl::BuildCompactionJobInfo(
    for (const auto fmd : *c->inputs(i)) {
      const FileDescriptor& desc = fmd->fd;
      const uint64_t file_number = desc.GetNumber();
-      auto fn = TableFileName(c->immutable_options().cf_paths, file_number,
+      auto fn = TableFileName(c->immutable_options()->cf_paths, file_number,
                              desc.GetPathId());
      compaction_job_info->input_files.push_back(fn);
      compaction_job_info->input_file_infos.push_back(CompactionFileInfo{
@@ -4229,17 +4242,17 @@ void DBImpl::BuildCompactionJobInfo(
    const FileDescriptor& desc = meta.fd;
    const uint64_t file_number = desc.GetNumber();
    compaction_job_info->output_files.push_back(TableFileName(
-        c->immutable_options().cf_paths, file_number, desc.GetPathId()));
+        c->immutable_options()->cf_paths, file_number, desc.GetPathId()));
    compaction_job_info->output_file_infos.push_back(CompactionFileInfo{
        newf.first, file_number, meta.oldest_blob_file_number});
  }
  compaction_job_info->blob_compression_type =
-      c->mutable_cf_options().blob_compression_type;
+      c->mutable_cf_options()->blob_compression_type;

  // Update BlobFilesInfo.
  for (const auto& blob_file : c->edit()->GetBlobFileAdditions()) {
    BlobFileAdditionInfo blob_file_addition_info(
-        BlobFileName(c->immutable_options().cf_paths.front().path,
+        BlobFileName(c->immutable_options()->cf_paths.front().path,
                     blob_file.GetBlobFileNumber()) /*blob_file_path*/,
        blob_file.GetBlobFileNumber(), blob_file.GetTotalBlobCount(),
        blob_file.GetTotalBlobBytes());
@@ -4250,7 +4263,7 @@ void DBImpl::BuildCompactionJobInfo(
  // Update BlobFilesGarbageInfo.
  for (const auto& blob_file : c->edit()->GetBlobFileGarbages()) {
    BlobFileGarbageInfo blob_file_garbage_info(
-        BlobFileName(c->immutable_options().cf_paths.front().path,
+        BlobFileName(c->immutable_options()->cf_paths.front().path,
                     blob_file.GetBlobFileNumber()) /*blob_file_path*/,
        blob_file.GetBlobFileNumber(), blob_file.GetGarbageBlobCount(),
        blob_file.GetGarbageBlobBytes());
@@ -4272,9 +4285,9 @@ void DBImpl::BuildCompactionJobInfo(
 // for superversion_to_free

 void DBImpl::InstallSuperVersionAndScheduleWork(
-    ColumnFamilyData* cfd, SuperVersionContext* sv_context) {
+    ColumnFamilyData* cfd, SuperVersionContext* sv_context,
+    const MutableCFOptions& mutable_cf_options) {
  mutex_.AssertHeld();
-  const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();

  // Update max_total_in_memory_state_
  size_t old_memtable_size = 0;
@@ -4297,7 +4310,7 @@ void DBImpl::InstallSuperVersionAndScheduleWork(
  bottommost_files_mark_threshold_ = kMaxSequenceNumber;
  standalone_range_deletion_files_mark_threshold_ = kMaxSequenceNumber;
  for (auto* my_cfd : *versions_->GetColumnFamilySet()) {
-    if (!my_cfd->ioptions().allow_ingest_behind) {
+    if (!my_cfd->ioptions()->allow_ingest_behind) {
      bottommost_files_mark_threshold_ = std::min(
          bottommost_files_mark_threshold_,
          my_cfd->current()->storage_info()->bottommost_files_mark_threshold());
@@ -103,8 +103,8 @@ Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
    cfd = cfh->cfd();
  }
  int output_level =
-      (cfd->ioptions().compaction_style == kCompactionStyleUniversal ||
-       cfd->ioptions().compaction_style == kCompactionStyleFIFO)
+      (cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
+       cfd->ioptions()->compaction_style == kCompactionStyleFIFO)
          ? level
          : level + 1;
  return RunManualCompaction(
@@ -239,7 +239,7 @@ void DBImpl::TEST_GetAllBlockCaches(
  for (auto cfd : *versions_->GetColumnFamilySet()) {
    if (const auto bbto =
            cfd->GetCurrentMutableCFOptions()
-                .table_factory->GetOptions<BlockBasedTableOptions>()) {
+                ->table_factory->GetOptions<BlockBasedTableOptions>()) {
      cache_set->insert(bbto->block_cache.get());
    }
  }
@@ -267,7 +267,7 @@ Status DBImpl::TEST_GetLatestMutableCFOptions(
  InstrumentedMutexLock l(&mutex_);

  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
-  *mutable_cf_options = cfh->cfd()->GetLatestMutableCFOptions();
+  *mutable_cf_options = *cfh->cfd()->GetLatestMutableCFOptions();
  return Status::OK();
 }

@@ -45,8 +45,8 @@ Status DBImpl::SuggestCompactRange(ColumnFamilyHandle* column_family,
    }
    // Since we have some more files to compact, we should also recompute
    // compaction score
-    vstorage->ComputeCompactionScore(cfd->ioptions(),
-                                     cfd->GetLatestMutableCFOptions());
+    vstorage->ComputeCompactionScore(*cfd->ioptions(),
+                                     *cfd->GetLatestMutableCFOptions());
    EnqueuePendingCompaction(cfd);
    MaybeScheduleFlushOrCompaction();
  }
@@ -146,11 +146,13 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
                   f->user_defined_timestamps_persisted);
    }

-    status = versions_->LogAndApply(cfd, read_options, write_options, &edit,
-                                    &mutex_, directories_.GetDbDir());
+    status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
+                                    read_options, write_options, &edit, &mutex_,
+                                    directories_.GetDbDir());
    if (status.ok()) {
      InstallSuperVersionAndScheduleWork(
-          cfd, job_context.superversion_contexts.data());
+          cfd, job_context.superversion_contexts.data(),
+          *cfd->GetLatestMutableCFOptions());
    }
  }  // lock released here
  LogFlush(immutable_db_options_.info_log);
@@ -159,7 +159,6 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,

  // Get obsolete files.  This function will also update the list of
  // pending files in VersionSet().
-  assert(versions_);
  versions_->GetObsoleteFiles(
      &job_context->sst_delete_files, &job_context->blob_delete_files,
      &job_context->manifest_delete_files, job_context->min_pending_output);
@@ -1040,7 +1039,7 @@ std::set<std::string> DBImpl::CollectAllDBPaths() {
    all_db_paths.insert(NormalizePath(db_path.path));
  }
  for (const auto* cfd : *versions_->GetColumnFamilySet()) {
-    for (const auto& cf_path : cfd->ioptions().cf_paths) {
+    for (const auto& cf_path : cfd->ioptions()->cf_paths) {
      all_db_paths.insert(NormalizePath(cf_path.path));
    }
  }
@@ -140,8 +140,10 @@ Status DBImplFollower::TryCatchUpWithLeader() {
          // to super versions in a lock free manner, it checks the earliest
          // sequence number to detect if there was a change in version in
          // the meantime.
+          const MutableCFOptions mutable_cf_options =
+              *cfd->GetLatestMutableCFOptions();
          MemTable* new_mem = cfd->ConstructNewMemtable(
-              cfd->GetLatestMutableCFOptions(), versions_->LastSequence());
+              mutable_cf_options, versions_->LastSequence());
          cfd->mem()->SetNextLogNumber(cfd->GetLogNumber());
          cfd->mem()->ConstructFragmentedRangeTombstones();
          cfd->imm()->Add(cfd->mem(), &job_context.memtables_to_free);
@@ -16,6 +16,7 @@

 namespace ROCKSDB_NAMESPACE {

+
 DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
                               const std::string& dbname)
    : DBImpl(db_options, dbname, /*seq_per_batch*/ false,
@@ -172,7 +173,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& _read_options,
          : latest_snapshot;
  ReadCallback* read_callback = nullptr;  // No read callback provided.
  auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, cfd->ioptions(), super_version->mutable_cf_options,
+      env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options,
      super_version->current, read_seq,
      super_version->mutable_cf_options.max_sequential_skip_in_iterations,
      super_version->version_number, read_callback);
@@ -239,7 +240,7 @@ Status DBImplReadOnly::NewIterators(
  assert(cfd_to_sv.size() == column_families.size());
  for (auto [cfd, sv] : cfd_to_sv) {
    auto* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, cfd->ioptions(), sv->mutable_cf_options,
+        env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
        sv->current, read_seq,
        sv->mutable_cf_options.max_sequential_skip_in_iterations,
        sv->version_number, read_callback);
@@ -275,8 +276,7 @@ Status OpenForReadOnlyCheckExistence(const DBOptions& db_options,
 }  // namespace

 Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
-                           std::unique_ptr<DB>* dbptr,
-                           bool /*error_if_wal_file_exists*/) {
+                           DB** dbptr, bool /*error_if_wal_file_exists*/) {
  Status s = OpenForReadOnlyCheckExistence(options, dbname);
  if (!s.ok()) {
    return s;
@@ -310,7 +310,7 @@ Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
 Status DB::OpenForReadOnly(
    const DBOptions& db_options, const std::string& dbname,
    const std::vector<ColumnFamilyDescriptor>& column_families,
-    std::vector<ColumnFamilyHandle*>* handles, std::unique_ptr<DB>* dbptr,
+    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
    bool error_if_wal_file_exists) {
  // If dbname does not exist in the file system, should not do anything
  Status s = OpenForReadOnlyCheckExistence(db_options, dbname);
@@ -326,7 +326,7 @@ Status DB::OpenForReadOnly(
 Status DBImplReadOnly::OpenForReadOnlyWithoutCheck(
    const DBOptions& db_options, const std::string& dbname,
    const std::vector<ColumnFamilyDescriptor>& column_families,
-    std::vector<ColumnFamilyHandle*>* handles, std::unique_ptr<DB>* dbptr,
+    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
    bool error_if_wal_file_exists) {
  *dbptr = nullptr;
  handles->clear();
@@ -357,7 +357,7 @@ Status DBImplReadOnly::OpenForReadOnlyWithoutCheck(
  impl->mutex_.Unlock();
  sv_context.Clean();
  if (s.ok()) {
-    dbptr->reset(impl);
+    *dbptr = impl;
    for (auto* h : *handles) {
      impl->NewThreadStatusCfInfo(
          static_cast_with_check<ColumnFamilyHandleImpl>(h)->cfd());
@@ -372,4 +372,5 @@ Status DBImplReadOnly::OpenForReadOnlyWithoutCheck(
  return s;
 }

+
 }  // namespace ROCKSDB_NAMESPACE
@@ -5,6 +5,7 @@

 #pragma once

+
 #include <string>
 #include <vector>

@@ -171,7 +172,7 @@ class DBImplReadOnly : public DBImpl {
  static Status OpenForReadOnlyWithoutCheck(
      const DBOptions& db_options, const std::string& dbname,
      const std::vector<ColumnFamilyDescriptor>& column_families,
-      std::vector<ColumnFamilyHandle*>* handles, std::unique_ptr<DB>* dbptr,
+      std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
      bool error_if_wal_file_exists = false);
  friend class DB;
 };
@@ -55,9 +55,9 @@ Status DBImplSecondary::Recover(
  // Initial max_total_in_memory_state_ before recovery logs.
  max_total_in_memory_state_ = 0;
  for (auto cfd : *versions_->GetColumnFamilySet()) {
-    const auto& mutable_cf_options = cfd->GetLatestMutableCFOptions();
-    max_total_in_memory_state_ += mutable_cf_options.write_buffer_size *
-                                  mutable_cf_options.max_write_buffer_number;
+    auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
+    max_total_in_memory_state_ += mutable_cf_options->write_buffer_size *
+                                  mutable_cf_options->max_write_buffer_number;
  }
  if (s.ok()) {
    default_cf_handle_ = new ColumnFamilyHandleImpl(
@@ -270,8 +270,10 @@ Status DBImplSecondary::RecoverLogFiles(
          if (!cfd->mem()->IsEmpty() &&
              (curr_log_num == std::numeric_limits<uint64_t>::max() ||
               curr_log_num != log_number)) {
-            MemTable* new_mem = cfd->ConstructNewMemtable(
-                cfd->GetLatestMutableCFOptions(), seq_of_batch);
+            const MutableCFOptions mutable_cf_options =
+                *cfd->GetLatestMutableCFOptions();
+            MemTable* new_mem =
+                cfd->ConstructNewMemtable(mutable_cf_options, seq_of_batch);
            cfd->mem()->SetNextLogNumber(log_number);
            cfd->mem()->ConstructFragmentedRangeTombstones();
            cfd->imm()->Add(cfd->mem(), &job_context->memtables_to_free);
@@ -531,7 +533,7 @@ ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl(
  snapshot = versions_->LastSequence();
  assert(snapshot != kMaxSequenceNumber);
  auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, cfh->cfd()->ioptions(),
+      env_, read_options, *cfh->cfd()->ioptions(),
      super_version->mutable_cf_options, super_version->current, snapshot,
      super_version->mutable_cf_options.max_sequential_skip_in_iterations,
      super_version->version_number, read_callback, cfh, expose_blob_index,
@@ -735,8 +737,7 @@ Status DBImplSecondary::TryCatchUpWithPrimary() {
 }

 Status DB::OpenAsSecondary(const Options& options, const std::string& dbname,
-                           const std::string& secondary_path,
-                           std::unique_ptr<DB>* dbptr) {
+                           const std::string& secondary_path, DB** dbptr) {
  *dbptr = nullptr;

  DBOptions db_options(options);
@@ -758,7 +759,7 @@ Status DB::OpenAsSecondary(
    const DBOptions& db_options, const std::string& dbname,
    const std::string& secondary_path,
    const std::vector<ColumnFamilyDescriptor>& column_families,
-    std::vector<ColumnFamilyHandle*>* handles, std::unique_ptr<DB>* dbptr) {
+    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr) {
  *dbptr = nullptr;

  DBOptions tmp_opts(db_options);
@@ -825,7 +826,7 @@ Status DB::OpenAsSecondary(
  impl->mutex_.Unlock();
  sv_context.Clean();
  if (s.ok()) {
-    dbptr->reset(impl);
+    *dbptr = impl;
    for (auto h : *handles) {
      impl->NewThreadStatusCfInfo(
          static_cast_with_check<ColumnFamilyHandleImpl>(h)->cfd());
@@ -863,15 +864,16 @@ Status DBImplSecondary::CompactWithoutInstallation(
  ColumnFamilyMetaData cf_meta;
  version->GetColumnFamilyMetaData(&cf_meta);

+  const MutableCFOptions* mutable_cf_options = cfd->GetLatestMutableCFOptions();
+  ColumnFamilyOptions cf_options = cfd->GetLatestCFOptions();
  VersionStorageInfo* vstorage = version->storage_info();

  // Use comp_options to reuse some CompactFiles functions
  CompactionOptions comp_options;
  comp_options.compression = kDisableCompressionOption;
  comp_options.output_file_size_limit = MaxFileSizeForLevel(
-      cfd->GetLatestMutableCFOptions(), input.output_level,
-      cfd->ioptions().compaction_style, vstorage->base_level(),
-      cfd->ioptions().level_compaction_dynamic_level_bytes);
+      *mutable_cf_options, input.output_level, cf_options.compaction_style,
+      vstorage->base_level(), cf_options.level_compaction_dynamic_level_bytes);

  std::vector<CompactionInputFiles> input_files;
  Status s = cfd->compaction_picker()->GetCompactionInputsFromFileNumbers(
@@ -884,7 +886,7 @@ Status DBImplSecondary::CompactWithoutInstallation(
  assert(cfd->compaction_picker());
  c.reset(cfd->compaction_picker()->CompactFiles(
      comp_options, input_files, input.output_level, vstorage,
-      cfd->GetLatestMutableCFOptions(), mutable_db_options_, 0));
+      *mutable_cf_options, mutable_db_options_, 0));
  assert(c != nullptr);

  c->FinalizeInputInfo(version);
@@ -915,8 +917,6 @@ Status DBImplSecondary::CompactWithoutInstallation(
      options.canceled ? *options.canceled : kManualCompactionCanceledFalse_,
      input.db_id, db_session_id_, secondary_path_, input, result);

-  compaction_job.Prepare();
-
  mutex_.Unlock();
  s = compaction_job.Run();
  mutex_.Lock();
@@ -1057,4 +1057,5 @@ Status DB::OpenAndCompact(
                        output, override_options);
 }

+
 }  // namespace ROCKSDB_NAMESPACE
@@ -5,6 +5,7 @@

 #pragma once

+
 #include <string>
 #include <vector>

@@ -67,7 +67,7 @@ Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family,
    return s;
  }
  auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
-  if (!cfh->cfd()->ioptions().merge_operator) {
+  if (!cfh->cfd()->ioptions()->merge_operator) {
    return Status::NotSupported("Provide a merge_operator when opening DB");
  } else {
    return DB::Merge(o, column_family, key, val);
@@ -232,8 +232,8 @@ Status DBImpl::IngestWBWI(std::shared_ptr<WriteBatchWithIndex> wbwi,
      return s;
    }
    WBWIMemTable* wbwi_memtable =
-        new WBWIMemTable(wbwi, cfd->user_comparator(), cf_id, &cfd->ioptions(),
-                         &cfd->GetLatestMutableCFOptions(), stat);
+        new WBWIMemTable(wbwi, cfd->user_comparator(), cf_id, cfd->ioptions(),
+                         cfd->GetLatestMutableCFOptions(), stat);
    wbwi_memtable->Ref();
    wbwi_memtable->AssignSequenceNumbers(assigned_seqno);
    // This is needed to keep the WAL that contains Prepare alive until
@@ -1558,8 +1558,7 @@ IOStatus DBImpl::WriteToWAL(const WriteBatch& merged_batch,
                            const WriteOptions& write_options,
                            log::Writer* log_writer, uint64_t* log_used,
                            uint64_t* log_size,
-                            LogFileNumberSize& log_file_number_size,
-                            SequenceNumber sequence) {
+                            LogFileNumberSize& log_file_number_size) {
  assert(log_size != nullptr);

  Slice log_entry = WriteBatchInternal::Contents(&merged_batch);
@@ -1585,7 +1584,7 @@ IOStatus DBImpl::WriteToWAL(const WriteBatch& merged_batch,
  if (!io_s.ok()) {
    return io_s;
  }
-  io_s = log_writer->AddRecord(write_options, log_entry, sequence);
+  io_s = log_writer->AddRecord(write_options, log_entry);

  if (UNLIKELY(needs_locking)) {
    log_write_mutex_.Unlock();
@@ -1635,7 +1634,7 @@ IOStatus DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
  write_options.rate_limiter_priority =
      write_group.leader->rate_limiter_priority;
  io_s = WriteToWAL(*merged_batch, write_options, log_writer, log_used,
-                    &log_size, log_file_number_size, sequence);
+                    &log_size, log_file_number_size);
  if (to_be_cached_state) {
    cached_recoverable_state_ = *to_be_cached_state;
    cached_recoverable_state_empty_ = false;
@@ -1761,7 +1760,7 @@ IOStatus DBImpl::ConcurrentWriteToWAL(
  write_options.rate_limiter_priority =
      write_group.leader->rate_limiter_priority;
  io_s = WriteToWAL(*merged_batch, write_options, log_writer, log_used,
-                    &log_size, log_file_number_size, sequence);
+                    &log_size, log_file_number_size);
  if (to_be_cached_state) {
    cached_recoverable_state_ = *to_be_cached_state;
    cached_recoverable_state_empty_ = false;
@@ -2422,9 +2421,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
  }
  uint64_t new_log_number =
      creating_new_log ? versions_->NewFileNumber() : logfile_number_;
-  // For use outside of holding DB mutex
-  const MutableCFOptions mutable_cf_options_copy =
-      cfd->GetLatestMutableCFOptions();
+  const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();

  // Set memtable_info for memtable sealed callback
  // TODO: memtable_info for `new_imm`
@@ -2434,7 +2431,7 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
  memtable_info.earliest_seqno = cfd->mem()->GetEarliestSequenceNumber();
  memtable_info.num_entries = cfd->mem()->NumEntries();
  memtable_info.num_deletes = cfd->mem()->NumDeletion();
-  if (!cfd->ioptions().persist_user_defined_timestamps &&
+  if (!cfd->ioptions()->persist_user_defined_timestamps &&
      cfd->user_comparator()->timestamp_size() > 0) {
    const Slice& newest_udt = cfd->mem()->GetNewestUDT();
    memtable_info.newest_udt.assign(newest_udt.data(), newest_udt.size());
@@ -2443,22 +2440,13 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
  // flush happens before logging, but that should be ok.
  int num_imm_unflushed = cfd->imm()->NumNotFlushed();
  const auto preallocate_block_size =
-      GetWalPreallocateBlockSize(mutable_cf_options_copy.write_buffer_size);
+      GetWalPreallocateBlockSize(mutable_cf_options.write_buffer_size);
  mutex_.Unlock();
  if (creating_new_log) {
-    PredecessorWALInfo info;
-    log_write_mutex_.Lock();
-    if (!logs_.empty()) {
-      log::Writer* cur_log_writer = logs_.back().writer;
-      info = PredecessorWALInfo(cur_log_writer->get_log_number(),
-                                cur_log_writer->file()->GetFileSize(),
-                                cur_log_writer->GetLastSeqnoRecorded());
-    }
-    log_write_mutex_.Unlock();
    // TODO: Write buffer size passed in should be max of all CF's instead
    // of mutable_cf_options.write_buffer_size.
    io_s = CreateWAL(write_options, new_log_number, recycle_log_number,
-                     preallocate_block_size, info, &new_log);
+                     preallocate_block_size, &new_log);
    if (s.ok()) {
      s = io_s;
    }
@@ -2476,8 +2464,8 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
    } else {
      seq = versions_->LastSequence();
    }
-    new_mem = cfd->ConstructNewMemtable(mutable_cf_options_copy,
-                                        /*earliest_seq=*/seq);
+    new_mem =
+        cfd->ConstructNewMemtable(mutable_cf_options, /*earliest_seq=*/seq);
    context->superversion_context.NewSuperVersion();

    ROCKS_LOG_INFO(immutable_db_options_.info_log,
@@ -2632,7 +2620,8 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context,
  }
  new_mem->Ref();
  cfd->SetMemtable(new_mem);
-  InstallSuperVersionAndScheduleWork(cfd, &context->superversion_context);
+  InstallSuperVersionAndScheduleWork(cfd, &context->superversion_context,
+                                     mutable_cf_options);

  // Notify client that memtable is sealed, now that we have successfully
  // installed a new memtable
@@ -97,20 +97,12 @@ class CorruptionFS : public FileSystemWrapper {
        for (size_t i = 0; i < num_reqs; ++i) {
          FSReadRequest& req = reqs[i];
          if (fs_.fs_buffer_) {
-            // See https://github.com/facebook/rocksdb/pull/13195 for why we
-            // want to set up our test implementation for FSAllocationPtr this
-            // way.
-            char* internalData = new char[req.len];
-            req.status = Read(req.offset, req.len, options, &req.result,
-                              internalData, dbg);
-
-            Slice* internalSlice = new Slice(internalData, req.len);
-            FSAllocationPtr internalPtr(internalSlice, [](void* ptr) {
-              delete[] static_cast<const char*>(
-                  static_cast<Slice*>(ptr)->data_);
-              delete static_cast<Slice*>(ptr);
+            FSAllocationPtr buffer(new char[req.len], [](void* ptr) {
+              delete[] static_cast<char*>(ptr);
            });
-            req.fs_scratch = std::move(internalPtr);
+            req.fs_scratch = std::move(buffer);
+            req.status = Read(req.offset, req.len, options, &req.result,
+                              static_cast<char*>(req.fs_scratch.get()), dbg);
          } else {
            req.status = Read(req.offset, req.len, options, &req.result,
                              req.scratch, dbg);
@@ -332,6 +332,7 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorBlobs) {
 }
 }  // namespace ROCKSDB_NAMESPACE

+
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -5,11 +5,11 @@

 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
-#include "rocksdb/merge_operator.h"
 #include "rocksdb/perf_context.h"
 #include "rocksdb/utilities/debug.h"
 #include "table/block_based/block_builder.h"
 #include "test_util/sync_point.h"
+#include "rocksdb/merge_operator.h"
 #include "utilities/fault_injection_env.h"
 #include "utilities/merge_operators.h"
 #include "utilities/merge_operators/sortlist.h"
@@ -1451,6 +1451,7 @@ TEST_F(DBOptionsTest, ChangeCompression) {
  SyncPoint::GetInstance()->DisableProcessing();
 }

+
 TEST_F(DBOptionsTest, BottommostCompressionOptsWithFallbackType) {
  // Verify the bottommost compression options still take effect even when the
  // bottommost compression type is left at its default value. Verify for both
@@ -226,6 +226,7 @@ TEST_P(DBRateLimiterOnReadTest, Iterator) {
  ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER));
 }

+
 TEST_P(DBRateLimiterOnReadTest, VerifyChecksum) {
  if (use_direct_io_ && !IsDirectIOSupported()) {
    return;
@@ -270,6 +271,7 @@ TEST_P(DBRateLimiterOnReadTest, VerifyFileChecksums) {
  ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER));
 }

+
 class DBRateLimiterOnWriteTest : public DBTestBase {
 public:
  explicit DBRateLimiterOnWriteTest()
@@ -1991,6 +1991,7 @@ TEST_F(DBSSTTest, DBWithSFMForBlobFilesAtomicFlush) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
 }

+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -198,6 +198,7 @@ TEST_F(DBStatisticsTest, ExcludeTickers) {
  ASSERT_GT(options.statistics->getTickerCount(BYTES_READ), 0);
 }

+
 TEST_F(DBStatisticsTest, VerifyChecksumReadStat) {
  Options options = CurrentOptions();
  options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
@@ -25,6 +25,7 @@
 #include "util/atomic.h"
 #include "util/random.h"

+
 namespace ROCKSDB_NAMESPACE {

 // A helper function that ensures the table properties returned in
@@ -743,6 +744,7 @@ INSTANTIATE_TEST_CASE_P(DBTablePropertiesTest, DBTablePropertiesTest,

 }  // namespace ROCKSDB_NAMESPACE

+
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -703,6 +703,7 @@ TEST_P(DBTestTailingIterator, SeekToFirstWithUpperBoundBug) {

 }  // namespace ROCKSDB_NAMESPACE

+
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -1161,6 +1161,7 @@ class DelayFilterFactory : public CompactionFilterFactory {
 };
 }  // anonymous namespace

+
 static std::string CompressibleString(Random* rnd, int len) {
  std::string r;
  test::CompressibleString(rnd, 0.8, len, &r);
@@ -3409,9 +3410,7 @@ class ModelDB : public DB {
    return Status::NotSupported();
  }

-  Status DEPRECATED_DeleteFile(std::string /*name*/) override {
-    return Status::OK();
-  }
+  Status DeleteFile(std::string /*name*/) override { return Status::OK(); }

  Status GetUpdatesSince(
      ROCKSDB_NAMESPACE::SequenceNumber,
@@ -3716,139 +3715,6 @@ TEST_F(DBTest, BlockBasedTablePrefixIndexTest) {
  ASSERT_EQ("v2", Get("k2"));
 }

-TEST_F(DBTest, SetOptionsEffectiveInSuperVersions) {
-  // Basically, to test the SetOptions take effect with (and only with)
-  // new SuperVersions, and remain in effect through some things like
-  // flush and compaction, we use some queries that depend on the current
-  // prefix extractor.
-  //
-  // Making the semantics of read options dependent on the current state of
-  // mutable options is kind of an anti-pattern that prefix_seek_opt_in_only
-  // is helping to phase out. However, this is useful for rather directly
-  // testing the expected behavior of mutable options handling.
-  ReadOptions ropts;
-  ropts.prefix_same_as_start = true;
-  Options options = CurrentOptions();
-  options.prefix_extractor.reset(NewFixedPrefixTransform(5));
-  options.prefix_seek_opt_in_only = false;
-  Reopen(options);
-
-  ASSERT_OK(Put("goat1", "g1"));
-  ASSERT_OK(Put("goat2", "g2"));
-
-  std::unique_ptr<Iterator> iter(db_->NewIterator(ropts));
-
-  auto VerifyTransform4 = [&](int caller_line) {
-    SCOPED_TRACE("Called from " + std::to_string(caller_line));
-    // Nothing with this prefix
-    iter->Seek("game1");
-    ASSERT_OK(iter->status());
-    ASSERT_FALSE(iter->Valid());
-
-    iter->Seek("goat1");
-    ASSERT_OK(iter->status());
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("goat1", iter->key());
-    iter->Next();
-    ASSERT_OK(iter->status());
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("goat2", iter->key());
-  };
-
-  auto VerifyTransform5 = [&](int caller_line) {
-    SCOPED_TRACE("Called from " + std::to_string(caller_line));
-    // Nothing with this prefix
-    iter->Seek("game1");
-    ASSERT_OK(iter->status());
-    ASSERT_FALSE(iter->Valid());
-
-    iter->Seek("goat1");
-    ASSERT_OK(iter->status());
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("goat1", iter->key());
-    iter->Next();
-    ASSERT_OK(iter->status());
-    ASSERT_FALSE(iter->Valid());
-
-    iter->Seek("goat2");
-    ASSERT_OK(iter->status());
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("goat2", iter->key());
-    iter->Next();
-    ASSERT_OK(iter->status());
-    ASSERT_FALSE(iter->Valid());
-  };
-
-  for (int i = 0;; ++i) {
-    SCOPED_TRACE("Iteration " + std::to_string(i));
-    // Baseline
-    VerifyTransform5(__LINE__);
-
-    if (i == 0) {
-      // Test a "normal" change with nothing happening in parallel
-      ASSERT_OK(db_->SetOptions({{"prefix_extractor", "fixed:4"}}));
-
-      // Iterator still uses old superversion
-      VerifyTransform5(__LINE__);
-
-      // Refresh updates the SuperVersion
-      ASSERT_OK(iter->Refresh());
-    } else if (i == 1) {
-      // Test a setting change in parallel with flush
-      iter = nullptr;
-      SyncPoint::GetInstance()->DisableProcessing();
-      SyncPoint::GetInstance()->ClearAllCallBacks();
-      SyncPoint::GetInstance()->SetCallBack(
-          "FlushJob::WriteLevel0Table:num_memtables", [&](void*) {
-            // During flush, without DB mutex held
-            ASSERT_OK(db_->SetOptions({{"prefix_extractor", "fixed:4"}}));
-            iter.reset(db_->NewIterator(ropts));
-            VerifyTransform4(__LINE__);
-          });
-      SyncPoint::GetInstance()->EnableProcessing();
-      ASSERT_OK(Flush());
-      SyncPoint::GetInstance()->DisableProcessing();
-      SyncPoint::GetInstance()->ClearAllCallBacks();
-      // Callback was called
-      ASSERT_NE(iter, nullptr);
-    } else if (i == 2) {
-      // Test a setting change in parallel with compaction
-      iter = nullptr;
-      SyncPoint::GetInstance()->DisableProcessing();
-      SyncPoint::GetInstance()->ClearAllCallBacks();
-      SyncPoint::GetInstance()->SetCallBack(
-          "CompactionJob::Run():EndStatusSet", [&](void*) {
-            // During compaction, without DB mutex held
-            ASSERT_OK(db_->SetOptions({{"prefix_extractor", "fixed:4"}}));
-            iter.reset(db_->NewIterator(ropts));
-            VerifyTransform4(__LINE__);
-          });
-      SyncPoint::GetInstance()->EnableProcessing();
-      // Need data overlapping that L0 file to prevent trivial move
-      ASSERT_OK(Put("aaaaa", "a"));
-      ASSERT_OK(Put("zzzzz", "a"));
-      ASSERT_OK(CompactRange({}, {}, {}));
-      SyncPoint::GetInstance()->DisableProcessing();
-      SyncPoint::GetInstance()->ClearAllCallBacks();
-      // Callback was called
-      ASSERT_NE(iter, nullptr);
-    } else {
-      break;
-    }
-    // Change has taken effect
-    VerifyTransform4(__LINE__);
-    // Same after a new iterator (in case a new SuperVersion reverted the
-    // setting)
-    iter.reset(db_->NewIterator(ropts));
-    VerifyTransform4(__LINE__);
-
-    // Back to baseline setting
-    ASSERT_OK(db_->SetOptions({{"prefix_extractor", "fixed:5"}}));
-    // New iterator uses latest SuperVersion
-    iter.reset(db_->NewIterator(ropts));
-  }
-}
-
 TEST_F(DBTest, BlockBasedTablePrefixHashIndexTest) {
  // create a DB with block prefix index
  BlockBasedTableOptions table_options;
@@ -4512,6 +4378,7 @@ TEST_F(DBTest, ConcurrentMemtableNotSupported) {
  ASSERT_NOK(db_->CreateColumnFamily(cf_options, "name", &handle));
 }

+
 TEST_F(DBTest, SanitizeNumThreads) {
  for (int attempt = 0; attempt < 2; attempt++) {
    const size_t kTotalTasks = 8;
@@ -5373,7 +5240,7 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel) {
  db_->GetColumnFamilyMetaData(&cf_meta);
  for (const auto& file : cf_meta.levels[4].files) {
    listener->SetExpectedFileName(dbname_ + file.name);
-    ASSERT_OK(dbfull()->DEPRECATED_DeleteFile(file.name));
+    ASSERT_OK(dbfull()->DeleteFile(file.name));
  }
  listener->VerifyMatchedCount(cf_meta.levels[4].files.size());

@@ -5921,6 +5788,7 @@ TEST_F(DBTest, FileCreationRandomFailure) {
  }
 }

+
 TEST_F(DBTest, DynamicMiscOptions) {
  // Test max_sequential_skip_in_iterations
  Options options;
@@ -7362,6 +7230,7 @@ TEST_F(DBTest, ReusePinnableSlice) {
            1);
 }

+
 TEST_F(DBTest, DeletingOldWalAfterDrop) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
      {{"Test:AllowFlushes", "DBImpl::BGWorkFlush"},
@@ -7486,6 +7355,7 @@ TEST_F(DBTest, LargeBlockSizeTest) {
  ASSERT_NOK(TryReopenWithColumnFamilies({"default", "pikachu"}, options));
 }

+
 TEST_F(DBTest, CreationTimeOfOldestFile) {
  const int kNumKeysPerFile = 32;
  const int kNumLevelFiles = 2;
@@ -135,6 +135,7 @@ TEST_F(DBTest2, PartitionedIndexUserToInternalKey) {
  }
 }

+
 class PrefixFullBloomWithReverseComparator
    : public DBTestBase,
      public ::testing::WithParamInterface<bool> {
@@ -1976,6 +1977,7 @@ TEST_F(DBTest2, CompactionStall) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

+
 TEST_F(DBTest2, FirstSnapshotTest) {
  Options options;
  options.write_buffer_size = 100000;  // Small write buffer
@@ -3598,6 +3600,7 @@ TEST_F(DBTest2, OptimizeForSmallDB) {
  value.Reset();
 }

+
 TEST_F(DBTest2, IterRaceFlush1) {
  ASSERT_OK(Put("foo", "v1"));

@@ -4101,6 +4104,7 @@ TEST_F(DBTest2, ReadCallbackTest) {
  }
 }

+
 TEST_F(DBTest2, LiveFilesOmitObsoleteFiles) {
  // Regression test for race condition where an obsolete file is returned to
  // user as a "live file" but then deleted, all while file deletions are
@@ -4428,7 +4432,7 @@ TEST_F(DBTest2, TraceAndReplay) {
      db2->NewDefaultReplayer(handles, std::move(trace_reader), &replayer));

  TraceExecutionResultHandler res_handler;
-  std::function<void(Status, std::unique_ptr<TraceRecordResult>&&)> res_cb =
+  std::function<void(Status, std::unique_ptr<TraceRecordResult> &&)> res_cb =
      [&res_handler](Status exec_s, std::unique_ptr<TraceRecordResult>&& res) {
        ASSERT_TRUE(exec_s.ok() || exec_s.IsNotSupported());
        if (res != nullptr) {
@@ -5160,6 +5164,7 @@ TEST_F(DBTest2, TraceWithFilter) {
  ASSERT_EQ(count, 6);
 }

+
 TEST_F(DBTest2, PinnableSliceAndMmapReads) {
  Options options = CurrentOptions();
  options.env = env_;
@@ -7688,6 +7693,7 @@ TEST_F(DBTest2, RecoverEpochNumber) {
  }
 }

+
 TEST_F(DBTest2, RenameDirectory) {
  Options options = CurrentOptions();
  DestroyAndReopen(options);
@@ -122,6 +122,7 @@ DBTestBase::~DBTestBase() {
 }

 bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) {
+
  if ((skip_mask & kSkipUniversalCompaction) &&
      (option_config == kUniversalCompaction ||
       option_config == kUniversalCompactionMultiLevel ||
@@ -1207,6 +1208,7 @@ std::string DBTestBase::FilesPerLevel(int cf) {
  return result;
 }

+
 std::vector<uint64_t> DBTestBase::GetBlobFileNumbers() {
  VersionSet* const versions = dbfull()->GetVersionSet();
  assert(versions);
@@ -1665,7 +1667,7 @@ void DBTestBase::VerifyDBFromMap(
    data_iter = true_data.begin();
    for (iter->SeekToFirst(); iter->Valid(); iter->Next(), ++data_iter) {
      ASSERT_EQ(iter->key().ToString(), data_iter->first);
-      ASSERT_EQ(iter->value().ToString(), data_iter->second);
+        ASSERT_EQ(iter->value().ToString(), data_iter->second);
      iter_cnt++;
      total_reads++;
    }
@@ -1681,7 +1683,7 @@ void DBTestBase::VerifyDBFromMap(
    auto data_rev = true_data.rbegin();
    for (iter->SeekToLast(); iter->Valid(); iter->Prev(), data_rev++) {
      ASSERT_EQ(iter->key().ToString(), data_rev->first);
-      ASSERT_EQ(iter->value().ToString(), data_rev->second);
+        ASSERT_EQ(iter->value().ToString(), data_rev->second);
      iter_cnt++;
      total_reads++;
    }
@@ -1763,6 +1765,7 @@ void DBTestBase::VerifyDBInternal(
  iter->~InternalIterator();
 }

+
 uint64_t DBTestBase::GetNumberOfSstFilesForColumnFamily(
    DB* db, std::string column_family_name) {
  std::vector<LiveFileMetaData> metadata;
@@ -2369,6 +2369,7 @@ TEST_F(DBTestUniversalCompaction2, PeriodicCompactionOffpeak) {

 }  // namespace ROCKSDB_NAMESPACE

+
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -1838,194 +1838,9 @@ class RecoveryTestHelper {
  }
 };

-TEST_F(DBWALTest, TrackAndVerifyWALsRecycleWAL) {
-  Options options = CurrentOptions();
-  options.avoid_flush_during_shutdown = true;
-  options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
-  options.recycle_log_file_num = 1;
-  options.track_and_verify_wals = true;
-
-  DestroyAndReopen(options);
-
-  ASSERT_OK(Put("key_ignore", "wal_to_recycle"));
-  ASSERT_OK(Put("key_ignore1", "wal_to_recycle"));
-  ASSERT_OK(Put("key_ignore2", "wal_to_recycle"));
-  ASSERT_OK(Flush());
-
-  ASSERT_OK(Put("key_ignore", "wal_to_recycle"));
-  ASSERT_OK(Put("key_ignore1", "wal_to_recycle"));
-  ASSERT_OK(Put("key_ignore2", "wal_to_recycle"));
-  ASSERT_OK(Flush());
-
-  // Stop background flush to avoid deleting any WAL
-  options.env->SetBackgroundThreads(1, Env::HIGH);
-  test::SleepingBackgroundTask sleeping_task;
-  options.env->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
-                        &sleeping_task, Env::Priority::HIGH);
-
-  // Recycle the first WAL
-  ASSERT_OK(Put("key1", "old_value"));
-  // Recycle the second WAL
-  ASSERT_OK(dbfull()->TEST_SwitchWAL());
-  ASSERT_OK(Put("key1", "new_value"));
-
-  // Create a WAL hole on sequence number by truncating the first WAL to 0 byte
-  VectorWalPtr log_files;
-  ASSERT_OK(db_->GetSortedWalFiles(log_files));
-  ASSERT_EQ(log_files.size(), 2);
-  std::string log_name = LogFileName(dbname_, log_files.front()->LogNumber());
-  Close();
-  // Drop `Put("key1", "old_value")` in the first WAL
-  ASSERT_OK(test::TruncateFile(options.env, log_name, 0 /* new_length */));
-
-  Status s = DB::Open(options, dbname_, &db_);
-
-  ASSERT_OK(s);
-
-  ASSERT_EQ("wal_to_recycle", Get("key_ignore2"));
-  ASSERT_EQ("NOT_FOUND", Get("key1"));
-
-  Close();
-}
-
-class DBWALTrackAndVerifyWALsWithParamsTest
-    : public DBWALTestBase,
-      public ::testing::WithParamInterface<WALRecoveryMode> {
- public:
-  DBWALTrackAndVerifyWALsWithParamsTest()
-      : DBWALTestBase("/db_wal_track_and_verify_wals_with_params_test") {}
-};
-
-INSTANTIATE_TEST_CASE_P(
-    DBWALTrackAndVerifyWALsWithParamsTest,
-    DBWALTrackAndVerifyWALsWithParamsTest,
-    ::testing::Values(WALRecoveryMode::kTolerateCorruptedTailRecords,
-                      WALRecoveryMode::kAbsoluteConsistency,
-                      WALRecoveryMode::kPointInTimeRecovery,
-                      WALRecoveryMode::kSkipAnyCorruptedRecords));
-
-TEST_P(DBWALTrackAndVerifyWALsWithParamsTest, Basic) {
-  Options options = CurrentOptions();
-  options.avoid_flush_during_shutdown = true;
-  options.track_and_verify_wals = true;
-  options.wal_recovery_mode = GetParam();
-
-  // Stop background flush to avoid deleting any WAL
-  options.env->SetBackgroundThreads(1, Env::HIGH);
-  test::SleepingBackgroundTask sleeping_task;
-  options.env->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
-                        &sleeping_task, Env::Priority::HIGH);
-
-  for (int i = 0; i < 5; i++) {
-    DestroyAndReopen(options);
-
-    ASSERT_OK(Put("key1", "old_value"));
-    SequenceNumber last_seqno_recorded_in_fist_wal =
-        dbfull()->GetLatestSequenceNumber();
-
-    ASSERT_OK(dbfull()->TEST_SwitchWAL());
-    ASSERT_OK(Put("key1", "new_value"));
-
-    VectorWalPtr log_files;
-    ASSERT_OK(db_->GetSortedWalFiles(log_files));
-    ASSERT_EQ(log_files.size(), 2);
-    uint64_t first_log_number = log_files.front()->LogNumber();
-    std::string first_log_name = LogFileName(dbname_, first_log_number);
-    std::string second_log_name =
-        LogFileName(dbname_, log_files.back()->LogNumber());
-
-    if (i == 0) {
-      // Delete the obsolete WAL and verify it will not be seen as a WAL hole
-      sleeping_task.WakeUp();
-      sleeping_task.WaitUntilDone();
-      ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
-      // Stop background flush to avoid deleting any WAL
-      sleeping_task.Reset();
-      options.env->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
-                            &sleeping_task, Env::Priority::HIGH);
-      Close();
-    } else if (i == 1) {
-      // Create a WAL hole on WAL number by deleting the first WAL and verify
-      // the hole will be detected
-      Close();
-      ASSERT_OK(options.env->DeleteFile(first_log_name));
-    } else if (i == 2) {
-      // Create a WAL hole on sequence number by truncating the first WAL and
-      // verify the hole will be detected
-      Close();
-      ASSERT_OK(
-          test::TruncateFile(options.env, first_log_name, 0 /* new_length */));
-    } else if (i == 3) {
-      // Create a WAL hole on size difference by truncating the first WAL and
-      // mocking a correct sequence number to force triggering corruption based
-      // on size instead of sequence number and verify the hole will be detected
-      Close();
-      ASSERT_OK(
-          test::TruncateFile(options.env, first_log_name, 0 /* new_length */));
-      SyncPoint::GetInstance()->SetCallBack(
-          "DBImpl::UpdatePredecessorWALInfo", [&](void* arg) {
-            std::pair<uint64_t, SequenceNumber*>* pair =
-                static_cast<std::pair<uint64_t, SequenceNumber*>*>(arg);
-            if (pair->first == first_log_number) {
-              *(pair->second) = last_seqno_recorded_in_fist_wal;
-            }
-          });
-      SyncPoint::GetInstance()->EnableProcessing();
-    } else if (i == 4) {
-      // Delete all wals and verify opening a DB with no WAL will be detected
-      Close();
-      ASSERT_OK(options.env->DeleteFile(first_log_name));
-      ASSERT_OK(options.env->DeleteFile(second_log_name));
-    }
-
-    Status s = DB::Open(options, dbname_, &db_);
-
-    if (i == 0) {
-      ASSERT_OK(s);
-      ASSERT_EQ("new_value", Get("key1"));
-      continue;
-    } else if (i == 3) {
-      SyncPoint::GetInstance()->ClearAllCallBacks();
-      SyncPoint::GetInstance()->DisableProcessing();
-    } else if (i == 4) {
-      ASSERT_TRUE(s.IsCorruption());
-      ASSERT_TRUE(
-          s.ToString().find("Opening an existing DB with no WAL files") !=
-          std::string::npos);
-      Close();
-      continue;
-    }
-
-    if (options.wal_recovery_mode == WALRecoveryMode::kPointInTimeRecovery) {
-      ASSERT_OK(s);
-      ASSERT_EQ("NOT_FOUND", Get("key1"));
-    } else if (options.wal_recovery_mode ==
-                   WALRecoveryMode::kAbsoluteConsistency ||
-               options.wal_recovery_mode ==
-                   WALRecoveryMode::kTolerateCorruptedTailRecords) {
-      ASSERT_TRUE(s.IsCorruption());
-      std::string msg;
-      if (i == 1) {
-        msg = "Missing WAL";
-      } else if (i == 2) {
-        msg = "Mismatched last sequence number recorded in the WAL";
-      } else if (i == 3) {
-        msg = "Mismatched size of the WAL";
-      }
-      ASSERT_TRUE(s.ToString().find(msg) != std::string::npos);
-    } else {
-      ASSERT_OK(s);
-      ASSERT_EQ("new_value", Get("key1"));
-    }
-
-    Close();
-  }
-}
-
-class DBWALTestWithParams
-    : public DBWALTestBase,
-      public ::testing::WithParamInterface<
-          std::tuple<bool, int, int, CompressionType, bool>> {
+class DBWALTestWithParams : public DBWALTestBase,
+                            public ::testing::WithParamInterface<
+                                std::tuple<bool, int, int, CompressionType>> {
 public:
  DBWALTestWithParams() : DBWALTestBase("/db_wal_test_with_params") {}
 };
@@ -2038,8 +1853,7 @@ INSTANTIATE_TEST_CASE_P(
                                            RecoveryTestHelper::kWALFilesCount,
                                        1),
                       ::testing::Values(CompressionType::kNoCompression,
-                                         CompressionType::kZSTD),
-                       ::testing::Bool()));
+                                         CompressionType::kZSTD)));

 class DBWALTestWithParamsVaryingRecoveryMode
    : public DBWALTestBase,
@@ -2077,7 +1891,6 @@ TEST_P(DBWALTestWithParams, kTolerateCorruptedTailRecords) {

  // Fill data for testing
  Options options = CurrentOptions();
-  options.track_and_verify_wals = std::get<4>(GetParam());
  const size_t row_count = RecoveryTestHelper::FillData(this, &options);
  // test checksum failure or parsing
  RecoveryTestHelper::CorruptWAL(this, options, corrupt_offset * .3,
@@ -2101,7 +1914,6 @@ TEST_P(DBWALTestWithParams, kTolerateCorruptedTailRecords) {
 TEST_P(DBWALTestWithParams, kAbsoluteConsistency) {
  // Verify clean slate behavior
  Options options = CurrentOptions();
-  options.track_and_verify_wals = std::get<4>(GetParam());
  const size_t row_count = RecoveryTestHelper::FillData(this, &options);
  options.create_if_missing = false;
  ASSERT_OK(TryReopen(options));
@@ -2312,6 +2124,7 @@ TEST_F(DBWALTest, FixSyncWalOnObseletedWalWithNewManifestCausingMissingWAL) {
        wal_synced = true;
      });

+
  SyncPoint::GetInstance()->EnableProcessing();

  ASSERT_OK(Flush());
@@ -2352,7 +2165,6 @@ TEST_P(DBWALTestWithParams, kPointInTimeRecovery) {

  // Fill data for testing
  Options options = CurrentOptions();
-  options.track_and_verify_wals = std::get<4>(GetParam());
  options.wal_compression = compression_type;
  const size_t row_count = RecoveryTestHelper::FillData(this, &options);

@@ -2410,7 +2222,6 @@ TEST_P(DBWALTestWithParams, kSkipAnyCorruptedRecords) {

  // Fill data for testing
  Options options = CurrentOptions();
-  options.track_and_verify_wals = std::get<4>(GetParam());
  options.wal_compression = compression_type;
  const size_t row_count = RecoveryTestHelper::FillData(this, &options);

@@ -2888,12 +2699,12 @@ TEST_F(DBWALTest, ReadOnlyRecoveryNoTruncate) {
  // create DB and close with file truncate disabled
  std::atomic_bool enable_truncate{false};

-  SyncPoint::GetInstance()->SetCallBack("PosixWritableFile::Close",
-                                        [&](void* arg) {
-                                          if (!enable_truncate) {
-                                            *(static_cast<size_t*>(arg)) = 0;
-                                          }
-                                        });
+  SyncPoint::GetInstance()->SetCallBack(
+      "PosixWritableFile::Close", [&](void* arg) {
+        if (!enable_truncate) {
+          *(static_cast<size_t*>(arg)) = 0;
+        }
+      });
  SyncPoint::GetInstance()->EnableProcessing();

  DestroyAndReopen(options);
@@ -2982,6 +2793,7 @@ TEST_F(DBWALTest, WalInManifestButNotInSortedWals) {
  Close();
 }

+
 TEST_F(DBWALTest, WalTermTest) {
  Options options = CurrentOptions();
  options.env = env_;
@@ -3257,6 +3257,7 @@ TEST_F(DBBasicTestWithTimestamp, MultiGetNoReturnTs) {
  Close();
 }

+
 INSTANTIATE_TEST_CASE_P(
    Timestamp, DBBasicTestWithTimestampCompressionSettings,
    ::testing::Combine(
@@ -780,6 +780,7 @@ TEST_P(DBWriteBufferManagerTest, MixedSlowDownOptionsMultipleDB) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

+
 // Tests a `WriteBufferManager` constructed with `allow_stall == false` does not
 // thrash memtable switching when full and a CF receives multiple writes.
 // Instead, we expect to switch a CF's memtable for flush only when that CF does
@@ -498,8 +498,6 @@ class InternalKey {
  // Intended only to be used together with ConvertFromUserKey().
  std::string* rep() { return &rep_; }

-  const std::string* const_rep() const { return &rep_; }
-
  // Assuming that *rep() contains a user key, this method makes internal key
  // out of it in-place. This saves a memcpy compared to Set()/SetFrom().
  void ConvertFromUserKey(SequenceNumber s, ValueType t) {
@@ -1181,64 +1179,4 @@ struct ParsedInternalKeyComparator {
  const InternalKeyComparator* cmp;
 };

-class PredecessorWALInfo {
- public:
-  PredecessorWALInfo()
-      : log_number_(0),
-        size_bytes_(0),
-        last_seqno_recorded_(0),
-        initialized_(false) {}
-
-  explicit PredecessorWALInfo(uint64_t log_number, uint64_t size_bytes,
-                              SequenceNumber last_seqno_recorded)
-      : log_number_(log_number),
-        size_bytes_(size_bytes),
-        last_seqno_recorded_(last_seqno_recorded),
-        initialized_(true) {}
-
-  uint64_t GetLogNumber() const {
-    assert(initialized_);
-    return log_number_;
-  }
-
-  uint64_t GetSizeBytes() const {
-    assert(initialized_);
-    return size_bytes_;
-  }
-
-  SequenceNumber GetLastSeqnoRecorded() const {
-    assert(initialized_);
-    return last_seqno_recorded_;
-  }
-
-  bool IsInitialized() const { return initialized_; }
-
-  inline void EncodeTo(std::string* dst) const {
-    assert(dst != nullptr);
-    assert(initialized_);
-    PutFixed64(dst, log_number_);
-    PutFixed64(dst, size_bytes_);
-    PutFixed64(dst, last_seqno_recorded_);
-  }
-
-  inline Status DecodeFrom(Slice* src) {
-    if (!GetFixed64(src, &log_number_)) {
-      return Status::Corruption("Error decoding log number");
-    }
-    if (!GetFixed64(src, &size_bytes_)) {
-      return Status::Corruption("Error decoding size bytes");
-    }
-    if (!GetFixed64(src, &last_seqno_recorded_)) {
-      return Status::Corruption("Error decoding last seqno recorded");
-    }
-    initialized_ = true;
-    return Status::OK();
-  }
-
- private:
-  uint64_t log_number_;
-  uint64_t size_bytes_;
-  SequenceNumber last_seqno_recorded_;
-  bool initialized_;
-};
 }  // namespace ROCKSDB_NAMESPACE
@@ -174,15 +174,15 @@ TEST_F(DeleteFileTest, AddKeysAndQueryLevels) {
  ASSERT_EQ(level1keycount, 50000);
  ASSERT_EQ(level2keycount, 50000);

-  Status status = db_->DEPRECATED_DeleteFile("0.sst");
+  Status status = db_->DeleteFile("0.sst");
  ASSERT_TRUE(status.IsInvalidArgument());

  // intermediate level files cannot be deleted.
-  status = db_->DEPRECATED_DeleteFile(level1file);
+  status = db_->DeleteFile(level1file);
  ASSERT_TRUE(status.IsInvalidArgument());

  // Lowest level file deletion should succeed.
-  status = db_->DEPRECATED_DeleteFile(level2file);
+  status = db_->DeleteFile(level2file);
  ASSERT_OK(status);
 }

@@ -519,7 +519,7 @@ TEST_F(DeleteFileTest, DeleteFileWithIterator) {
    level2file = metadata[0].name;
  }

-  Status status = db_->DEPRECATED_DeleteFile(level2file);
+  Status status = db_->DeleteFile(level2file);
  fprintf(stdout, "Deletion status %s: %s\n", level2file.c_str(),
          status.ToString().c_str());
  ASSERT_OK(status);
@@ -551,7 +551,7 @@ TEST_F(DeleteFileTest, DeleteLogFiles) {
  ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName()));
  fprintf(stdout, "Deleting alive log file %s\n",
          alive_log->PathName().c_str());
-  ASSERT_NOK(db_->DEPRECATED_DeleteFile(alive_log->PathName()));
+  ASSERT_NOK(db_->DeleteFile(alive_log->PathName()));
  ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName()));
  logfiles.clear();

@@ -569,7 +569,7 @@ TEST_F(DeleteFileTest, DeleteLogFiles) {
  ASSERT_OK(env_->FileExists(wal_dir_ + "/" + archived_log->PathName()));
  fprintf(stdout, "Deleting archived log file %s\n",
          archived_log->PathName().c_str());
-  ASSERT_OK(db_->DEPRECATED_DeleteFile(archived_log->PathName()));
+  ASSERT_OK(db_->DeleteFile(archived_log->PathName()));
  ASSERT_TRUE(
      env_->FileExists(wal_dir_ + "/" + archived_log->PathName()).IsNotFound());
 }
@@ -605,8 +605,8 @@ TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) {
  auto new_file = metadata[0].smallest_seqno > metadata[1].smallest_seqno
                      ? metadata[0].name
                      : metadata[1].name;
-  ASSERT_TRUE(db_->DEPRECATED_DeleteFile(new_file).IsInvalidArgument());
-  ASSERT_OK(db_->DEPRECATED_DeleteFile(old_file));
+  ASSERT_TRUE(db_->DeleteFile(new_file).IsInvalidArgument());
+  ASSERT_OK(db_->DeleteFile(old_file));

  {
    std::unique_ptr<Iterator> itr(db_->NewIterator(ReadOptions(), handles_[1]));
@@ -227,7 +227,7 @@ std::map<std::tuple<BackgroundErrorReason, bool>, Status::Severity>
         Status::Severity::kFatalError},
 };

-void ErrorHandler::CancelErrorRecoveryForShutDown() {
+void ErrorHandler::CancelErrorRecovery() {
  db_mutex_->AssertHeld();

  // We'll release the lock before calling sfm, so make sure no new
@@ -573,8 +573,6 @@ Status ErrorHandler::ClearBGError() {

  // Signal that recovery succeeded
  if (recovery_error_.ok()) {
-    // If this assertion fails, it means likely bg error is not set after a
-    // file is quarantined during MANIFEST write.
    assert(files_to_quarantine_.empty());
    Status old_bg_error = bg_error_;
    // old_bg_error is only for notifying listeners, so may not be checked
@@ -587,15 +585,8 @@ Status ErrorHandler::ClearBGError() {
    recovery_error_.PermitUncheckedError();
    recovery_in_prog_ = false;
    soft_error_no_bg_work_ = false;
-    if (!db_->shutdown_initiated_) {
-      // NotifyOnErrorRecoveryEnd() may release and re-acquire db_mutex_.
-      // Prevent DB from being closed while we notify listeners. DB close will
-      // wait until allow_db_shutdown_ = true, see ReadyForShutdown().
-      allow_db_shutdown_ = false;
-      EventHelpers::NotifyOnErrorRecoveryEnd(
-          db_options_.listeners, old_bg_error, bg_error_, db_mutex_);
-      allow_db_shutdown_ = true;
-    }
+    EventHelpers::NotifyOnErrorRecoveryEnd(db_options_.listeners, old_bg_error,
+                                           bg_error_, db_mutex_);
  }
  return recovery_error_;
 }
@@ -44,7 +44,6 @@ class ErrorHandler {
        auto_recovery_(false),
        recovery_in_prog_(false),
        soft_error_no_bg_work_(false),
-        allow_db_shutdown_(true),
        is_db_stopped_(false),
        bg_error_stats_(db_options.statistics) {
    // Clear the checked flag for uninitialized errors
@@ -64,12 +63,6 @@ class ErrorHandler {

  Status GetRecoveryError() const { return recovery_error_; }

-  // REQUIREs: db mutex held
-  //
-  // Returns non-OK status if encountered error during recovery.
-  // Returns OK if bg error is successfully cleared. May releases and
-  // re-acquire db mutex to notify listeners. However, DB close (if initiated)
-  // will be blocked until db mutex is released after return.
  Status ClearBGError();

  bool IsDBStopped() { return is_db_stopped_.load(std::memory_order_acquire); }
@@ -86,14 +79,8 @@ class ErrorHandler {

  bool IsRecoveryInProgress() { return recovery_in_prog_; }

-  // REQUIRES: db mutex held
-  bool ReadyForShutdown() {
-    db_mutex_->AssertHeld();
-    return !recovery_in_prog_ && allow_db_shutdown_;
-  }
-
  Status RecoverFromBGError(bool is_manual = false);
-  void CancelErrorRecoveryForShutDown();
+  void CancelErrorRecovery();

  void EndAutoRecovery();

@@ -134,8 +121,6 @@ class ErrorHandler {
  // A flag to indicate that for the soft error, we should not allow any
  // background work except the work is from recovery.
  bool soft_error_no_bg_work_;
-  // Used in ClearBGError() to prevent DB from being closed.
-  bool allow_db_shutdown_;

  // Used to store the context for recover, such as flush reason.
  DBRecoverContext recover_context_;
@@ -81,8 +81,9 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished(
    JSONWriter jwriter;
    AppendCurrentTime(&jwriter);
    jwriter << "cf_name" << cf_name << "job" << job_id << "event"
-            << "table_file_creation" << "file_number" << fd.GetNumber()
-            << "file_size" << fd.GetFileSize() << "file_checksum"
+            << "table_file_creation"
+            << "file_number" << fd.GetNumber() << "file_size"
+            << fd.GetFileSize() << "file_checksum"
            << Slice(file_checksum).ToString(true) << "file_checksum_func_name"
            << file_checksum_func_name << "smallest_seqno" << fd.smallest_seqno
            << "largest_seqno" << fd.largest_seqno;
@@ -198,7 +199,8 @@ void EventHelpers::LogAndNotifyTableFileDeletion(
  JSONWriter jwriter;
  AppendCurrentTime(&jwriter);

-  jwriter << "job" << job_id << "event" << "table_file_deletion"
+  jwriter << "job" << job_id << "event"
+          << "table_file_deletion"
          << "file_number" << file_number;
  if (!status.ok()) {
    jwriter << "status" << status.ToString();
@@ -278,11 +280,11 @@ void EventHelpers::LogAndNotifyBlobFileCreationFinished(
    JSONWriter jwriter;
    AppendCurrentTime(&jwriter);
    jwriter << "cf_name" << cf_name << "job" << job_id << "event"
-            << "blob_file_creation" << "file_number" << file_number
-            << "total_blob_count" << total_blob_count << "total_blob_bytes"
-            << total_blob_bytes << "file_checksum" << file_checksum
-            << "file_checksum_func_name" << file_checksum_func_name << "status"
-            << s.ToString();
+            << "blob_file_creation"
+            << "file_number" << file_number << "total_blob_count"
+            << total_blob_count << "total_blob_bytes" << total_blob_bytes
+            << "file_checksum" << file_checksum << "file_checksum_func_name"
+            << file_checksum_func_name << "status" << s.ToString();

    jwriter.EndObject();
    event_logger->Log(jwriter);
@@ -309,7 +311,8 @@ void EventHelpers::LogAndNotifyBlobFileDeletion(
    JSONWriter jwriter;
    AppendCurrentTime(&jwriter);

-    jwriter << "job" << job_id << "event" << "blob_file_deletion"
+    jwriter << "job" << job_id << "event"
+            << "blob_file_deletion"
            << "file_number" << file_number;
    if (!status.ok()) {
      jwriter << "status" << status.ToString();
@@ -37,6 +37,7 @@ Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) {
  return db->PromoteL0(column_family, target_level);
 }

+
 Status SuggestCompactRange(DB* db, const Slice* begin, const Slice* end) {
  return SuggestCompactRange(db, db->DefaultColumnFamily(), begin, end);
 }
@@ -11,7 +11,6 @@
 #include "port/stack_trace.h"
 #include "rocksdb/advanced_options.h"
 #include "rocksdb/options.h"
-#include "rocksdb/perf_context.h"
 #include "rocksdb/sst_file_writer.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
@@ -261,54 +260,14 @@ TEST_F(ExternalSSTFileBasicTest, Basic) {
  ASSERT_NOK(s) << s.ToString();

  DestroyAndReopen(options);
-
-  SyncPoint::GetInstance()->LoadDependency({
-      {"DBImpl::IngestExternalFile:AfterIncIngestFileCounter",
-       "ExternalSSTFileBasicTest.LiveWriteStart"},
-      {"ExternalSSTFileBasicTest.LiveWriteStart",
-       "DBImpl::IngestExternalFiles:InstallSVForFirstCF:0"},
-  });
-  SyncPoint::GetInstance()->EnableProcessing();
-  PerfContext* write_thread_perf_context;
-  std::thread write_thread([&] {
-    TEST_SYNC_POINT("ExternalSSTFileBasicTest.LiveWriteStart");
-    SetPerfLevel(kEnableWait);
-    write_thread_perf_context = get_perf_context();
-    write_thread_perf_context->Reset();
-    ASSERT_OK(db_->Put(WriteOptions(), "bar", "v2"));
-    ASSERT_GT(write_thread_perf_context->write_thread_wait_nanos, 0);
-    // Test sync points were used to make sure this live write enter write
-    // thread after the file ingestion entered write thread. So by the time this
-    // live write finishes, the latest seqno is 1 means file ingestion used
-    // seqno 0.
-    ASSERT_EQ(db_->GetLatestSequenceNumber(), 1U);
-  });
-
  // Add file using file path
-  SetPerfLevel(kEnableTimeExceptForMutex);
-  PerfContext* perf_ctx = get_perf_context();
-  perf_ctx->Reset();
  s = DeprecatedAddFile({file1});
-  ASSERT_GT(perf_context.file_ingestion_nanos, 0);
-  ASSERT_GT(perf_context.file_ingestion_blocking_live_writes_nanos, 0);
  ASSERT_OK(s) << s.ToString();
+  ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
  for (int k = 0; k < 100; k++) {
    ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
  }

-  write_thread.join();
-  SyncPoint::GetInstance()->DisableProcessing();
-
-  // Re-ingest the file just to check the perf context not enabled at and below
-  // kEnableWait.
-  SetPerfLevel(kEnableWait);
-  perf_ctx->Reset();
-  IngestExternalFileOptions opts;
-  opts.allow_global_seqno = true;
-  opts.allow_blocking_flush = true;
-  ASSERT_OK(db_->IngestExternalFile({file1}, opts));
-  ASSERT_EQ(perf_context.file_ingestion_nanos, 0);
-  ASSERT_EQ(perf_context.file_ingestion_blocking_live_writes_nanos, 0);
  DestroyAndRecreateExternalSSTFilesDir();
 }

@@ -2919,6 +2878,7 @@ INSTANTIATE_TEST_CASE_P(ExternalSSTFileBasicTest, ExternalSSTFileBasicTest,
                                        std::make_tuple(false, true),
                                        std::make_tuple(false, false)));

+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -115,7 +115,7 @@ Status ExternalSstFileIngestionJob::Prepare(
    f.copy_file = false;
    const std::string path_outside_db = f.external_file_path;
    const std::string path_inside_db = TableFileName(
-        cfd_->ioptions().cf_paths, f.fd.GetNumber(), f.fd.GetPathId());
+        cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId());
    if (ingestion_options_.move_files || ingestion_options_.link_files) {
      status =
          fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr);
@@ -463,7 +463,7 @@ Status ExternalSstFileIngestionJob::AssignLevelsForOneBatch(
      status = CheckLevelForIngestedBehindFile(file);
    } else {
      status = AssignLevelAndSeqnoForIngestedFile(
-          super_version, force_global_seqno, cfd_->ioptions().compaction_style,
+          super_version, force_global_seqno, cfd_->ioptions()->compaction_style,
          *last_seqno, file, &assigned_seqno, prev_batch_uppermost_level);
    }

@@ -584,13 +584,13 @@ void ExternalSstFileIngestionJob::CreateEquivalentFileIngestingCompactions() {
    int output_level = pair.first;
    const CompactionInputFiles& input = pair.second;

-    const auto& mutable_cf_options = cfd_->GetLatestMutableCFOptions();
+    const auto& mutable_cf_options = *(cfd_->GetLatestMutableCFOptions());
    file_ingesting_compactions_.push_back(new Compaction(
-        cfd_->current()->storage_info(), cfd_->ioptions(), mutable_cf_options,
+        cfd_->current()->storage_info(), *cfd_->ioptions(), mutable_cf_options,
        mutable_db_options_, {input}, output_level,
        /* output file size limit not applicable */
        MaxFileSizeForLevel(mutable_cf_options, output_level,
-                            cfd_->ioptions().compaction_style),
+                            cfd_->ioptions()->compaction_style),
        LLONG_MAX /* max compaction bytes, not applicable */,
        0 /* output path ID, not applicable */, mutable_cf_options.compression,
        mutable_cf_options.compression_opts,
@@ -631,7 +631,8 @@ void ExternalSstFileIngestionJob::UpdateStats() {
  uint64_t total_time = clock_->NowMicros() - job_start_time_;

  EventLoggerStream stream = event_logger_->Log();
-  stream << "event" << "ingest_finished";
+  stream << "event"
+         << "ingest_finished";
  stream << "files_ingested";
  stream.StartArray();

@@ -749,7 +750,7 @@ Status ExternalSstFileIngestionJob::ResetTableReader(
  status = sv->mutable_cf_options.table_factory->NewTableReader(
      ro,
      TableReaderOptions(
-          cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor,
+          *cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor,
          env_options_, cfd_->internal_comparator(),
          sv->mutable_cf_options.block_protection_bytes_per_key,
          /*skip_filters*/ false, /*immortal*/ false,
@@ -843,7 +844,7 @@ Status ExternalSstFileIngestionJob::SanityCheckTableProperties(
  bool mark_sst_file_has_no_udt = false;
  Status s = ValidateUserDefinedTimestampsOptions(
      cfd_->user_comparator(), props->comparator_name,
-      cfd_->ioptions().persist_user_defined_timestamps,
+      cfd_->ioptions()->persist_user_defined_timestamps,
      file_to_ingest->user_defined_timestamps_persisted,
      &mark_sst_file_has_no_udt);
  if (s.ok() && mark_sst_file_has_no_udt) {
@@ -329,7 +329,7 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta,
      TEST_SYNC_POINT("FlushJob::InstallResults");
      // Replace immutable memtable with the generated Table
      s = cfd_->imm()->TryInstallMemtableFlushResults(
-              cfd_, mems_, prep_tracker, versions_, db_mutex_,
+              cfd_, mutable_cf_options_, mems_, prep_tracker, versions_, db_mutex_,
              meta_.fd.GetNumber(), &job_context_->memtables_to_free, db_directory_,
              log_buffer_, &committed_flush_jobs_info_,
              !(mempurge_s.ok()) /* write_edit : true if no mempurge happened (or if aborted),
@@ -345,7 +345,8 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta,

  // When measure_io_stats_ is true, the default 512 bytes is not enough.
  auto stream = event_logger_->LogToBuffer(log_buffer_, 1024);
-  stream << "job" << job_context_->job_id << "event" << "flush_finished";
+  stream << "job" << job_context_->job_id << "event"
+         << "flush_finished";
  stream << "output_compression"
         << CompressionTypeToString(output_compression_);
  stream << "lsm_state";
@@ -448,7 +449,7 @@ Status FlushJob::MemPurge() {
      NewMergingIterator(&(cfd_->internal_comparator()), memtables.data(),
                         static_cast<int>(memtables.size()), &arena));

-  const auto& ioptions = cfd_->ioptions();
+  auto* ioptions = cfd_->ioptions();

  // Place iterator at the First (meaning most recent) key node.
  iter->SeekToFirst();
@@ -469,8 +470,8 @@ Status FlushJob::MemPurge() {
    // MaxSize is the size of a memtable.
    size_t maxSize = mutable_cf_options_.write_buffer_size;
    std::unique_ptr<CompactionFilter> compaction_filter;
-    if (ioptions.compaction_filter_factory != nullptr &&
-        ioptions.compaction_filter_factory->ShouldFilterTableFileCreation(
+    if (ioptions->compaction_filter_factory != nullptr &&
+        ioptions->compaction_filter_factory->ShouldFilterTableFileCreation(
            TableFileCreationReason::kFlush)) {
      CompactionFilter::Context ctx;
      ctx.is_full_compaction = false;
@@ -478,7 +479,7 @@ Status FlushJob::MemPurge() {
      ctx.column_family_id = cfd_->GetID();
      ctx.reason = TableFileCreationReason::kFlush;
      compaction_filter =
-          ioptions.compaction_filter_factory->CreateCompactionFilter(ctx);
+          ioptions->compaction_filter_factory->CreateCompactionFilter(ctx);
      if (compaction_filter != nullptr &&
          !compaction_filter->IgnoreSnapshots()) {
        s = Status::NotSupported(
@@ -488,7 +489,7 @@ Status FlushJob::MemPurge() {
      }
    }

-    new_mem = new MemTable(cfd_->internal_comparator(), cfd_->ioptions(),
+    new_mem = new MemTable((cfd_->internal_comparator()), *(cfd_->ioptions()),
                           mutable_cf_options_, cfd_->write_buffer_mgr(),
                           earliest_seqno, cfd_->GetID());
    assert(new_mem != nullptr);
@@ -497,8 +498,8 @@ Status FlushJob::MemPurge() {
    assert(env);
    MergeHelper merge(
        env, (cfd_->internal_comparator()).user_comparator(),
-        (ioptions.merge_operator).get(), compaction_filter.get(),
-        ioptions.logger, true /* internal key corruption is not ok */,
+        (ioptions->merge_operator).get(), compaction_filter.get(),
+        ioptions->logger, true /* internal key corruption is not ok */,
        existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
        snapshot_checker_);
    assert(job_context_);
@@ -508,14 +509,14 @@ Status FlushJob::MemPurge() {
        iter.get(), (cfd_->internal_comparator()).user_comparator(), &merge,
        kMaxSequenceNumber, &existing_snapshots_, earliest_snapshot_,
        earliest_write_conflict_snapshot_, job_snapshot_seq, snapshot_checker_,
-        env, ShouldReportDetailedTime(env, ioptions.stats),
+        env, ShouldReportDetailedTime(env, ioptions->stats),
        true /* internal key corruption is not ok */, range_del_agg.get(),
-        nullptr, ioptions.allow_data_in_errors,
-        ioptions.enforce_single_del_contracts,
+        nullptr, ioptions->allow_data_in_errors,
+        ioptions->enforce_single_del_contracts,
        /*manual_compaction_canceled=*/kManualCompactionCanceledFalse,
        false /* must_count_input_entries */,
        /*compaction=*/nullptr, compaction_filter.get(),
-        /*shutting_down=*/nullptr, ioptions.info_log, full_history_ts_low);
+        /*shutting_down=*/nullptr, ioptions->info_log, full_history_ts_low);

    // Set earliest sequence number in the new memtable
    // to be equal to the earliest sequence number of the
@@ -865,7 +866,7 @@ Status FlushJob::WriteLevel0Table() {
  assert(ucmp);
  const size_t ts_sz = ucmp->timestamp_size();
  const bool logical_strip_timestamp =
-      ts_sz > 0 && !cfd_->ioptions().persist_user_defined_timestamps;
+      ts_sz > 0 && !cfd_->ioptions()->persist_user_defined_timestamps;

  std::vector<BlobFileAddition> blob_file_additions;

@@ -932,8 +933,9 @@ Status FlushJob::WriteLevel0Table() {
    //  hitting limit memtable_max_range_deletions, flush_reason_ is still
    //  "Write Buffer Full", should make update flush_reason_ accordingly.
    event_logger_->Log() << "job" << job_context_->job_id << "event"
-                         << "flush_started" << "num_memtables" << mems_.size()
-                         << "num_entries" << total_num_entries << "num_deletes"
+                         << "flush_started"
+                         << "num_memtables" << mems_.size() << "num_entries"
+                         << total_num_entries << "num_deletes"
                         << total_num_deletes << "total_data_size"
                         << total_data_size << "memory_usage"
                         << total_memory_usage << "num_range_deletes"
@@ -986,7 +988,7 @@ Status FlushJob::WriteLevel0Table() {
      read_options.rate_limiter_priority = io_priority;
      const WriteOptions write_options(io_priority, Env::IOActivity::kFlush);
      TableBuilderOptions tboptions(
-          cfd_->ioptions(), mutable_cf_options_, read_options, write_options,
+          *cfd_->ioptions(), mutable_cf_options_, read_options, write_options,
          cfd_->internal_comparator(), cfd_->internal_tbl_prop_coll_factories(),
          output_compression_, mutable_cf_options_.compression_opts,
          cfd_->GetID(), cfd_->GetName(), 0 /* level */,
@@ -1140,7 +1142,7 @@ std::unique_ptr<FlushJobInfo> FlushJob::GetFlushJobInfo() const {

  const uint64_t file_number = meta_.fd.GetNumber();
  info->file_path =
-      MakeTableFileName(cfd_->ioptions().cf_paths[0].path, file_number);
+      MakeTableFileName(cfd_->ioptions()->cf_paths[0].path, file_number);
  info->file_number = file_number;
  info->oldest_blob_file_number = meta_.oldest_blob_file_number;
  info->thread_id = db_options_.env->GetThreadID();
@@ -1154,7 +1156,7 @@ std::unique_ptr<FlushJobInfo> FlushJob::GetFlushJobInfo() const {
  // Update BlobFilesInfo.
  for (const auto& blob_file : edit_->GetBlobFileAdditions()) {
    BlobFileAdditionInfo blob_file_addition_info(
-        BlobFileName(cfd_->ioptions().cf_paths.front().path,
+        BlobFileName(cfd_->ioptions()->cf_paths.front().path,
                     blob_file.GetBlobFileNumber()) /*blob_file_path*/,
        blob_file.GetBlobFileNumber(), blob_file.GetTotalBlobCount(),
        blob_file.GetTotalBlobBytes());
@@ -1171,7 +1173,7 @@ void FlushJob::GetEffectiveCutoffUDTForPickedMemTables() {
  assert(ucmp);
  const size_t ts_sz = ucmp->timestamp_size();
  if (db_options_.atomic_flush || ts_sz == 0 ||
-      cfd_->ioptions().persist_user_defined_timestamps) {
+      cfd_->ioptions()->persist_user_defined_timestamps) {
    return;
  }
  // Find the newest user-defined timestamps from all the flushed memtables.
@@ -1193,11 +1195,7 @@ void FlushJob::GetEffectiveCutoffUDTForPickedMemTables() {
 }

 void FlushJob::GetPrecludeLastLevelMinSeqno() {
-  if (mutable_cf_options_.preclude_last_level_data_seconds == 0 ||
-      // FIXME: create FlushJob and build SuperVersions such that
-      // preclude_last_level_data_seconds > 0 implies
-      // seqno_to_time_mapping_ != nullptr
-      seqno_to_time_mapping_ == nullptr) {
+  if (mutable_cf_options_.preclude_last_level_data_seconds == 0) {
    return;
  }
  int64_t current_time = 0;
@@ -1238,7 +1236,8 @@ Status FlushJob::MaybeIncreaseFullHistoryTsLowToAboveCutoffUDT() {
  VersionEdit edit;
  edit.SetColumnFamily(cfd_->GetID());
  edit.SetFullHistoryTsLow(new_full_history_ts_low);
-  return versions_->LogAndApply(cfd_, ReadOptions(Env::IOActivity::kFlush),
+  return versions_->LogAndApply(cfd_, *cfd_->GetLatestMutableCFOptions(),
+                                ReadOptions(Env::IOActivity::kFlush),
                                WriteOptions(Env::IOActivity::kFlush), &edit,
                                db_mutex_, output_file_directory_);
 }
@@ -189,7 +189,7 @@ TEST_F(FlushJobTest, Empty) {
  SnapshotChecker* snapshot_checker = nullptr;  // not relavant
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      cfd->GetLatestMutableCFOptions(),
+      *cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -207,7 +207,7 @@ TEST_F(FlushJobTest, Empty) {
 TEST_F(FlushJobTest, NonEmpty) {
  JobContext job_context(0);
  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
-  auto new_mem = cfd->ConstructNewMemtable(cfd->GetLatestMutableCFOptions(),
+  auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
                                           kMaxSequenceNumber);
  new_mem->Ref();
  auto inserted_keys = mock::MakeMockFile();
@@ -275,7 +275,7 @@ TEST_F(FlushJobTest, NonEmpty) {
  SnapshotChecker* snapshot_checker = nullptr;  // not relavant
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      cfd->GetLatestMutableCFOptions(),
+      *cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -310,7 +310,7 @@ TEST_F(FlushJobTest, FlushMemTablesSingleColumnFamily) {
  std::vector<uint64_t> memtable_ids;
  std::vector<MemTable*> new_mems;
  for (size_t i = 0; i != num_mems; ++i) {
-    MemTable* mem = cfd->ConstructNewMemtable(cfd->GetLatestMutableCFOptions(),
+    MemTable* mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
                                              kMaxSequenceNumber);
    mem->SetID(i);
    mem->Ref();
@@ -339,7 +339,7 @@ TEST_F(FlushJobTest, FlushMemTablesSingleColumnFamily) {
  uint64_t flush_memtable_id = smallest_memtable_id + num_mems_to_flush - 1;
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_,
+      *cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_,
      versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
      nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
@@ -387,7 +387,7 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
    smallest_seqs.push_back(curr_seqno);
    for (size_t i = 0; i != num_memtables[k]; ++i) {
      MemTable* mem = cfd->ConstructNewMemtable(
-          cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
+          *cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
      mem->SetID(i);
      mem->Ref();

@@ -411,7 +411,7 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
  for (auto cfd : all_cfds) {
    std::vector<SequenceNumber> snapshot_seqs;
    flush_jobs.emplace_back(new FlushJob(
-        dbname_, cfd, db_options_, cfd->GetLatestMutableCFOptions(),
+        dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
        memtable_ids[k], env_options_, versions_.get(), &mutex_,
        &shutting_down_, snapshot_seqs, kMaxSequenceNumber, snapshot_checker,
        &job_context, FlushReason::kTest, nullptr, nullptr, nullptr,
@@ -444,6 +444,10 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
    const auto& mems = flush_jobs[i]->GetMemTables();
    mems_list.push_back(&mems);
  }
+  autovector<const MutableCFOptions*> mutable_cf_options_list;
+  for (auto cfd : all_cfds) {
+    mutable_cf_options_list.push_back(cfd->GetLatestMutableCFOptions());
+  }
  autovector<std::list<std::unique_ptr<FlushJobInfo>>*>
      committed_flush_jobs_info;
  for (auto& job : flush_jobs) {
@@ -451,8 +455,8 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
  }

  Status s = InstallMemtableAtomicFlushResults(
-      nullptr /* imm_lists */, all_cfds, mems_list, versions_.get(),
-      nullptr /* prep_tracker */, &mutex_, file_meta_ptrs,
+      nullptr /* imm_lists */, all_cfds, mutable_cf_options_list, mems_list,
+      versions_.get(), nullptr /* prep_tracker */, &mutex_, file_meta_ptrs,
      committed_flush_jobs_info, &job_context.memtables_to_free,
      nullptr /* db_directory */, nullptr /* log_buffer */);
  ASSERT_OK(s);
@@ -485,7 +489,7 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
 TEST_F(FlushJobTest, Snapshots) {
  JobContext job_context(0);
  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
-  auto new_mem = cfd->ConstructNewMemtable(cfd->GetLatestMutableCFOptions(),
+  auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
                                           kMaxSequenceNumber);

  std::set<SequenceNumber> snapshots_set;
@@ -535,7 +539,7 @@ TEST_F(FlushJobTest, Snapshots) {
  SnapshotChecker* snapshot_checker = nullptr;  // not relavant
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      cfd->GetLatestMutableCFOptions(),
+      *cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -563,7 +567,7 @@ TEST_F(FlushJobTest, GetRateLimiterPriorityForWrite) {
  std::vector<uint64_t> memtable_ids;
  std::vector<MemTable*> new_mems;
  for (size_t i = 0; i != num_mems; ++i) {
-    MemTable* mem = cfd->ConstructNewMemtable(cfd->GetLatestMutableCFOptions(),
+    MemTable* mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
                                              kMaxSequenceNumber);
    mem->SetID(i);
    mem->Ref();
@@ -592,7 +596,7 @@ TEST_F(FlushJobTest, GetRateLimiterPriorityForWrite) {
  uint64_t flush_memtable_id = smallest_memtable_id + num_mems_to_flush - 1;
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_,
+      *cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_,
      versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
      nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
@@ -623,7 +627,7 @@ TEST_F(FlushJobTest, GetRateLimiterPriorityForWrite) {
 TEST_F(FlushJobTest, ReplaceTimedPutWriteTimeWithPreferredSeqno) {
  JobContext job_context(0);
  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
-  auto new_mem = cfd->ConstructNewMemtable(cfd->GetLatestMutableCFOptions(),
+  auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
                                           kMaxSequenceNumber);
  new_mem->Ref();
  std::shared_ptr<SeqnoToTimeMapping> seqno_to_time_mapping =
@@ -661,7 +665,7 @@ TEST_F(FlushJobTest, ReplaceTimedPutWriteTimeWithPreferredSeqno) {
  SnapshotChecker* snapshot_checker = nullptr;  // not relevant
  FlushJob flush_job(
      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      cfd->GetLatestMutableCFOptions(),
+      *cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -744,7 +748,7 @@ TEST_P(FlushJobTimestampTest, AllKeysExpired) {

  {
    MemTable* new_mem = cfd->ConstructNewMemtable(
-        cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
+        *cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
    new_mem->Ref();
    for (int i = 0; i < 100; ++i) {
      uint64_t ts = curr_ts_.fetch_add(1);
@@ -768,7 +772,7 @@ TEST_P(FlushJobTimestampTest, AllKeysExpired) {
  PutFixed64(&full_history_ts_low, std::numeric_limits<uint64_t>::max());
  cfd->SetFullHistoryTsLow(full_history_ts_low);
  FlushJob flush_job(
-      dbname_, cfd, db_options_, cfd->GetLatestMutableCFOptions(),
+      dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -810,7 +814,7 @@ TEST_P(FlushJobTimestampTest, NoKeyExpired) {

  {
    MemTable* new_mem = cfd->ConstructNewMemtable(
-        cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
+        *cfd->GetLatestMutableCFOptions(), kMaxSequenceNumber);
    new_mem->Ref();
    for (int i = 0; i < 100; ++i) {
      uint64_t ts = curr_ts_.fetch_add(1);
@@ -830,7 +834,7 @@ TEST_P(FlushJobTimestampTest, NoKeyExpired) {
  PutFixed64(&full_history_ts_low, 0);
  cfd->SetFullHistoryTsLow(full_history_ts_low);
  FlushJob flush_job(
-      dbname_, cfd, db_options_, cfd->GetLatestMutableCFOptions(),
+      dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
      versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
@@ -242,7 +242,7 @@ ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
  if (sv_) {
    RebuildIterators(false);
  }
-  if (!CheckFSFeatureSupport(cfd_->ioptions().env->GetFileSystem().get(),
+  if (!CheckFSFeatureSupport(cfd_->ioptions()->env->GetFileSystem().get(),
                             FSSupportedOps::kAsyncIO)) {
    read_options_.async_io = false;
  }
@@ -4,12 +4,13 @@
 //  (found in the LICENSE.Apache file in the root directory).
 #pragma once

+#include "rocksdb/comparator.h"
+
 #include <queue>
 #include <string>
 #include <vector>

 #include "memory/arena.h"
-#include "rocksdb/comparator.h"
 #include "rocksdb/db.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/options.h"
@@ -4,6 +4,8 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

+#include "db/version_builder.h"
+
 #include "db/import_column_family_job.h"

 #include <algorithm>
@@ -11,7 +13,6 @@
 #include <string>
 #include <vector>

-#include "db/version_builder.h"
 #include "db/version_edit.h"
 #include "file/file_util.h"
 #include "file/random_access_file_reader.h"
@@ -108,7 +109,7 @@ Status ImportColumnFamilyJob::Prepare(uint64_t next_file_number,
    for (auto& f : files_to_import_per_cf) {
      const auto path_outside_db = f.external_file_path;
      const auto path_inside_db = TableFileName(
-          cfd_->ioptions().cf_paths, f.fd.GetNumber(), f.fd.GetPathId());
+          cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId());

      if (hardlink_files) {
        status = fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(),
@@ -186,16 +187,16 @@ Status ImportColumnFamilyJob::Run() {
  // RecoverEpochNumbers() below.
  for (size_t i = 0; s.ok() && i < files_to_import_.size(); ++i) {
    VersionBuilder dummy_version_builder(
-        cfd_->current()->version_set()->file_options(), &cfd_->ioptions(),
+        cfd_->current()->version_set()->file_options(), cfd_->ioptions(),
        cfd_->table_cache(), cfd_->current()->storage_info(),
        cfd_->current()->version_set(),
        cfd_->GetFileMetadataCacheReservationManager());
    VersionStorageInfo dummy_vstorage(
        &cfd_->internal_comparator(), cfd_->user_comparator(),
-        cfd_->NumberLevels(), cfd_->ioptions().compaction_style,
-        nullptr /* src_vstorage */, cfd_->ioptions().force_consistency_checks,
-        EpochNumberRequirement::kMightMissing, cfd_->ioptions().clock,
-        cfd_->GetLatestMutableCFOptions().bottommost_file_compaction_delay,
+        cfd_->NumberLevels(), cfd_->ioptions()->compaction_style,
+        nullptr /* src_vstorage */, cfd_->ioptions()->force_consistency_checks,
+        EpochNumberRequirement::kMightMissing, cfd_->ioptions()->clock,
+        cfd_->GetLatestMutableCFOptions()->bottommost_file_compaction_delay,
        cfd_->current()->version_set()->offpeak_time_option());
    for (size_t j = 0; s.ok() && j < files_to_import_[i].size(); ++j) {
      const auto& f = files_to_import_[i][j];
@@ -330,7 +331,7 @@ Status ImportColumnFamilyJob::GetIngestedFileInfo(
  //  creating `TableReaderOptions` when the support is there.
  status = sv->mutable_cf_options.table_factory->NewTableReader(
      TableReaderOptions(
-          cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor,
+          *cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor,
          env_options_, cfd_->internal_comparator(),
          sv->mutable_cf_options.block_protection_bytes_per_key,
          /*skip_filters*/ false, /*immortal*/ false,
@@ -4,6 +4,7 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

+
 #include <functional>

 #include "db/db_test_util.h"
@@ -1056,3 +1057,4 @@ int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
 }
+
@@ -924,7 +924,7 @@ bool InternalStats::HandleLiveBlobFileGarbageSize(uint64_t* value,
 }

 Cache* InternalStats::GetBlobCacheForStats() {
-  return cfd_->ioptions().blob_cache.get();
+  return cfd_->ioptions()->blob_cache.get();
 }

 bool InternalStats::HandleBlobCacheCapacity(uint64_t* value, DBImpl* /*db*/,
@@ -1468,9 +1468,9 @@ bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/,
  // TODO(yiwu): The property is currently available for fifo compaction
  // with allow_compaction = false. This is because we don't propagate
  // oldest_key_time on compaction.
-  if (cfd_->ioptions().compaction_style != kCompactionStyleFIFO ||
+  if (cfd_->ioptions()->compaction_style != kCompactionStyleFIFO ||
      cfd_->GetCurrentMutableCFOptions()
-          .compaction_options_fifo.allow_compaction) {
+          ->compaction_options_fifo.allow_compaction) {
    return false;
  }
  // TODO: plumb Env::IOActivity, Env::IOPriority
@@ -1496,7 +1496,7 @@ bool InternalStats::HandleEstimateOldestKeyTime(uint64_t* value, DBImpl* /*db*/,

 Cache* InternalStats::GetBlockCacheForStats() {
  // NOTE: called in startup before GetCurrentMutableCFOptions() is ready
-  auto* table_factory = cfd_->GetLatestMutableCFOptions().table_factory.get();
+  auto* table_factory = cfd_->GetLatestMutableCFOptions()->table_factory.get();
  assert(table_factory != nullptr);
  // FIXME: need to a shared_ptr if/when block_cache is going to be mutable
  return table_factory->GetOptions<Cache>(TableFactory::kBlockCacheOpts());
@@ -1752,7 +1752,7 @@ void InternalStats::DumpCFMapStats(
  assert(vstorage);

  int num_levels_to_check =
-      (cfd_->ioptions().compaction_style == kCompactionStyleLevel)
+      (cfd_->ioptions()->compaction_style == kCompactionStyleLevel)
          ? vstorage->num_levels() - 1
          : 1;

@@ -2163,7 +2163,8 @@ class BlockCachePropertyAggregator : public IntPropertyAggregator {
  virtual ~BlockCachePropertyAggregator() override = default;

  void Add(ColumnFamilyData* cfd, uint64_t value) override {
-    auto* table_factory = cfd->GetCurrentMutableCFOptions().table_factory.get();
+    auto* table_factory =
+        cfd->GetCurrentMutableCFOptions()->table_factory.get();
    assert(table_factory != nullptr);
    Cache* cache =
        table_factory->GetOptions<Cache>(TableFactory::kBlockCacheOpts());
@@ -33,6 +33,7 @@
 #include "util/string_util.h"
 #include "utilities/merge_operators.h"

+
 namespace ROCKSDB_NAMESPACE {

 class EventListenerTest : public DBTestBase {
@@ -1590,6 +1591,7 @@ TEST_F(EventListenerTest, BlobDBFileTest) {

 }  // namespace ROCKSDB_NAMESPACE

+
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -12,14 +12,12 @@

 #pragma once

-#include <cstdint>
-
 #include "rocksdb/rocksdb_namespace.h"

 namespace ROCKSDB_NAMESPACE {
 namespace log {

-enum RecordType : uint8_t {
+enum RecordType {
  // Zero is reserved for preallocated files
  kZeroType = 0,
  kFullType = 1,
@@ -38,18 +36,11 @@ enum RecordType : uint8_t {
  // Compression Type
  kSetCompressionType = 9,

-  // For all the values >= 10, the 1 bit indicates whether it's recyclable
  // User-defined timestamp sizes
  kUserDefinedTimestampSizeType = 10,
  kRecyclableUserDefinedTimestampSizeType = 11,
-
-  // For WAL verification
-  kPredecessorWALInfoType = 130,
-  kRecyclePredecessorWALInfoType = 131,
 };
-// Unknown type of value with the 8-th bit set will be ignored
-constexpr uint8_t kRecordTypeSafeIgnoreMask = 1 << 7;
-constexpr uint8_t kMaxRecordType = kRecyclePredecessorWALInfoType;
+constexpr int kMaxRecordType = kRecyclableUserDefinedTimestampSizeType;

 constexpr unsigned int kBlockSize = 32768;

@@ -24,10 +24,7 @@ Reader::Reporter::~Reporter() = default;

 Reader::Reader(std::shared_ptr<Logger> info_log,
               std::unique_ptr<SequentialFileReader>&& _file,
-               Reporter* reporter, bool checksum, uint64_t log_num,
-               bool track_and_verify_wals, bool stop_replay_for_corruption,
-               uint64_t min_wal_number_to_keep,
-               const PredecessorWALInfo& observed_predecessor_wal_info)
+               Reporter* reporter, bool checksum, uint64_t log_num)
    : info_log_(info_log),
      file_(std::move(_file)),
      reporter_(reporter),
@@ -40,17 +37,13 @@ Reader::Reader(std::shared_ptr<Logger> info_log,
      last_record_offset_(0),
      end_of_buffer_offset_(0),
      log_number_(log_num),
-      track_and_verify_wals_(track_and_verify_wals),
-      stop_replay_for_corruption_(stop_replay_for_corruption),
-      min_wal_number_to_keep_(min_wal_number_to_keep),
-      observed_predecessor_wal_info_(observed_predecessor_wal_info),
      recycled_(false),
      first_record_read_(false),
      compression_type_(kNoCompression),
      compression_type_record_read_(false),
      uncompress_(nullptr),
      hash_state_(nullptr),
-      uncompress_hash_state_(nullptr) {}
+      uncompress_hash_state_(nullptr){}

 Reader::~Reader() {
  delete[] backing_store_;
@@ -72,9 +65,6 @@ Reader::~Reader() {
 //
 // TODO krad: Evaluate if we need to move to a more strict mode where we
 // restrict the inconsistency to only the last log
-// TODO (hx235): move `wal_recovery_mode` to be a member data like other
-// information (e.g, `stop_replay_for_corruption`) to decide whether to
-// check for and surface corruption in `ReadRecord()`
 bool Reader::ReadRecord(Slice* record, std::string* scratch,
                        WALRecoveryMode wal_recovery_mode,
                        uint64_t* record_checksum) {
@@ -98,7 +88,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
  while (true) {
    uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
    size_t drop_size = 0;
-    const uint8_t record_type =
+    const unsigned int record_type =
        ReadPhysicalRecord(&fragment, &drop_size, record_checksum);
    switch (record_type) {
      case kFullType:
@@ -195,23 +185,6 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
        }
        break;
      }
-      case kPredecessorWALInfoType:
-      case kRecyclePredecessorWALInfoType: {
-        prospective_record_offset = physical_record_offset;
-        scratch->clear();
-        last_record_offset_ = prospective_record_offset;
-
-        PredecessorWALInfo recorded_predecessor_wal_info;
-        Status s = recorded_predecessor_wal_info.DecodeFrom(&fragment);
-        if (!s.ok()) {
-          ReportCorruption(fragment.size(),
-                           "could not decode PredecessorWALInfoType record");
-        } else {
-          MaybeVerifyPredecessorWALInfo(wal_recovery_mode, fragment,
-                                        recorded_predecessor_wal_info);
-        }
-        break;
-      }
      case kUserDefinedTimestampSizeType:
      case kRecyclableUserDefinedTimestampSizeType: {
        if (in_fragmented_record && !scratch->empty()) {
@@ -340,13 +313,11 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
        break;

      default: {
-        if ((record_type & kRecordTypeSafeIgnoreMask) == 0) {
-          std::string reason =
-              "unknown record type " + std::to_string(record_type);
-          ReportCorruption(
-              (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
-              reason.c_str());
-        }
+        std::string reason =
+            "unknown record type " + std::to_string(record_type);
+        ReportCorruption(
+            (fragment.size() + (in_fragmented_record ? scratch->size() : 0)),
+            reason.c_str());
        in_fragmented_record = false;
        scratch->clear();
        break;
@@ -356,57 +327,6 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
  return false;
 }

-void Reader::MaybeVerifyPredecessorWALInfo(
-    WALRecoveryMode wal_recovery_mode, Slice fragment,
-    const PredecessorWALInfo& recorded_predecessor_wal_info) {
-  if (!track_and_verify_wals_ ||
-      wal_recovery_mode == WALRecoveryMode::kSkipAnyCorruptedRecords ||
-      stop_replay_for_corruption_) {
-    return;
-  }
-  assert(recorded_predecessor_wal_info.IsInitialized());
-  uint64_t recorded_predecessor_log_number =
-      recorded_predecessor_wal_info.GetLogNumber();
-
-  // This is the first WAL recovered thus with no predecessor WAL info has been
-  // initialized
-  if (!observed_predecessor_wal_info_.IsInitialized()) {
-    if (recorded_predecessor_log_number >= min_wal_number_to_keep_) {
-      std::string reason = "Missing WAL of log number " +
-                           std::to_string(recorded_predecessor_log_number);
-      ReportCorruption(fragment.size(), reason.c_str());
-    }
-  } else {
-    if (observed_predecessor_wal_info_.GetLogNumber() !=
-        recorded_predecessor_log_number) {
-      std::string reason = "Missing WAL of log number " +
-                           std::to_string(recorded_predecessor_log_number);
-      ReportCorruption(fragment.size(), reason.c_str());
-    } else if (observed_predecessor_wal_info_.GetLastSeqnoRecorded() !=
-               recorded_predecessor_wal_info.GetLastSeqnoRecorded()) {
-      std::string reason =
-          "Mismatched last sequence number recorded in the WAL of log number " +
-          std::to_string(recorded_predecessor_log_number) + ". Recorded " +
-          std::to_string(recorded_predecessor_wal_info.GetLastSeqnoRecorded()) +
-          ". Observed " +
-          std::to_string(
-              observed_predecessor_wal_info_.GetLastSeqnoRecorded()) +
-          ". (Last sequence number equal to 0 indicates no WAL records)";
-      ReportCorruption(fragment.size(), reason.c_str());
-    } else if (observed_predecessor_wal_info_.GetSizeBytes() !=
-               recorded_predecessor_wal_info.GetSizeBytes()) {
-      std::string reason =
-          "Mismatched size of the WAL of log number " +
-          std::to_string(recorded_predecessor_log_number) + ". Recorded " +
-          std::to_string(recorded_predecessor_wal_info.GetSizeBytes()) +
-          " bytes. Observed " +
-          std::to_string(observed_predecessor_wal_info_.GetSizeBytes()) +
-          " bytes.";
-      ReportCorruption(fragment.size(), reason.c_str());
-    }
-  }
-}
-
 uint64_t Reader::LastRecordOffset() { return last_record_offset_; }

 uint64_t Reader::LastRecordEnd() {
@@ -499,7 +419,7 @@ void Reader::ReportOldLogRecord(size_t bytes) {
  }
 }

-bool Reader::ReadMore(size_t* drop_size, uint8_t* error) {
+bool Reader::ReadMore(size_t* drop_size, int* error) {
  if (!eof_ && !read_error_) {
    // Last read was a full read, so this is a trailer to skip
    buffer_.clear();
@@ -540,15 +460,15 @@ bool Reader::ReadMore(size_t* drop_size, uint8_t* error) {
  }
 }

-uint8_t Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
-                                   uint64_t* fragment_checksum) {
+unsigned int Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
+                                        uint64_t* fragment_checksum) {
  while (true) {
    // We need at least the minimum header size
    if (buffer_.size() < static_cast<size_t>(kHeaderSize)) {
      // the default value of r is meaningless because ReadMore will overwrite
      // it if it returns false; in case it returns true, the return value will
      // not be used anyway
-      uint8_t r = kEof;
+      int r = kEof;
      if (!ReadMore(drop_size, &r)) {
        return r;
      }
@@ -559,13 +479,12 @@ uint8_t Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
    const char* header = buffer_.data();
    const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
    const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
-    const uint8_t type = static_cast<uint8_t>(header[6]);
+    const unsigned int type = header[6];
    const uint32_t length = a | (b << 8);
    int header_size = kHeaderSize;
    const bool is_recyclable_type =
        ((type >= kRecyclableFullType && type <= kRecyclableLastType) ||
-         type == kRecyclableUserDefinedTimestampSizeType ||
-         type == kRecyclePredecessorWALInfoType);
+         type == kRecyclableUserDefinedTimestampSizeType);
    if (is_recyclable_type) {
      header_size = kRecyclableHeaderSize;
      if (first_record_read_ && !recycled_) {
@@ -575,7 +494,7 @@ uint8_t Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
      recycled_ = true;
      // We need enough for the larger header
      if (buffer_.size() < static_cast<size_t>(kRecyclableHeaderSize)) {
-        uint8_t r = kEof;
+        int r = kEof;
        if (!ReadMore(drop_size, &r)) {
          return r;
        }
@@ -630,8 +549,6 @@ uint8_t Reader::ReadPhysicalRecord(Slice* result, size_t* drop_size,
    buffer_.remove_prefix(header_size + length);

    if (!uncompress_ || type == kSetCompressionType ||
-        type == kPredecessorWALInfoType ||
-        type == kRecyclePredecessorWALInfoType ||
        type == kUserDefinedTimestampSizeType ||
        type == kRecyclableUserDefinedTimestampSizeType) {
      *result = Slice(header + header_size, length);
@@ -721,9 +638,7 @@ Status Reader::UpdateRecordedTimestampSize(
 }

 bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
-                                        WALRecoveryMode wal_recovery_mode
-
-                                        ,
+                                        WALRecoveryMode /*unused*/,
                                        uint64_t* /* checksum */) {
  assert(record != nullptr);
  assert(scratch != nullptr);
@@ -736,7 +651,7 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
  uint64_t prospective_record_offset = 0;
  uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
  size_t drop_size = 0;
-  uint8_t fragment_type_or_err = 0;  // Initialize to make compiler happy
+  unsigned int fragment_type_or_err = 0;  // Initialize to make compiler happy
  Slice fragment;
  while (TryReadFragment(&fragment, &drop_size, &fragment_type_or_err)) {
    switch (fragment_type_or_err) {
@@ -813,24 +728,7 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
        }
        break;
      }
-      case kPredecessorWALInfoType:
-      case kRecyclePredecessorWALInfoType: {
-        fragments_.clear();
-        prospective_record_offset = physical_record_offset;
-        last_record_offset_ = prospective_record_offset;
-        in_fragmented_record_ = false;

-        PredecessorWALInfo recorded_predecessor_wal_info;
-        Status s = recorded_predecessor_wal_info.DecodeFrom(&fragment);
-        if (!s.ok()) {
-          ReportCorruption(fragment.size(),
-                           "could not decode PredecessorWALInfoType record");
-        } else {
-          MaybeVerifyPredecessorWALInfo(wal_recovery_mode, fragment,
-                                        recorded_predecessor_wal_info);
-        }
-        break;
-      }
      case kUserDefinedTimestampSizeType:
      case kRecyclableUserDefinedTimestampSizeType: {
        if (in_fragmented_record_ && !scratch->empty()) {
@@ -883,13 +781,11 @@ bool FragmentBufferedReader::ReadRecord(Slice* record, std::string* scratch,
        break;

      default: {
-        if ((fragment_type_or_err & kRecordTypeSafeIgnoreMask) == 0) {
-          std::string reason =
-              "unknown record type " + std::to_string(fragment_type_or_err);
-          ReportCorruption(
-              fragment.size() + (in_fragmented_record_ ? fragments_.size() : 0),
-              reason.c_str());
-        }
+        std::string reason =
+            "unknown record type " + std::to_string(fragment_type_or_err);
+        ReportCorruption(
+            fragment.size() + (in_fragmented_record_ ? fragments_.size() : 0),
+            reason.c_str());
        in_fragmented_record_ = false;
        fragments_.clear();
        break;
@@ -907,7 +803,7 @@ void FragmentBufferedReader::UnmarkEOF() {
  UnmarkEOFInternal();
 }

-bool FragmentBufferedReader::TryReadMore(size_t* drop_size, uint8_t* error) {
+bool FragmentBufferedReader::TryReadMore(size_t* drop_size, int* error) {
  if (!eof_ && !read_error_) {
    // Last read was a full read, so this is a trailer to skip
    buffer_.clear();
@@ -948,15 +844,15 @@ bool FragmentBufferedReader::TryReadMore(size_t* drop_size, uint8_t* error) {
 }

 // return true if the caller should process the fragment_type_or_err.
-bool FragmentBufferedReader::TryReadFragment(Slice* fragment, size_t* drop_size,
-                                             uint8_t* fragment_type_or_err) {
+bool FragmentBufferedReader::TryReadFragment(
+    Slice* fragment, size_t* drop_size, unsigned int* fragment_type_or_err) {
  assert(fragment != nullptr);
  assert(drop_size != nullptr);
  assert(fragment_type_or_err != nullptr);

  while (buffer_.size() < static_cast<size_t>(kHeaderSize)) {
    size_t old_size = buffer_.size();
-    uint8_t error = kEof;
+    int error = kEof;
    if (!TryReadMore(drop_size, &error)) {
      *fragment_type_or_err = error;
      return false;
@@ -967,12 +863,11 @@ bool FragmentBufferedReader::TryReadFragment(Slice* fragment, size_t* drop_size,
  const char* header = buffer_.data();
  const uint32_t a = static_cast<uint32_t>(header[4]) & 0xff;
  const uint32_t b = static_cast<uint32_t>(header[5]) & 0xff;
-  const uint8_t type = static_cast<uint8_t>(header[6]);
+  const unsigned int type = header[6];
  const uint32_t length = a | (b << 8);
  int header_size = kHeaderSize;
  if ((type >= kRecyclableFullType && type <= kRecyclableLastType) ||
-      type == kRecyclableUserDefinedTimestampSizeType ||
-      type == kRecyclePredecessorWALInfoType) {
+      type == kRecyclableUserDefinedTimestampSizeType) {
    if (first_record_read_ && !recycled_) {
      // A recycled log should have started with a recycled record
      *fragment_type_or_err = kBadRecord;
@@ -982,7 +877,7 @@ bool FragmentBufferedReader::TryReadFragment(Slice* fragment, size_t* drop_size,
    header_size = kRecyclableHeaderSize;
    while (buffer_.size() < static_cast<size_t>(kRecyclableHeaderSize)) {
      size_t old_size = buffer_.size();
-      uint8_t error = kEof;
+      int error = kEof;
      if (!TryReadMore(drop_size, &error)) {
        *fragment_type_or_err = error;
        return false;
@@ -999,7 +894,7 @@ bool FragmentBufferedReader::TryReadFragment(Slice* fragment, size_t* drop_size,

  while (header_size + length > buffer_.size()) {
    size_t old_size = buffer_.size();
-    uint8_t error = kEof;
+    int error = kEof;
    if (!TryReadMore(drop_size, &error)) {
      *fragment_type_or_err = error;
      return false;
@@ -1028,8 +923,6 @@ bool FragmentBufferedReader::TryReadFragment(Slice* fragment, size_t* drop_size,
  buffer_.remove_prefix(header_size + length);

  if (!uncompress_ || type == kSetCompressionType ||
-      type == kPredecessorWALInfoType ||
-      type == kRecyclePredecessorWALInfoType ||
      type == kUserDefinedTimestampSizeType ||
      type == kRecyclableUserDefinedTimestampSizeType) {
    *fragment = Slice(header + header_size, length);
@@ -10,7 +10,6 @@
 #pragma once
 #include <stdint.h>

-#include <cstdint>
 #include <memory>
 #include <unordered_map>
 #include <vector>
@@ -58,15 +57,9 @@ class Reader {
  // live while this Reader is in use.
  //
  // If "checksum" is true, verify checksums if available.
-  // TODO(hx235): seperate WAL related parameters from general `Reader`
-  // parameters
  Reader(std::shared_ptr<Logger> info_log,
         std::unique_ptr<SequentialFileReader>&& file, Reporter* reporter,
-         bool checksum, uint64_t log_num, bool track_and_verify_wals = false,
-         bool stop_replay_for_corruption = false,
-         uint64_t min_wal_number_to_keep = std::numeric_limits<uint64_t>::max(),
-         const PredecessorWALInfo& observed_predecessor_wal_info =
-             PredecessorWALInfo());
+         bool checksum, uint64_t log_num);
  // No copying allowed
  Reader(const Reader&) = delete;
  void operator=(const Reader&) = delete;
@@ -154,17 +147,6 @@ class Reader {
  // which log number this is
  uint64_t const log_number_;

-  // See `Optinos::track_and_verify_wals`
-  bool track_and_verify_wals_;
-  // Below variables are used for WAL verification
-  // TODO(hx235): To revise `stop_replay_for_corruption_` inside `LogReader`
-  // since we have `observed_predecessor_wal_info_` to verify against the
-  // `recorded_predecessor_wal_info_` recorded in current WAL. If there is no
-  // WAL hole, we can revise `stop_replay_for_corruption_` to be false.
-  bool stop_replay_for_corruption_;
-  uint64_t min_wal_number_to_keep_;
-  PredecessorWALInfo observed_predecessor_wal_info_;
-
  // Whether this is a recycled log file
  bool recycled_;

@@ -189,7 +171,7 @@ class Reader {
  UnorderedMap<uint32_t, size_t> recorded_cf_to_ts_sz_;

  // Extend record types with the following special values
-  enum : uint8_t {
+  enum {
    kEof = kMaxRecordType + 1,
    // Returned whenever we find an invalid physical record.
    // Currently there are three situations in which this happens:
@@ -210,11 +192,11 @@ class Reader {
  // If WAL compressioned is enabled, fragment_checksum is the checksum of the
  // fragment computed from the orginal buffer containinng uncompressed
  // fragment.
-  uint8_t ReadPhysicalRecord(Slice* result, size_t* drop_size,
-                             uint64_t* fragment_checksum = nullptr);
+  unsigned int ReadPhysicalRecord(Slice* result, size_t* drop_size,
+                                  uint64_t* fragment_checksum = nullptr);

  // Read some more
-  bool ReadMore(size_t* drop_size, uint8_t* error);
+  bool ReadMore(size_t* drop_size, int* error);

  void UnmarkEOFInternal();

@@ -228,25 +210,14 @@ class Reader {

  Status UpdateRecordedTimestampSize(
      const std::vector<std::pair<uint32_t, size_t>>& cf_to_ts_sz);
-
-  void MaybeVerifyPredecessorWALInfo(
-      WALRecoveryMode wal_recovery_mode, Slice fragment,
-      const PredecessorWALInfo& recorded_predecessor_wal_info);
 };

 class FragmentBufferedReader : public Reader {
 public:
-  FragmentBufferedReader(
-      std::shared_ptr<Logger> info_log,
-      std::unique_ptr<SequentialFileReader>&& _file, Reporter* reporter,
-      bool checksum, uint64_t log_num, bool verify_and_track_wals = false,
-      bool stop_replay_for_corruption = false,
-      uint64_t min_wal_number_to_keep = std::numeric_limits<uint64_t>::max(),
-      const PredecessorWALInfo& observed_predecessor_wal_info =
-          PredecessorWALInfo())
-      : Reader(info_log, std::move(_file), reporter, checksum, log_num,
-               verify_and_track_wals, stop_replay_for_corruption,
-               min_wal_number_to_keep, observed_predecessor_wal_info),
+  FragmentBufferedReader(std::shared_ptr<Logger> info_log,
+                         std::unique_ptr<SequentialFileReader>&& _file,
+                         Reporter* reporter, bool checksum, uint64_t log_num)
+      : Reader(info_log, std::move(_file), reporter, checksum, log_num),
        fragments_(),
        in_fragmented_record_(false) {}
  ~FragmentBufferedReader() override {}
@@ -261,9 +232,9 @@ class FragmentBufferedReader : public Reader {
  bool in_fragmented_record_;

  bool TryReadFragment(Slice* result, size_t* drop_size,
-                       uint8_t* fragment_type_or_err);
+                       unsigned int* fragment_type_or_err);

-  bool TryReadMore(size_t* drop_size, uint8_t* error);
+  bool TryReadMore(size_t* drop_size, int* error);

  // No copy allowed
  FragmentBufferedReader(const FragmentBufferedReader&);