Remove yield in point lock manager

The yield is actually of not much use because waitFor should already be doing that internally.
Ensure Property Bags are Pushed Down to BlockBasedIterator (#13795 )
2026-11-06 11:02:36 -05:00 · 2025-07-23 09:33:33 -07:00 · 2025-07-22 17:46:45 -07:00 · 2025-07-22 11:49:12 -07:00 · 2025-07-22 10:56:07 -07:00 · 2025-07-21 16:21:19 -07:00
144 changed files with 5882 additions and 2660 deletions
@@ -4,8 +4,8 @@ runs:
  steps:
  - name: Install Maven
    run: |
-      wget --no-check-certificate https://dlcdn.apache.org/maven/maven-3/3.9.10/binaries/apache-maven-3.9.10-bin.tar.gz
-      tar zxf apache-maven-3.9.10-bin.tar.gz
-      echo "export M2_HOME=$(pwd)/apache-maven-3.9.10" >> $GITHUB_ENV
-      echo "$(pwd)/apache-maven-3.9.10/bin" >> $GITHUB_PATH
+      wget --no-check-certificate https://archive.apache.org/dist/maven/maven-3/3.9.11/binaries/apache-maven-3.9.11-bin.tar.gz
+      tar zxf apache-maven-3.9.11-bin.tar.gz
+      echo "export M2_HOME=$(pwd)/apache-maven-3.9.11" >> $GITHUB_ENV
+      echo "$(pwd)/apache-maven-3.9.11/bin" >> $GITHUB_PATH
    shell: bash
@@ -27,18 +27,6 @@ jobs:
        git config --global --add safe.directory /__w/rocksdb/rocksdb
        tools/check_format_compatible.sh
    - uses: "./.github/actions/post-steps"
-  build-linux-run-microbench:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - run: DEBUG_LEVEL=0 make -j32 run_microbench
-    - uses: "./.github/actions/post-steps"
  build-linux-non-shm:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
@@ -91,15 +79,6 @@ jobs:
    steps:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/windows-build-steps"
-  build-windows-vs2022:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: windows-2022
-    env:
-      CMAKE_GENERATOR: Visual Studio 17 2022
-      CMAKE_PORTABLE: 1
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/windows-build-steps"
  build-linux-arm-test-full:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
@@ -110,3 +89,59 @@ jobs:
      - run: sudo apt-get update && sudo apt-get install -y build-essential libgflags-dev
      - run: make V=1 J=4 -j4 check
      - uses: "./.github/actions/post-steps"
+  build-examples:
+    if: ${{ github.repository_owner == 'facebook' }}
+    runs-on:
+      labels: 4-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/pre-steps"
+    - name: Build examples
+      run: make V=1 -j4 static_lib && cd examples && make V=1 -j4
+    - uses: "./.github/actions/post-steps"
+  build-fuzzers:
+    if: ${{ github.repository_owner == 'facebook' }}
+    runs-on:
+      labels: 4-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/pre-steps"
+    - name: Build rocksdb lib
+      run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
+    - name: Build fuzzers
+      run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
+    - uses: "./.github/actions/post-steps"
+  build-linux-gcc-11-no_test_run:
+    if: ${{ github.repository_owner == 'facebook' }}
+    runs-on:
+      labels: 16-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/pre-steps"
+    - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench
+    - uses: "./.github/actions/post-steps"
+  build-linux-cmake-with-folly-lite-no-test:
+    if: ${{ github.repository_owner == 'facebook' }}
+    runs-on:
+      labels: 16-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    env:
+      CC: gcc-10
+      CXX: g++-10
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/pre-steps"
+    - uses: "./.github/actions/setup-folly"
+    - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)"
+    - uses: "./.github/actions/post-steps"
@@ -109,22 +109,6 @@ jobs:
    - uses: "./.github/actions/build-folly"
    - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)"
    - uses: "./.github/actions/post-steps"
-  build-linux-cmake-with-folly-lite-no-test:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    env:
-      CC: gcc-10
-      CXX: g++-10
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-folly"
-    - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)"
-    - uses: "./.github/actions/post-steps"
  build-linux-make-with-folly:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
@@ -198,7 +182,7 @@ jobs:
    - uses: actions/checkout@v4.1.0
    - uses: "./.github/actions/pre-steps"
    - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check
-    - run: "./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression\n"
+    - run: "./sst_dump --help | grep -E -q 'Supported built-in compression types: kNoCompression$' # Verify no compiled in compression\n"
    - uses: "./.github/actions/post-steps"
  # ======================== Linux No Test Runs ======================= #
  build-linux-release:
@@ -242,34 +226,6 @@ jobs:
    - run: apt-get remove -y libgflags-dev
    - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
    - run: if ./db_stress --version; then false; else true; fi
-  build-examples:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - name: Build examples
-      run: make V=1 -j4 static_lib && cd examples && make V=1 -j4
-    - uses: "./.github/actions/post-steps"
-  build-fuzzers:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - name: Build rocksdb lib
-      run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
-    - name: Build fuzzers
-      run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
-    - uses: "./.github/actions/post-steps"
  build-linux-clang-no_test_run:
    if: ${{ github.repository_owner == 'facebook' }}
    runs-on:
@@ -317,18 +273,7 @@ jobs:
    - uses: "./.github/actions/pre-steps"
    - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all
    - uses: "./.github/actions/post-steps"
-  build-linux-gcc-11-no_test_run:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench
-    - uses: "./.github/actions/post-steps"
+
  # ======================== Linux Other Checks ======================= #
  build-linux-clang10-clang-analyze:
    if: ${{ github.repository_owner == 'facebook' }}
@@ -469,11 +414,11 @@ jobs:
    - uses: "./.github/actions/post-steps"
  # ======================== Windows with Tests ======================= #
  # NOTE: some windows jobs are in "nightly" to save resources
-  build-windows-vs2019:
+  build-windows-vs2022:
    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: windows-2019
+    runs-on: windows-2022
    env:
-      CMAKE_GENERATOR: Visual Studio 16 2019
+      CMAKE_GENERATOR: Visual Studio 17 2022
      CMAKE_PORTABLE: 1
    steps:
    - uses: actions/checkout@v4.1.0
@@ -484,7 +429,7 @@ jobs:
    runs-on:
      labels: 4-core-ubuntu
    container:
-      image: evolvedbinary/rocksjava:centos6_x64-be
+      image: evolvedbinary/rocksjava:centos7_x64-be
      options: --shm-size=16gb
    steps:
    # The docker image is intentionally based on an OS that has an older GLIBC version.
@@ -512,7 +457,7 @@ jobs:
    runs-on:
      labels: 4-core-ubuntu
    container:
-      image: evolvedbinary/rocksjava:centos6_x64-be
+      image: evolvedbinary/rocksjava:centos7_x64-be
      options: --shm-size=16gb
    steps:
    # The docker image is intentionally based on an OS that has an older GLIBC version.
@@ -607,7 +552,7 @@ jobs:
    runs-on:
      labels: 4-core-ubuntu
    container:
-      image: evolvedbinary/rocksjava:rockylinux8_x64-be
+      image: evolvedbinary/rocksjava:alpine3_x64-be
      options: --shm-size=16gb
    steps:
    - uses: actions/checkout@v4.1.0
@@ -88,6 +88,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "db/memtable_list.cc",
        "db/merge_helper.cc",
        "db/merge_operator.cc",
+        "db/multi_scan.cc",
        "db/output_validator.cc",
        "db/periodic_task_scheduler.cc",
        "db/range_del_aggregator.cc",
@@ -249,7 +250,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
        "trace_replay/trace_record_result.cc",
        "trace_replay/trace_replay.cc",
        "util/async_file_reader.cc",
-        "util/auto_skip_compressor.cc",
+        "util/auto_tune_compressor.cc",
        "util/build_version.cc",
        "util/cleanable.cc",
        "util/coding.cc",
@@ -721,6 +721,7 @@ set(SOURCES
        db/memtable_list.cc
        db/merge_helper.cc
        db/merge_operator.cc
+        db/multi_scan.cc
        db/output_validator.cc
        db/periodic_task_scheduler.cc
        db/range_del_aggregator.cc
@@ -874,7 +875,7 @@ set(SOURCES
        trace_replay/trace_record.cc
        trace_replay/trace_replay.cc
        util/async_file_reader.cc
-        util/auto_skip_compressor.cc
+        util/auto_tune_compressor.cc
        util/cleanable.cc
        util/coding.cc
        util/compaction_job_stats_impl.cc
@@ -1,6 +1,42 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`

+## 10.5.0 (07/18/2025)
+### Public API Changes
+* DB option skip_checking_sst_file_sizes_on_db_open is deprecated, in favor of validating file size in parallel in a thread pool, when db is opened. When DB is opened, with paranoid check enabled, a file with the wrong size would fail the DB open. With paranoid check disabled, the DB open would succeed, the column family with the corrupted file would not be read or write, while the other healthy column families could be read and write normally. When max_open_files option is not set to -1, only a subset of the files will be opened and checked. The rest of the files will be opened and checked when they are accessed.
+
+### Behavior Changes
+* PessimisticTransaction::GetWaitingTxns now returns waiting transaction information even if the current transaction has timed out. This allows the information to be surfaced to users for debugging purposes once it is known that the timeout has occured.
+* A new API GetFileSize is added to FSRandomAccessFile interface class. It uses fstat vs stat on the posix implementation which is more efficient. Caller could use it to get file size faster. This function might be required in the future for FileSystem implementation outside of the RocksDB code base.
+* RocksDB now triggers eligible compactions every 12 hours when periodic compaction is configured. This solves a limitation of the compaction trigger mechanism, which would only trigger compaction after specific events like flush, compaction, or SetOptions.
+
+### Bug Fixes
+* Fix a bug in BackupEngine that can crash backup due to a null FSWritableFile passed to WritableFileWriter.
+* Fix DB::NewMultiScan iterator to respect the scan upper bound specified in ScanOptions
+
+### Performance Improvements
+* Optimized MultiScan using BlockBasedTable to coalesce I/Os and prefetch all data blocks.
+
+## 10.4.0 (06/20/2025)
+### New Features
+* Add a new CF option `memtable_avg_op_scan_flush_trigger` that supports triggering memtable flush when an iterator scans through an expensive range of keys, with the average number of skipped keys from the active memtable exceeding the threshold.
+* Vector based memtable now supports concurrent writers (DBOptions::allow_concurrent_memtable_write) #13675.
+* Add new experimental `TransactionOptions::large_txn_commit_optimize_byte_threshold` to enable optimizations for large transaction commit by transaction batch data size.
+* Add a new option `CompactionOptionsUniversal::reduce_file_locking` and if it's true, auto universal compaction picking will adjust to minimize locking of input files when bottom priority compactions are waiting to run. This can increase the likelihood of existing L0s being selected for compaction, thereby improving write stall and reducing read regression.
+* Add new `format_version=7` to aid experimental support of custom compression algorithms with CompressionManager and block-based table. This format version includes changing the format of `TableProperties::compression_name`.
+
+### Public API Changes
+* Change NewExternalTableFactory to return a unique_ptr instead of shared_ptr.
+* Add an optional min file size requirement for deletion triggered compaction. It can be specified when creating `CompactOnDeletionCollectorFactory`.
+
+### Behavior Changes
+* `TransactionOptions::large_txn_commit_optimize_threshold` now has default value 0 for disabled. `TransactionDBOptions::txn_commit_bypass_memtable_threshold` now has no effect on transactions.
+
+### Bug Fixes
+* Fix a bug where CreateColumnFamilyWithImport() could miss the SST file for the memtable flush it triggered. The exported CF then may not contain the updates in the memtable when CreateColumnFamilyWithImport() is called.
+* Fix iterator operations returning NotImplemented status if disallow_memtable_writes and paranoid_memory_checks CF options are both set.
+* Fixed handling of file checksums in IngestExternalFile() to allow providing checksums using recognized but not necessarily the DB's preferred checksum function, to ease migration between checksum functions.
+
 ## 10.3.0 (05/17/2025)
 ### New Features
 * Add new experimental `CompactionOptionsFIFO::allow_trivial_copy_when_change_temperature` along with `CompactionOptionsFIFO::trivial_copy_buffer_size` to allow optimizing FIFO compactions with tiering when kChangeTemperature to move files from source tier FileSystem to another tier FileSystem via trivial and direct copying raw sst file instead of reading thru the content of the SST file then rebuilding the table files.
@@ -2147,14 +2147,14 @@ ZLIB_DOWNLOAD_BASE ?= http://zlib.net
 BZIP2_VER ?= 1.0.8
 BZIP2_SHA256 ?= ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
 BZIP2_DOWNLOAD_BASE ?= http://sourceware.org/pub/bzip2
-SNAPPY_VER ?= 1.2.1
-SNAPPY_SHA256 ?= 736aeb64d86566d2236ddffa2865ee5d7a82d26c9016b36218fcc27ea4f09f86
+SNAPPY_VER ?= 1.2.2
+SNAPPY_SHA256 ?= 90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc
 SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/archive
-LZ4_VER ?= 1.9.4
-LZ4_SHA256 ?= 0b0e3aa07c8c063ddf40b082bdf7e37a1562bda40a0ff5272957f3e987e0e54b
+LZ4_VER ?= 1.10.0
+LZ4_SHA256 ?= 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b
 LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive
-ZSTD_VER ?= 1.5.5
-ZSTD_SHA256 ?= 98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1
+ZSTD_VER ?= 1.5.7
+ZSTD_SHA256 ?= 37d7284556b20954e56e1ca85b80226768902e2edabd3b649e9e72c0c9012ee3
 ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
 CURL_SSL_OPTS ?= --tlsv1

@@ -2375,27 +2375,27 @@ rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86

 rocksdbjavastaticdockerx86:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x86-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_x86-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_x86-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerx86_64:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_x64-be --platform linux/amd64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerppc64le:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_ppc64le-be --platform linux/ppc64le --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerarm64v8:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_arm64v8-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_arm64v8-be --platform linux/aarch64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockers390x:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_s390x-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu18_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_s390x-be --platform linux/s390x --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu18_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerriscv64:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_riscv64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu20_riscv64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_riscv64-be --platform linux/riscv64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu20_riscv64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerx86musl:
 	mkdir -p java/target
@@ -2403,19 +2403,19 @@ rocksdbjavastaticdockerx86musl:

 rocksdbjavastaticdockerx86_64musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x64-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_x64-musl-be --platform linux/amd64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerppc64lemusl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_ppc64le-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_ppc64le-musl-be --platform linux/ppc64le --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockerarm64v8musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_arm64v8-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_arm64v8-musl-be --platform linux/aarch64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticdockers390xmusl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_s390x-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_s390x-musl-be --platform linux/s390x --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh

 rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral

@@ -2470,8 +2470,8 @@ jtest_run:
 jtest: rocksdbjava
 	cd java;$(MAKE) sample test

-jpmd: rocksdbjava rocksdbjavageneratepom
-	cd java;$(MAKE) pmd
+jpmd: rocksdbjavageneratepom
+	cd java;$(MAKE) java java_test pmd

 jdb_bench:
 	cd java;$(MAKE) db_bench;
@@ -2492,7 +2492,7 @@ checkout_folly:
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically
-	cd third-party/folly && git reset --hard d17bf897cb5bbf8f07b122a614e8cffdc38edcde
+	cd third-party/folly && git reset --hard e95383b7c8b5b1e46cf47acf2f317d54f93c8268
 	@# Apparently missing include
 	perl -pi -e 's/(#include <atomic>)/$$1\n#include <cstring>/' third-party/folly/folly/lang/Exception.h
 	@# Warning-as-error on memcpy
@@ -130,7 +130,7 @@ CacheWithSecondaryAdapter::~CacheWithSecondaryAdapter() {
              "Secondary cache reserved: %zu\n",
              pri_cache_res_->GetTotalMemoryUsed(), sec_capacity,
              sec_reserved_);
-      assert(pri_cache_res_mismatch);
+      assert(!pri_cache_res_mismatch);
    }
  }
 #endif  // NDEBUG
@@ -489,12 +489,10 @@ const char* CacheWithSecondaryAdapter::Name() const {
 // as well. At the moment, we don't have a good way of handling the case
 // where the new capacity < total cache reservations.
 void CacheWithSecondaryAdapter::SetCapacity(size_t capacity) {
-  size_t sec_capacity = static_cast<size_t>(
-      capacity * (distribute_cache_res_ ? sec_cache_res_ratio_ : 0.0));
-  size_t old_sec_capacity = 0;
-
  if (distribute_cache_res_) {
    MutexLock m(&cache_res_mutex_);
+    size_t sec_capacity = static_cast<size_t>(capacity * sec_cache_res_ratio_);
+    size_t old_sec_capacity = 0;

    Status s = secondary_cache_->GetCapacity(old_sec_capacity);
    if (!s.ok()) {
@@ -613,6 +611,7 @@ Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio(
    //    cache utilization (increase in capacity - increase in share of cache
    //    reservation)
    // 3. Increase secondary cache capacity
+    assert(new_sec_reserved >= sec_reserved_);
    s = secondary_cache_->Deflate(new_sec_reserved - sec_reserved_);
    assert(s.ok());
    s = pri_cache_res_->UpdateCacheReservation(
@@ -925,6 +925,10 @@ void rocksdb_backup_engine_options_destroy(
  delete options;
 }

+void rocksdb_status_ptr_get_error(rocksdb_status_ptr_t* status, char** errptr) {
+  SaveError(errptr, *(status->rep));
+}
+
 rocksdb_checkpoint_t* rocksdb_checkpoint_object_create(rocksdb_t* db,
                                                       char** errptr) {
  Checkpoint* checkpoint;
@@ -3061,6 +3065,10 @@ uint64_t rocksdb_flushjobinfo_smallest_seqno(
  return info->rep.smallest_seqno;
 }

+uint32_t rocksdb_flushjobinfo_flush_reason(const rocksdb_flushjobinfo_t* info) {
+  return static_cast<uint32_t>(info->rep.flush_reason);
+}
+
 void rocksdb_reset_status(rocksdb_status_ptr_t* status_ptr) {
  auto ptr = status_ptr->rep;
  *ptr = Status::OK();
@@ -3192,6 +3200,11 @@ int rocksdb_subcompactionjobinfo_output_level(
  return info->rep.output_level;
 }

+uint32_t rocksdb_subcompactionjobinfo_compaction_reason(
+    const rocksdb_subcompactionjobinfo_t* info) {
+  return static_cast<uint32_t>(info->rep.compaction_reason);
+}
+
 /* ExternalFileIngestionInfo */

 const char* rocksdb_externalfileingestioninfo_cf_name(
@@ -2230,10 +2230,6 @@ int main(int argc, char** argv) {
    rocksdb_options_set_skip_stats_update_on_db_open(o, 1);
    CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o));

-    rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open(o, 1);
-    CheckCondition(
-        1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(o));
-
    rocksdb_options_set_max_write_buffer_number(o, 97);
    CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(o));

@@ -2493,8 +2489,6 @@ int main(int argc, char** argv) {
    CheckCondition(2.0 ==
                   rocksdb_options_get_max_bytes_for_level_multiplier(copy));
    CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(copy));
-    CheckCondition(
-        1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(copy));
    CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(copy));
    CheckCondition(23 ==
                   rocksdb_options_get_min_write_buffer_number_to_merge(copy));
@@ -2681,12 +2675,6 @@ int main(int argc, char** argv) {
    CheckCondition(0 == rocksdb_options_get_skip_stats_update_on_db_open(copy));
    CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o));

-    rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open(copy, 0);
-    CheckCondition(
-        0 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(copy));
-    CheckCondition(
-        1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(o));
-
    rocksdb_options_set_max_write_buffer_number(copy, 2000);
    CheckCondition(2000 == rocksdb_options_get_max_write_buffer_number(copy));
    CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(o));
@@ -1633,6 +1633,8 @@ Status ColumnFamilyData::SetOptions(
  Status s = GetColumnFamilyOptionsFromMap(config_opts, cf_opts, options_map,
                                           &cf_opts);
  if (s.ok()) {
+    // FIXME: we should call SanitizeOptions() too or consolidate it with
+    // ValidateOptions().
    s = ValidateOptions(db_opts, cf_opts);
  }
  if (s.ok()) {
@@ -284,9 +284,9 @@ Compaction::Compaction(
    CompressionOptions _compression_opts, Temperature _output_temperature,
    uint32_t _max_subcompactions, std::vector<FileMetaData*> _grandparents,
    std::optional<SequenceNumber> _earliest_snapshot,
-    const SnapshotChecker* _snapshot_checker, bool _manual_compaction,
-    const std::string& _trim_ts, double _score, bool _deletion_compaction,
-    bool l0_files_might_overlap, CompactionReason _compaction_reason,
+    const SnapshotChecker* _snapshot_checker,
+    CompactionReason _compaction_reason, const std::string& _trim_ts,
+    double _score, bool l0_files_might_overlap,
    BlobGarbageCollectionPolicy _blob_garbage_collection_policy,
    double _blob_garbage_collection_age_cutoff)
    : input_vstorage_(vstorage),
@@ -304,7 +304,9 @@ Compaction::Compaction(
      output_compression_(_compression),
      output_compression_opts_(_compression_opts),
      output_temperature_(_output_temperature),
-      deletion_compaction_(_deletion_compaction),
+      deletion_compaction_(_compaction_reason == CompactionReason::kFIFOTtl ||
+                           _compaction_reason ==
+                               CompactionReason::kFIFOMaxSize),
      l0_files_might_overlap_(l0_files_might_overlap),
      inputs_(PopulateWithAtomicBoundaries(vstorage, std::move(_inputs))),
      grandparents_(std::move(_grandparents)),
@@ -321,7 +323,8 @@ Compaction::Compaction(
              ? false
              : IsBottommostLevel(output_level_, vstorage, inputs_)),
      is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
-      is_manual_compaction_(_manual_compaction),
+      is_manual_compaction_(_compaction_reason ==
+                            CompactionReason::kManualCompaction),
      trim_ts_(_trim_ts),
      is_trivial_move_(false),
      compaction_reason_(_compaction_reason),
@@ -349,9 +352,6 @@ Compaction::Compaction(
                                      immutable_options_, start_level_,
                                      output_level_)) {
  MarkFilesBeingCompacted(true);
-  if (is_manual_compaction_) {
-    compaction_reason_ = CompactionReason::kManualCompaction;
-  }
  if (max_subcompactions_ == 0) {
    max_subcompactions_ = _mutable_db_options.max_subcompactions;
  }
@@ -94,10 +94,9 @@ class Compaction {
             std::vector<FileMetaData*> grandparents,
             std::optional<SequenceNumber> earliest_snapshot,
             const SnapshotChecker* snapshot_checker,
-             bool manual_compaction = false, const std::string& trim_ts = "",
-             double score = -1, bool deletion_compaction = false,
+             CompactionReason compaction_reason,
+             const std::string& trim_ts = "", double score = -1,
             bool l0_files_might_overlap = true,
-             CompactionReason compaction_reason = CompactionReason::kUnknown,
             BlobGarbageCollectionPolicy blob_garbage_collection_policy =
                 BlobGarbageCollectionPolicy::kUseDefault,
             double blob_garbage_collection_age_cutoff = -1);
@@ -651,7 +651,8 @@ class CompactionJobTestBase : public testing::Test {
        mutable_cf_options_.max_compaction_bytes, 0, kNoCompression,
        cfd->GetLatestMutableCFOptions().compression_opts,
        Temperature::kUnknown, max_subcompactions, grandparents,
-        /*earliest_snapshot*/ std::nullopt, /*snapshot_checker*/ nullptr, true);
+        /*earliest_snapshot*/ std::nullopt, /*snapshot_checker*/ nullptr,
+        CompactionReason::kManualCompaction);
    compaction.FinalizeInputInfo(cfd->current());

    assert(db_options_.info_log);
@@ -381,7 +381,7 @@ Compaction* CompactionPicker::CompactFiles(
      mutable_cf_options.default_write_temperature,
      compact_options.max_subcompactions,
      /* grandparents */ {}, /* earliest_snapshot */ std::nullopt,
-      /* snapshot_checker */ nullptr, true);
+      /* snapshot_checker */ nullptr, CompactionReason::kManualCompaction);
  RegisterCompaction(c);
  return c;
 }
@@ -680,10 +680,9 @@ Compaction* CompactionPicker::CompactRange(
        mutable_cf_options.default_write_temperature,
        compact_range_options.max_subcompactions,
        /* grandparents */ {}, /* earliest_snapshot */ std::nullopt,
-        /* snapshot_checker */ nullptr,
-        /* is manual */ true, trim_ts, /* score */ -1,
-        /* deletion_compaction */ false, /* l0_files_might_overlap */ true,
-        CompactionReason::kUnknown,
+        /* snapshot_checker */ nullptr, CompactionReason::kManualCompaction,
+        trim_ts, /* score */ -1,
+        /* l0_files_might_overlap */ true,
        compact_range_options.blob_garbage_collection_policy,
        compact_range_options.blob_garbage_collection_age_cutoff);

@@ -873,9 +872,8 @@ Compaction* CompactionPicker::CompactRange(
      mutable_cf_options.default_write_temperature,
      compact_range_options.max_subcompactions, std::move(grandparents),
      /* earliest_snapshot */ std::nullopt, /* snapshot_checker */ nullptr,
-      /* is manual */ true, trim_ts, /* score */ -1,
-      /* deletion_compaction */ false, /* l0_files_might_overlap */ true,
-      CompactionReason::kUnknown,
+      CompactionReason::kManualCompaction, trim_ts, /* score */ -1,
+      /* l0_files_might_overlap */ true,
      compact_range_options.blob_garbage_collection_policy,
      compact_range_options.blob_garbage_collection_age_cutoff);

@@ -127,11 +127,9 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
      mutable_cf_options.compression_opts,
      mutable_cf_options.default_write_temperature,
      /* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
-      /* snapshot_checker */ nullptr,
-      /* is manual */ false,
+      /* snapshot_checker */ nullptr, CompactionReason::kFIFOTtl,
      /* trim_ts */ "", vstorage->CompactionScore(0),
-      /* is deletion compaction */ true, /* l0_files_might_overlap */ true,
-      CompactionReason::kFIFOTtl);
+      /* l0_files_might_overlap */ true);
  return c;
 }

@@ -200,11 +198,10 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
            mutable_cf_options.default_write_temperature,
            0 /* max_subcompactions */, {},
            /* earliest_snapshot */ std::nullopt,
-            /* snapshot_checker */ nullptr, /* is manual */ false,
+            /* snapshot_checker */ nullptr,
+            CompactionReason::kFIFOReduceNumFiles,
            /* trim_ts */ "", vstorage->CompactionScore(0),
-            /* is deletion compaction */ false,
-            /* l0_files_might_overlap */ true,
-            CompactionReason::kFIFOReduceNumFiles);
+            /* l0_files_might_overlap */ true);
        return c;
      }
    }
@@ -297,11 +294,9 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
      mutable_cf_options.compression_opts,
      mutable_cf_options.default_write_temperature,
      /* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
-      /* snapshot_checker */ nullptr,
-      /* is manual */ false,
+      /* snapshot_checker */ nullptr, CompactionReason::kFIFOMaxSize,
      /* trim_ts */ "", vstorage->CompactionScore(0),
-      /* is deletion compaction */ true,
-      /* l0_files_might_overlap */ true, CompactionReason::kFIFOMaxSize);
+      /* l0_files_might_overlap */ true);
  return c;
 }

@@ -416,10 +411,9 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
      mutable_cf_options.compression, mutable_cf_options.compression_opts,
      compaction_target_temp,
      /* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
-      /* snapshot_checker */ nullptr,
-      /* is manual */ false, /* trim_ts */ "", vstorage->CompactionScore(0),
-      /* is deletion compaction */ false, /* l0_files_might_overlap */ true,
-      CompactionReason::kChangeTemperature);
+      /* snapshot_checker */ nullptr, CompactionReason::kChangeTemperature,
+      /* trim_ts */ "", vstorage->CompactionScore(0),
+      /* l0_files_might_overlap */ true);
  return c;
 }

@@ -145,7 +145,6 @@ class LevelCompactionBuilder {
  int parent_index_ = -1;
  int base_index_ = -1;
  double start_level_score_ = 0;
-  bool is_manual_ = false;
  bool is_l0_trivial_move_ = false;
  CompactionInputFiles start_level_inputs_;
  std::vector<CompactionInputFiles> compaction_inputs_;
@@ -561,9 +560,8 @@ Compaction* LevelCompactionBuilder::GetCompaction() {
      mutable_cf_options_.default_write_temperature,
      /* max_subcompactions */ 0, std::move(grandparents_),
      /* earliest_snapshot */ std::nullopt, /* snapshot_checker */ nullptr,
-      is_manual_,
-      /* trim_ts */ "", start_level_score_, false /* deletion_compaction */,
-      l0_files_might_overlap, compaction_reason_);
+      compaction_reason_,
+      /* trim_ts */ "", start_level_score_, l0_files_might_overlap);

  // If it's level 0 compaction, make sure we don't execute any other level 0
  // compactions in parallel
@@ -1097,10 +1097,9 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(
                        mutable_cf_options_.default_write_temperature,
                        /* max_subcompactions */ 0, grandparents,
                        /* earliest_snapshot */ std::nullopt,
-                        /* snapshot_checker */ nullptr,
-                        /* is manual */ false, /* trim_ts */ "", score_,
-                        false /* deletion_compaction */,
-                        /* l0_files_might_overlap */ true, compaction_reason);
+                        /* snapshot_checker */ nullptr, compaction_reason,
+                        /* trim_ts */ "", score_,
+                        /* l0_files_might_overlap */ true);
 }

 // Look at overall size amplification. If size amplification
@@ -1447,10 +1446,9 @@ Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
      /* max_subcompactions */ 0, /* grandparents */ {},
      /* earliest_snapshot */ std::nullopt,
      /* snapshot_checker */ nullptr,
-      /* is manual */ false,
-      /* trim_ts */ "", score_, false /* deletion_compaction */,
-      /* l0_files_might_overlap */ true,
-      CompactionReason::kUniversalSizeAmplification);
+      CompactionReason::kUniversalSizeAmplification,
+      /* trim_ts */ "", score_,
+      /* l0_files_might_overlap */ true);
 }

 // Pick files marked for compaction. Typically, files are marked by
@@ -1600,11 +1598,9 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
      GetCompressionOptions(mutable_cf_options_, vstorage_, output_level),
      mutable_cf_options_.default_write_temperature,
      /* max_subcompactions */ 0, grandparents, earliest_snapshot_,
-      snapshot_checker_,
-      /* is manual */ false,
-      /* trim_ts */ "", score_, false /* deletion_compaction */,
-      /* l0_files_might_overlap */ true,
-      CompactionReason::kFilesMarkedForCompaction);
+      snapshot_checker_, CompactionReason::kFilesMarkedForCompaction,
+      /* trim_ts */ "", score_,
+      /* l0_files_might_overlap */ true);
 }

 Compaction* UniversalCompactionBuilder::PickCompactionToOldest(
@@ -1700,10 +1696,9 @@ Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange(
      mutable_cf_options_.default_write_temperature,
      /* max_subcompactions */ 0, /* grandparents */ {},
      /* earliest_snapshot */ std::nullopt,
-      /* snapshot_checker */ nullptr,
-      /* is manual */ false,
-      /* trim_ts */ "", score_, false /* deletion_compaction */,
-      /* l0_files_might_overlap */ true, compaction_reason);
+      /* snapshot_checker */ nullptr, compaction_reason,
+      /* trim_ts */ "", score_,
+      /* l0_files_might_overlap */ true);
 }

 Compaction* UniversalCompactionBuilder::PickPeriodicCompaction() {
@@ -556,6 +556,74 @@ TEST_F(CorruptionTest, TableFileFooterNotMagic) {
  ASSERT_TRUE(s.ToString().find(".sst") != std::string::npos);
 }

+TEST_F(CorruptionTest, DBOpenWithWrongFileSize) {
+  // Validate that when paranoid flag is true, DB::Open() fails if one of the
+  // file corrupted. Validate that when paranoid flag is false, DB::Open()
+  // succeed if one of the file corrupted, and the healthy file is readable.
+  CloseDb();
+
+  const std::string test_cf_name = "test_cf";
+  std::vector<ColumnFamilyDescriptor> cf_descs;
+  cf_descs.emplace_back(kDefaultColumnFamilyName, ColumnFamilyOptions());
+  cf_descs.emplace_back(test_cf_name, ColumnFamilyOptions());
+
+  {
+    options_.create_missing_column_families = true;
+    std::vector<ColumnFamilyHandle*> cfhs;
+    ASSERT_OK(DB::Open(options_, dbname_, cf_descs, &cfhs, &db_));
+    assert(db_ != nullptr);  // suppress false clang-analyze report
+
+    ASSERT_OK(db_->Put(WriteOptions(), cfhs[0], "k", "v"));
+    ASSERT_OK(db_->Put(WriteOptions(), cfhs[1], "k1", "v1"));
+    ASSERT_OK(db_->Put(WriteOptions(), cfhs[0], "k2", "v2"));
+    for (auto* cfh : cfhs) {
+      delete cfh;
+    }
+    DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
+    ASSERT_OK(dbi->TEST_FlushMemTable());
+
+    // ********************************************
+    // Corrupt the file by making the file bigger
+    std::vector<LiveFileMetaData> metadata;
+    db_->GetLiveFilesMetaData(&metadata);
+    std::string filename = dbname_ + metadata[0].name;
+    const auto& fs = options_.env->GetFileSystem();
+    {
+      std::unique_ptr<FSWritableFile> f;
+      ASSERT_OK(fs->ReopenWritableFile(filename, FileOptions(), &f, nullptr));
+      ASSERT_OK(f->Append("blahblah", IOOptions(), nullptr));
+      ASSERT_OK(f->Close(IOOptions(), nullptr));
+    }
+    CloseDb();
+  }
+
+  // DB failed to open due to one of the file is corrupted, as paranoid flag is
+  // true
+  options_.paranoid_checks = true;
+  std::vector<ColumnFamilyHandle*> cfhs;
+  auto s = DB::Open(options_, dbname_, cf_descs, &cfhs, &db_);
+  ASSERT_TRUE(s.IsCorruption());
+  ASSERT_TRUE(s.ToString().find("file size mismatch") != std::string::npos);
+
+  // DB opened successfully, as paranoid flag is false, validate the one that is
+  // healthy is still accessible
+  options_.paranoid_checks = false;
+  ASSERT_OK(DB::Open(options_, dbname_, cf_descs, &cfhs, &db_));
+  assert(db_ != nullptr);  // suppress false clang-analyze report
+
+  std::string v;
+  ASSERT_OK(db_->Get(ReadOptions(), cfhs[1], "k1", &v));
+  ASSERT_EQ(v, "v1");
+
+  // Validate the default column family is corrupted
+  Check(0, 0);
+  s = db_->Get(ReadOptions(), cfhs[0], "k1", &v);
+  ASSERT_TRUE(s.IsCorruption());
+
+  delete cfhs[1];
+  delete cfhs[0];
+}
+
 TEST_F(CorruptionTest, TableFileWrongSize) {
  Build(100);
  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
@@ -579,13 +647,16 @@ TEST_F(CorruptionTest, TableFileWrongSize) {
  // DB actually accepts this without paranoid checks, relying on size
  // recorded in manifest to locate the SST footer.
  options_.paranoid_checks = false;
-  options_.skip_checking_sst_file_sizes_on_db_open = false;
  Reopen();
-  Check(100, 100);
+  // As footer could not be extraced, file is completely unreadable
+  Check(0, 0);
+  std::string v;
+  auto s = db_->Get(ReadOptions(), "k1", &v);
+  ASSERT_TRUE(s.IsCorruption());

  // But reports the issue with paranoid checks
  options_.paranoid_checks = true;
-  Status s = TryReopen();
+  s = TryReopen();
  ASSERT_TRUE(s.IsCorruption());
  ASSERT_TRUE(s.ToString().find("file size mismatch") != std::string::npos);

@@ -11093,6 +11093,59 @@ TEST_F(DBCompactionTest, RecordNewestKeyTimeForTtlCompaction) {
  ASSERT_OK(dbfull()->TEST_WaitForCompact());
  ASSERT_EQ(NumTableFilesAtLevel(0), 0);
 }
+
+class PeriodicCompactionListener : public EventListener {
+ public:
+  explicit PeriodicCompactionListener() {}
+  void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override {
+    if (ci.compaction_reason == CompactionReason::kPeriodicCompaction) {
+      ++num_periodic_compactions;
+    }
+  }
+
+  std::atomic<int> num_periodic_compactions = 0;
+};
+
+TEST_F(DBCompactionTest, PeriodicTask) {
+  // Tests that when no trigger event is fired (flush/compaction/setoptions),
+  // periodic compaction is still triggered by a scheduled periodic function.
+  auto mock_clock = std::make_shared<MockSystemClock>(env_->GetSystemClock());
+  mock_clock->SetCurrentTime(100);
+  mock_clock->InstallTimedWaitFixCallback();
+  auto mock_env = std::make_unique<CompositeEnvWrapper>(env_, mock_clock);
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::StartPeriodicTaskScheduler:Init", [&](void* arg) {
+        auto periodic_task_scheduler_ptr =
+            static_cast<PeriodicTaskScheduler*>(arg);
+        periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock.get());
+      });
+
+  Options options;
+  options.env = mock_env.get();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.statistics = CreateDBStatistics();
+  int kPeriodicCompactionSeconds = 7 * 24 * 60 * 60;  // 1 week
+  options.periodic_compaction_seconds = kPeriodicCompactionSeconds;
+  options.num_levels = 50;
+  auto listener = std::make_shared<PeriodicCompactionListener>();
+  options.listeners.push_back(listener);
+  ASSERT_OK(TryReopen(options));
+
+  Random* rnd = Random::GetTLSInstance();
+  for (int k = 0; k < 10; ++k) {
+    ASSERT_OK(Put(Key(k), rnd->RandomString(100)));
+  }
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->CompactRange({}, nullptr, nullptr));
+  ASSERT_EQ(1, NumTableFilesAtLevel(49));
+
+  dbfull()->TEST_WaitForPeriodicTaskRun(
+      [&] { mock_clock->MockSleepForSeconds(kPeriodicCompactionSeconds + 1); });
+  ASSERT_OK(db_->WaitForCompact({}));
+
+  ASSERT_EQ(listener->num_periodic_compactions, 1);
+  Close();
+}
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -17,9 +17,10 @@ class DBEncryptionTest : public DBTestBase {
 public:
  DBEncryptionTest()
      : DBTestBase("db_encryption_test", /*env_do_fsync=*/true) {}
-  Env* GetTargetEnv() {
+  Env* GetNonEncryptedEnv() {
    if (encrypted_env_ != nullptr) {
-      return (static_cast<EnvWrapper*>(encrypted_env_))->target();
+      return (static_cast_with_check<CompositeEnvWrapper>(encrypted_env_))
+          ->env_target();
    } else {
      return env_;
    }
@@ -38,7 +39,7 @@ TEST_F(DBEncryptionTest, CheckEncrypted) {
  auto status = env_->GetChildren(dbname_, &fileNames);
  ASSERT_OK(status);

-  Env* target = GetTargetEnv();
+  Env* target = GetNonEncryptedEnv();
  int hits = 0;
  for (auto it = fileNames.begin(); it != fileNames.end(); ++it) {
    if (*it == "LOCK") {
@@ -89,7 +90,7 @@ TEST_F(DBEncryptionTest, CheckEncrypted) {
 }

 TEST_F(DBEncryptionTest, ReadEmptyFile) {
-  auto defaultEnv = GetTargetEnv();
+  auto defaultEnv = GetNonEncryptedEnv();

  // create empty file for reading it back in later
  auto envOptions = EnvOptions(CurrentOptions());
@@ -116,6 +117,40 @@ TEST_F(DBEncryptionTest, ReadEmptyFile) {
  ASSERT_TRUE(data.empty());
 }

+TEST_F(DBEncryptionTest, NotSupportedGetFileSize) {
+  // Validate envrypted env does not support GetFileSize.
+  // The goal of the test is to validate the encrypted env/fs does not support
+  // GetFileSize API on FSRandomAccessFile interface.
+  // This test combined with the rest of the integration tests validate that
+  // the new API GetFileSize on FSRandomAccessFile interface is not required to
+  // be supported for database to work properly.
+  // The GetFileSize API is used in ReadFooterFromFile() API to get the file
+  // size. When GetFileSize API is not supported, the ReadFooterFromFile() API
+  // will use FileSystem GetFileSize API as fallback. Refer to the
+  // EncryptedRandomAccessFile class definition for more details.
+  if (!encrypted_env_) {
+    return;
+  }
+
+  auto fs = encrypted_env_->GetFileSystem();
+
+  // create empty file for reading it back in later
+  auto filePath = dbname_ + "/empty.empty";
+
+  // Create empty file
+  CreateFile(fs.get(), filePath, "", false);
+
+  // Open it for reading footer
+  std::unique_ptr<FSRandomAccessFile> randomAccessFile;
+  auto status = fs->NewRandomAccessFile(filePath, FileOptions(),
+                                        &randomAccessFile, nullptr);
+  ASSERT_OK(status);
+
+  uint64_t fileSize;
+  status = randomAccessFile->GetFileSize(&fileSize);
+  ASSERT_TRUE(status.IsNotSupported());
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -253,6 +253,9 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
  periodic_task_functions_.emplace(
      PeriodicTaskType::kRecordSeqnoTime,
      [this]() { this->RecordSeqnoToTimeMapping(); });
+  periodic_task_functions_.emplace(
+      PeriodicTaskType::kTriggerCompaction,
+      [this]() { this->TriggerPeriodicCompaction(); });

  versions_.reset(new VersionSet(
      dbname_, &immutable_db_options_, file_options_, table_cache_.get(),
@@ -787,7 +790,8 @@ Status DBImpl::StartPeriodicTaskScheduler() {
    Status s = periodic_task_scheduler_.Register(
        PeriodicTaskType::kDumpStats,
        periodic_task_functions_.at(PeriodicTaskType::kDumpStats),
-        mutable_db_options_.stats_dump_period_sec);
+        mutable_db_options_.stats_dump_period_sec,
+        /*run_immediately=*/true);
    if (!s.ok()) {
      return s;
    }
@@ -796,7 +800,8 @@ Status DBImpl::StartPeriodicTaskScheduler() {
    Status s = periodic_task_scheduler_.Register(
        PeriodicTaskType::kPersistStats,
        periodic_task_functions_.at(PeriodicTaskType::kPersistStats),
-        mutable_db_options_.stats_persist_period_sec);
+        mutable_db_options_.stats_persist_period_sec,
+        /*run_immediately=*/true);
    if (!s.ok()) {
      return s;
    }
@@ -804,7 +809,15 @@ Status DBImpl::StartPeriodicTaskScheduler() {

  Status s = periodic_task_scheduler_.Register(
      PeriodicTaskType::kFlushInfoLog,
-      periodic_task_functions_.at(PeriodicTaskType::kFlushInfoLog));
+      periodic_task_functions_.at(PeriodicTaskType::kFlushInfoLog),
+      /*run_immediately=*/true);
+
+  if (s.ok()) {
+    s = periodic_task_scheduler_.Register(
+        PeriodicTaskType::kTriggerCompaction,
+        periodic_task_functions_.at(PeriodicTaskType::kTriggerCompaction),
+        /*run_immediately=*/false);
+  }

  return s;
 }
@@ -855,7 +868,7 @@ Status DBImpl::RegisterRecordSeqnoTimeWorker() {
    s = periodic_task_scheduler_.Register(
        PeriodicTaskType::kRecordSeqnoTime,
        periodic_task_functions_.at(PeriodicTaskType::kRecordSeqnoTime),
-        seqno_time_cadence);
+        seqno_time_cadence, /*run_immediately=*/true);
  }

  return s;
@@ -1365,7 +1378,7 @@ Status DBImpl::SetDBOptions(
        s = periodic_task_scheduler_.Register(
            PeriodicTaskType::kDumpStats,
            periodic_task_functions_.at(PeriodicTaskType::kDumpStats),
-            new_options.stats_dump_period_sec);
+            new_options.stats_dump_period_sec, /*run_immediately=*/true);
      }
      if (new_options.max_total_wal_size !=
          mutable_db_options_.max_total_wal_size) {
@@ -1380,7 +1393,7 @@ Status DBImpl::SetDBOptions(
          s = periodic_task_scheduler_.Register(
              PeriodicTaskType::kPersistStats,
              periodic_task_functions_.at(PeriodicTaskType::kPersistStats),
-              new_options.stats_persist_period_sec);
+              new_options.stats_persist_period_sec, /*run_immediately=*/true);
        }
      }
      mutex_.Lock();
@@ -3823,10 +3836,8 @@ bool DBImpl::KeyMayExist(const ReadOptions& read_options,
 std::unique_ptr<MultiScan> DBImpl::NewMultiScan(
    const ReadOptions& _read_options, ColumnFamilyHandle* column_family,
    const std::vector<ScanOptions>& scan_opts) {
-  std::unique_ptr<Iterator> iter(NewIterator(_read_options, column_family));
-  iter->Prepare(scan_opts);
-  std::unique_ptr<MultiScan> ms_iter =
-      std::make_unique<MultiScan>(scan_opts, std::move(iter));
+  std::unique_ptr<MultiScan> ms_iter = std::make_unique<MultiScan>(
+      _read_options, scan_opts, this, column_family);
  return ms_iter;
 }

@@ -5044,85 +5055,6 @@ void DBImpl::GetAllColumnFamilyMetaData(
  }
 }

-Status DBImpl::CheckConsistency() {
-  mutex_.AssertHeld();
-  std::vector<LiveFileMetaData> metadata;
-  versions_->GetLiveFilesMetaData(&metadata);
-  TEST_SYNC_POINT("DBImpl::CheckConsistency:AfterGetLiveFilesMetaData");
-
-  std::string corruption_messages;
-
-  if (immutable_db_options_.skip_checking_sst_file_sizes_on_db_open) {
-    // Instead of calling GetFileSize() for each expected file, call
-    // GetChildren() for the DB directory and check that all expected files
-    // are listed, without checking their sizes.
-    // Since sst files might be in different directories, do it for each
-    // directory separately.
-    std::map<std::string, std::vector<std::string>> files_by_directory;
-    for (const auto& md : metadata) {
-      // md.name has a leading "/". Remove it.
-      std::string fname = md.name;
-      if (!fname.empty() && fname[0] == '/') {
-        fname = fname.substr(1);
-      }
-      files_by_directory[md.db_path].push_back(fname);
-    }
-
-    IOOptions io_opts;
-    io_opts.do_not_recurse = true;
-    for (const auto& dir_files : files_by_directory) {
-      std::string directory = dir_files.first;
-      std::vector<std::string> existing_files;
-      Status s = fs_->GetChildren(directory, io_opts, &existing_files,
-                                  /*IODebugContext*=*/nullptr);
-      if (!s.ok()) {
-        corruption_messages +=
-            "Can't list files in " + directory + ": " + s.ToString() + "\n";
-        continue;
-      }
-      std::sort(existing_files.begin(), existing_files.end());
-
-      for (const std::string& fname : dir_files.second) {
-        if (!std::binary_search(existing_files.begin(), existing_files.end(),
-                                fname) &&
-            !std::binary_search(existing_files.begin(), existing_files.end(),
-                                Rocks2LevelTableFileName(fname))) {
-          corruption_messages +=
-              "Missing sst file " + fname + " in " + directory + "\n";
-        }
-      }
-    }
-  } else {
-    for (const auto& md : metadata) {
-      // md.name has a leading "/".
-      std::string file_path = md.db_path + md.name;
-
-      uint64_t fsize = 0;
-      TEST_SYNC_POINT("DBImpl::CheckConsistency:BeforeGetFileSize");
-      Status s = env_->GetFileSize(file_path, &fsize);
-      if (!s.ok() &&
-          env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok()) {
-        s = Status::OK();
-      }
-      if (!s.ok()) {
-        corruption_messages +=
-            "Can't access " + md.name + ": " + s.ToString() + "\n";
-      } else if (fsize != md.size) {
-        corruption_messages += "Sst file size mismatch: " + file_path +
-                               ". Size recorded in manifest " +
-                               std::to_string(md.size) + ", actual size " +
-                               std::to_string(fsize) + "\n";
-      }
-    }
-  }
-
-  if (corruption_messages.size() == 0) {
-    return Status::OK();
-  } else {
-    return Status::Corruption(corruption_messages);
-  }
-}
-
 Status DBImpl::GetDbIdentity(std::string& identity) const {
  identity.assign(db_id_);
  return Status::OK();
@@ -6884,6 +6816,35 @@ void DBImpl::RecordSeqnoToTimeMapping() {
  sv_context.Clean();
 }

+void DBImpl::TriggerPeriodicCompaction() {
+  TEST_SYNC_POINT("DBImpl::TriggerPeriodicCompaction:StartRunning");
+  {
+    InstrumentedMutexLock l(&mutex_);
+    ROCKS_LOG_INFO(immutable_db_options_.info_log,
+                   "Running the periodic task to trigger compactions.");
+
+    for (ColumnFamilyData* cfd : *versions_->GetColumnFamilySet()) {
+      if (cfd->IsDropped()) {
+        continue;
+      }
+      if (cfd->GetLatestCFOptions().periodic_compaction_seconds &&
+          !cfd->queued_for_compaction()) {
+        cfd->current()->storage_info()->ComputeCompactionScore(
+            cfd->ioptions(), cfd->GetLatestMutableCFOptions());
+        EnqueuePendingCompaction(cfd);
+        if (cfd->queued_for_compaction()) {
+          ROCKS_LOG_INFO(immutable_db_options_.info_log,
+                         "Periodic task to trigger compaction queued Column "
+                         "family [%s] for compaction.",
+                         cfd->GetName().c_str());
+        }
+      }
+    }
+    MaybeScheduleFlushOrCompaction();
+    bg_cv_.SignalAll();
+  }
+}
+
 void DBImpl::TrackOrUntrackFiles(
    const std::vector<std::string>& existing_data_files, bool track) {
  auto sfm = static_cast_with_check<SstFileManagerImpl>(
@@ -804,10 +804,6 @@ class DBImpl : public DB {
  // being detected.
  const Snapshot* GetSnapshotForWriteConflictBoundary();

-  // checks if all live files exist on file system and that their file sizes
-  // match to our in-memory records
-  virtual Status CheckConsistency();
-
  // max_file_num_to_ignore allows bottom level compaction to filter out newly
  // compacted SST files. Setting max_file_num_to_ignore to kMaxUint64 will
  // disable the filtering
@@ -1286,6 +1282,12 @@ class DBImpl : public DB {
  // For the background timer job
  void RecordSeqnoToTimeMapping();

+  // Compactions rely on an event triggers like flush/compaction/SetOptions.
+  // We need to trigger periodic compactions even when there is no such trigger.
+  // This function checks and schedules available compactions and will run
+  // periodically.
+  void TriggerPeriodicCompaction();
+
  // REQUIRES: DB mutex held
  std::pair<SequenceNumber, uint64_t> GetSeqnoToTimeSample() const;

@@ -1849,10 +1849,9 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
        0 /* max_subcompactions, not applicable */,
        {} /* grandparents, not applicable */,
        std::nullopt /* earliest_snapshot */, nullptr /* snapshot_checker */,
-        false /* is manual */, "" /* trim_ts */, -1 /* score, not applicable */,
-        false /* is deletion compaction, not applicable */,
-        false /* l0_files_might_overlap, not applicable */,
-        CompactionReason::kRefitLevel));
+        CompactionReason::kRefitLevel, "" /* trim_ts */,
+        -1 /* score, not applicable */,
+        false /* l0_files_might_overlap, not applicable */));
    cfd->compaction_picker()->RegisterCompaction(c.get());
    TEST_SYNC_POINT("DBImpl::ReFitLevel:PostRegisterCompaction");
    VersionEdit edit;
@@ -4424,13 +4423,14 @@ Compaction* DBImpl::CreateIntendedCompactionForwardedToBottomPriorityPool(

  c->ReleaseCompactionFiles(Status::OK());

-  Compaction* intended_compaction = new Compaction(
-      vstorage, io, mo, mutable_db_options_, std::move(inputs),
-      c->output_level(), c->target_output_file_size(),
-      c->max_compaction_bytes(), c->output_path_id(), c->output_compression(),
-      c->output_compression_opts(), c->output_temperature(),
-      c->max_subcompactions(), c->grandparents(),
-      std::nullopt /* earliest_snapshot */, nullptr /* snapshot_checker */);
+  Compaction* intended_compaction =
+      new Compaction(vstorage, io, mo, mutable_db_options_, std::move(inputs),
+                     c->output_level(), c->target_output_file_size(),
+                     c->max_compaction_bytes(), c->output_path_id(),
+                     c->output_compression(), c->output_compression_opts(),
+                     c->output_temperature(), c->max_subcompactions(),
+                     c->grandparents(), std::nullopt /* earliest_snapshot */,
+                     nullptr /* snapshot_checker */, c->compaction_reason());

  cfd->compaction_picker()->RegisterCompaction(intended_compaction);
  vstorage->ComputeCompactionScore(io, mo);
@@ -70,9 +70,6 @@ Status DBImplFollower::Recover(
    }
    return s;
  }
-  if (immutable_db_options_.paranoid_checks && s.ok()) {
-    s = CheckConsistency();
-  }
  if (s.ok()) {
    default_cf_handle_ = new ColumnFamilyHandleImpl(
        versions_->GetColumnFamilySet()->GetDefault(), this, &mutex_);
@@ -191,12 +191,6 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src,
                   "wal_compression is disabled since only zstd is supported");
  }

-  if (!result.paranoid_checks) {
-    result.skip_checking_sst_file_sizes_on_db_open = true;
-    ROCKS_LOG_INFO(result.info_log,
-                   "file size check will be skipped during open.");
-  }
-
  return result;
 }

@@ -694,9 +688,6 @@ Status DBImpl::Recover(
    s = MaybeUpdateNextFileNumber(recovery_ctx);
  }

-  if (immutable_db_options_.paranoid_checks && s.ok()) {
-    s = CheckConsistency();
-  }
  if (s.ok() && !read_only) {
    // TODO: share file descriptors (FSDirectory) with SetDirectories above
    std::map<std::string, std::shared_ptr<FSDirectory>> created_dirs;
@@ -49,9 +49,6 @@ Status DBImplSecondary::Recover(
    }
    return s;
  }
-  if (immutable_db_options_.paranoid_checks && s.ok()) {
-    s = CheckConsistency();
-  }
  // Initial max_total_in_memory_state_ before recovery logs.
  max_total_in_memory_state_ = 0;
  for (auto cfd : *versions_->GetColumnFamilySet()) {
@@ -653,49 +650,6 @@ Status DBImplSecondary::NewIterators(
  return Status::OK();
 }

-Status DBImplSecondary::CheckConsistency() {
-  mutex_.AssertHeld();
-  Status s = DBImpl::CheckConsistency();
-  // If DBImpl::CheckConsistency() which is stricter returns success, then we
-  // do not need to give a second chance.
-  if (s.ok()) {
-    return s;
-  }
-  // It's possible that DBImpl::CheckConssitency() can fail because the primary
-  // may have removed certain files, causing the GetFileSize(name) call to
-  // fail and returning a PathNotFound. In this case, we take a best-effort
-  // approach and just proceed.
-  TEST_SYNC_POINT_CALLBACK(
-      "DBImplSecondary::CheckConsistency:AfterFirstAttempt", &s);
-
-  if (immutable_db_options_.skip_checking_sst_file_sizes_on_db_open) {
-    return Status::OK();
-  }
-
-  std::vector<LiveFileMetaData> metadata;
-  versions_->GetLiveFilesMetaData(&metadata);
-
-  std::string corruption_messages;
-  for (const auto& md : metadata) {
-    // md.name has a leading "/".
-    std::string file_path = md.db_path + md.name;
-
-    uint64_t fsize = 0;
-    s = env_->GetFileSize(file_path, &fsize);
-    if (!s.ok() &&
-        (env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok() ||
-         s.IsPathNotFound())) {
-      s = Status::OK();
-    }
-    if (!s.ok()) {
-      corruption_messages +=
-          "Can't access " + md.name + ": " + s.ToString() + "\n";
-    }
-  }
-  return corruption_messages.empty() ? Status::OK()
-                                     : Status::Corruption(corruption_messages);
-}
-
 Status DBImplSecondary::TryCatchUpWithPrimary() {
  assert(versions_.get() != nullptr);
  Status s;
@@ -248,12 +248,6 @@ class DBImplSecondary : public DBImpl {
  Status MaybeInitLogReader(uint64_t log_number,
                            log::FragmentBufferedReader** log_reader);

-  // Check if all live files exist on file system and that their file sizes
-  // matche to the in-memory records. It is possible that some live files may
-  // have been deleted by the primary. In this case, CheckConsistency() does
-  // not flag the missing file as inconsistency.
-  Status CheckConsistency() override;
-
 #ifndef NDEBUG
  Status TEST_CompactWithoutInstallation(const OpenAndCompactOptions& options,
                                         ColumnFamilyHandle* cfh,
@@ -8,6 +8,8 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

 #include <functional>
+#include <iomanip>
+#include <iostream>

 #include "db/arena_wrapped_db_iter.h"
 #include "db/db_iter.h"
@@ -4139,6 +4141,190 @@ TEST_P(DBIteratorTest, AverageMemtableOpsScanFlushTriggerByOverwrites) {
  ASSERT_OK(db_->WaitForCompact({}));
  ASSERT_EQ(1, NumTableFilesAtLevel(0));
 }
+
+class DBMultiScanIteratorTest : public DBTestBase {
+ public:
+  DBMultiScanIteratorTest()
+      : DBTestBase("db_multi_scan_iterator_test", /*env_do_fsync=*/true) {}
+};
+
+TEST_F(DBMultiScanIteratorTest, BasicTest) {
+  // Create a file
+  for (int i = 0; i < 100; ++i) {
+    std::stringstream ss;
+    ss << std::setw(2) << std::setfill('0') << i;
+    ASSERT_OK(Put("k" + ss.str(), "val" + ss.str()));
+  }
+  ASSERT_OK(Flush());
+
+  std::vector<std::string> key_ranges({"k03", "k10", "k25", "k50"});
+  ReadOptions ro;
+  std::vector<ScanOptions> scan_options(
+      {ScanOptions(key_ranges[0], key_ranges[1]),
+       ScanOptions(key_ranges[2], key_ranges[3])});
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    int idx = 0;
+    int count = 0;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString().compare(key_ranges[idx]), 0);
+        ASSERT_LT(it.first.ToString().compare(key_ranges[idx + 1]), 0);
+        count++;
+      }
+      idx += 2;
+    }
+    ASSERT_EQ(count, 32);
+  } catch (MultiScanException& ex) {
+    // Make sure exception contains the status
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
+  }
+  iter.reset();
+
+  // Test the overlapping scan case
+  key_ranges[1] = "k30";
+  scan_options[0] = ScanOptions(key_ranges[0], key_ranges[1]);
+  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    int idx = 0;
+    int count = 0;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString().compare(key_ranges[idx]), 0);
+        ASSERT_LT(it.first.ToString().compare(key_ranges[idx + 1]), 0);
+        count++;
+      }
+      idx += 2;
+    }
+    ASSERT_EQ(count, 52);
+  } catch (MultiScanException& ex) {
+    // Make sure exception contains the status
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
+  }
+  iter.reset();
+
+  // Test the no limit scan case
+  scan_options[0] = ScanOptions(key_ranges[0]);
+  scan_options[1] = ScanOptions(key_ranges[2]);
+  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    int idx = 0;
+    int count = 0;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString().compare(key_ranges[idx]), 0);
+        if (it.first.ToString().compare(key_ranges[idx + 1]) == 0) {
+          break;
+        }
+        count++;
+      }
+      idx += 2;
+    }
+    ASSERT_EQ(count, 52);
+  } catch (MultiScanException& ex) {
+    // Make sure exception contains the status
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
+  }
+  iter.reset();
+}
+
+TEST_F(DBMultiScanIteratorTest, MixedBoundsTest) {
+  // Create a file
+  for (int i = 0; i < 100; ++i) {
+    std::stringstream ss;
+    ss << std::setw(2) << std::setfill('0') << i;
+    ASSERT_OK(Put("k" + ss.str(), "val" + ss.str()));
+  }
+  ASSERT_OK(Flush());
+
+  std::vector<std::string> key_ranges(
+      {"k03", "k10", "k25", "k50", "k75", "k90"});
+  ReadOptions ro;
+  std::vector<ScanOptions> scan_options(
+      {ScanOptions(key_ranges[0], key_ranges[1]), ScanOptions(key_ranges[2]),
+       ScanOptions(key_ranges[4], key_ranges[5])});
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    int idx = 0;
+    int count = 0;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString().compare(
+                      scan_options[idx].range.start->ToString()),
+                  0);
+        if (scan_options[idx].range.limit) {
+          ASSERT_LT(it.first.ToString().compare(
+                        scan_options[idx].range.limit->ToString()),
+                    0);
+        }
+        count++;
+      }
+      idx++;
+    }
+    ASSERT_EQ(count, 97);
+  } catch (MultiScanException& ex) {
+    // Make sure exception contains the status
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
+  }
+  iter.reset();
+
+  scan_options[0] = ScanOptions(key_ranges[0]);
+  scan_options[1] = ScanOptions(key_ranges[2], key_ranges[3]);
+  scan_options[2] = ScanOptions(key_ranges[4]);
+  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
+  try {
+    int idx = 0;
+    int count = 0;
+    for (auto range : *iter) {
+      for (auto it : range) {
+        ASSERT_GE(it.first.ToString().compare(
+                      scan_options[idx].range.start->ToString()),
+                  0);
+        if (scan_options[idx].range.limit) {
+          ASSERT_LT(it.first.ToString().compare(
+                        scan_options[idx].range.limit->ToString()),
+                    0);
+        }
+        count++;
+      }
+      idx++;
+    }
+    ASSERT_EQ(count, 147);
+  } catch (MultiScanException& ex) {
+    // Make sure exception contains the status
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  } catch (std::logic_error& ex) {
+    std::cerr << "Iterator returned logic error " << ex.what();
+    abort();
+  }
+  iter.reset();
+}
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -605,6 +605,9 @@ class TraceFileEnv : public EnvWrapper {
                  char* scratch) const override {
        return target_->Read(offset, n, result, scratch);
      }
+      Status GetFileSize(uint64_t* file_size) override {
+        return target_->GetFileSize(file_size);
+      }

     private:
      std::unique_ptr<RandomAccessFile> target_;
@@ -1291,46 +1294,6 @@ TEST_F(DBSecondaryTest, CatchUpAfterFlush) {
  ASSERT_OK(iter3->status());
 }

-TEST_F(DBSecondaryTest, CheckConsistencyWhenOpen) {
-  bool called = false;
-  Options options;
-  options.env = env_;
-  options.disable_auto_compactions = true;
-  Reopen(options);
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-  SyncPoint::GetInstance()->SetCallBack(
-      "DBImplSecondary::CheckConsistency:AfterFirstAttempt", [&](void* arg) {
-        ASSERT_NE(nullptr, arg);
-        called = true;
-        auto* s = static_cast<Status*>(arg);
-        ASSERT_NOK(*s);
-      });
-  SyncPoint::GetInstance()->LoadDependency(
-      {{"DBImpl::CheckConsistency:AfterGetLiveFilesMetaData",
-        "BackgroundCallCompaction:0"},
-       {"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles",
-        "DBImpl::CheckConsistency:BeforeGetFileSize"}});
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(Put("a", "value0"));
-  ASSERT_OK(Put("c", "value0"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Put("b", "value1"));
-  ASSERT_OK(Put("d", "value1"));
-  ASSERT_OK(Flush());
-  port::Thread thread([this]() {
-    Options opts;
-    opts.env = env_;
-    opts.max_open_files = -1;
-    OpenSecondary(opts);
-  });
-  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  thread.join();
-  ASSERT_TRUE(called);
-}
-
 TEST_F(DBSecondaryTest, StartFromInconsistent) {
  Options options = CurrentOptions();
  DestroyAndReopen(options);
@@ -135,21 +135,6 @@ TEST_F(DBSSTTest, SSTsWithLdbSuffixHandling) {
  Destroy(options);
 }

-// Check that we don't crash when opening DB with
-// DBOptions::skip_checking_sst_file_sizes_on_db_open = true.
-TEST_F(DBSSTTest, SkipCheckingSSTFileSizesOnDBOpen) {
-  ASSERT_OK(Put("pika", "choo"));
-  ASSERT_OK(Flush());
-
-  // Just open the DB with the option set to true and check that we don't crash.
-  Options options;
-  options.env = env_;
-  options.skip_checking_sst_file_sizes_on_db_open = true;
-  Reopen(options);
-
-  ASSERT_EQ("choo", Get("pika"));
-}
-
 TEST_F(DBSSTTest, DontDeleteMovedFile) {
  // This test triggers move compaction and verifies that the file is not
  // deleted when it's part of move compaction
@@ -1748,45 +1733,6 @@ TEST_F(DBSSTTest, GetTotalSstFilesSize) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

-TEST_F(DBSSTTest, OpenDBWithoutGetFileSizeInvocations) {
-  Options options = CurrentOptions();
-  std::unique_ptr<MockEnv> env{MockEnv::Create(Env::Default())};
-  options.env = env.get();
-  options.disable_auto_compactions = true;
-  options.compression = kNoCompression;
-  options.enable_blob_files = true;
-  options.blob_file_size = 32;  // create one blob per file
-  options.skip_checking_sst_file_sizes_on_db_open = true;
-
-  DestroyAndReopen(options);
-  // Generate 5 files in L0
-  for (int i = 0; i < 5; i++) {
-    for (int j = 0; j < 10; j++) {
-      std::string val = "val_file_" + std::to_string(i);
-      ASSERT_OK(Put(Key(j), val));
-    }
-    ASSERT_OK(Flush());
-  }
-  Close();
-
-  bool is_get_file_size_called = false;
-  SyncPoint::GetInstance()->SetCallBack(
-      "MockFileSystem::GetFileSize:CheckFileType", [&](void* arg) {
-        std::string* filename = static_cast<std::string*>(arg);
-        if (filename->find(".blob") != std::string::npos) {
-          is_get_file_size_called = true;
-        }
-      });
-
-  SyncPoint::GetInstance()->EnableProcessing();
-  Reopen(options);
-  ASSERT_FALSE(is_get_file_size_called);
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  Destroy(options);
-}
-
 TEST_F(DBSSTTest, GetTotalSstFilesSizeVersionsFilesShared) {
  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
@@ -1278,12 +1278,6 @@ class DelayFilterFactory : public CompactionFilterFactory {
 };
 }  // anonymous namespace

-static std::string CompressibleString(Random* rnd, int len) {
-  std::string r;
-  test::CompressibleString(rnd, 0.8, len, &r);
-  return r;
-}
-
 TEST_F(DBTest, FailMoreDbPaths) {
  Options options = CurrentOptions();
  options.db_paths.emplace_back(dbname_, 10000000);
@@ -5407,271 +5401,6 @@ TEST_F(DBTest, FlushOnDestroy) {
  CancelAllBackgroundWork(db_);
 }

-TEST_F(DBTest, DynamicLevelCompressionPerLevel) {
-  if (!Snappy_Supported()) {
-    return;
-  }
-  const int kNKeys = 120;
-  int keys[kNKeys];
-  for (int i = 0; i < kNKeys; i++) {
-    keys[i] = i;
-  }
-
-  Random rnd(301);
-  Options options;
-  options.env = env_;
-  options.create_if_missing = true;
-  options.db_write_buffer_size = 20480;
-  options.write_buffer_size = 20480;
-  options.max_write_buffer_number = 2;
-  options.level0_file_num_compaction_trigger = 2;
-  options.level0_slowdown_writes_trigger = 2;
-  options.level0_stop_writes_trigger = 2;
-  options.target_file_size_base = 20480;
-  options.level_compaction_dynamic_level_bytes = true;
-  options.max_bytes_for_level_base = 102400;
-  options.max_bytes_for_level_multiplier = 4;
-  options.max_background_compactions = 1;
-  options.num_levels = 5;
-  options.statistics = CreateDBStatistics();
-
-  options.compression_per_level.resize(3);
-  // No compression for L0
-  options.compression_per_level[0] = kNoCompression;
-  // No compression for the Ln whre L0 is compacted to
-  options.compression_per_level[1] = kNoCompression;
-  // Snappy compression for Ln+1
-  options.compression_per_level[2] = kSnappyCompression;
-
-  OnFileDeletionListener* listener = new OnFileDeletionListener();
-  options.listeners.emplace_back(listener);
-
-  DestroyAndReopen(options);
-
-  // Insert more than 80K. L4 should be base level. Neither L0 nor L4 should
-  // be compressed, so there shouldn't be any compression.
-  for (int i = 0; i < 20; i++) {
-    ASSERT_OK(Put(Key(keys[i]), CompressibleString(&rnd, 4000)));
-    ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
-  }
-  ASSERT_OK(Flush());
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-
-  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
-  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
-  ASSERT_EQ(NumTableFilesAtLevel(3), 0);
-  ASSERT_TRUE(NumTableFilesAtLevel(0) > 0 || NumTableFilesAtLevel(4) > 0);
-
-  // Verify there was no compression
-  auto num_block_compressed =
-      options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
-  ASSERT_EQ(num_block_compressed, 0);
-
-  // Insert 400KB and there will be some files end up in L3. According to the
-  // above compression settings for each level, there will be some compression.
-  ASSERT_OK(options.statistics->Reset());
-  ASSERT_EQ(num_block_compressed, 0);
-  for (int i = 20; i < 120; i++) {
-    ASSERT_OK(Put(Key(keys[i]), CompressibleString(&rnd, 4000)));
-    ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
-  }
-  ASSERT_OK(Flush());
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
-  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
-  ASSERT_GE(NumTableFilesAtLevel(3), 1);
-  ASSERT_GE(NumTableFilesAtLevel(4), 1);
-
-  // Verify there was compression
-  num_block_compressed =
-      options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
-  ASSERT_GT(num_block_compressed, 0);
-
-  // Make sure data in files in L3 is not compacted by removing all files
-  // in L4 and calculate number of rows
-  ASSERT_OK(dbfull()->SetOptions({
-      {"disable_auto_compactions", "true"},
-  }));
-  ColumnFamilyMetaData cf_meta;
-  db_->GetColumnFamilyMetaData(&cf_meta);
-
-  // Ensure that L1+ files are non-overlapping and together with L0 encompass
-  // full key range between smallestkey and largestkey from CF file metadata.
-  int largestkey_in_prev_level = -1;
-  int keys_found = 0;
-  for (int level = (int)cf_meta.levels.size() - 1; level >= 0; level--) {
-    int files_in_level = (int)cf_meta.levels[level].files.size();
-    int largestkey_in_prev_file = -1;
-    for (int j = 0; j < files_in_level; j++) {
-      int smallestkey = IdFromKey(cf_meta.levels[level].files[j].smallestkey);
-      int largestkey = IdFromKey(cf_meta.levels[level].files[j].largestkey);
-      int num_entries = (int)cf_meta.levels[level].files[j].num_entries;
-      ASSERT_EQ(num_entries, largestkey - smallestkey + 1);
-      keys_found += num_entries;
-      if (level > 0) {
-        if (j == 0) {
-          ASSERT_GT(smallestkey, largestkey_in_prev_level);
-        }
-        if (j > 0) {
-          ASSERT_GT(smallestkey, largestkey_in_prev_file);
-        }
-        if (j == files_in_level - 1) {
-          largestkey_in_prev_level = largestkey;
-        }
-      }
-      largestkey_in_prev_file = largestkey;
-    }
-  }
-  ASSERT_EQ(keys_found, kNKeys);
-
-  for (const auto& file : cf_meta.levels[4].files) {
-    listener->SetExpectedFileName(dbname_ + file.name);
-    const RangeOpt ranges(file.smallestkey, file.largestkey);
-    // Given verification from above, we're guaranteed that by deleting all the
-    // files in [<smallestkey>, <largestkey>] range, we're effectively deleting
-    // that very single file and nothing more.
-    EXPECT_OK(dbfull()->DeleteFilesInRanges(dbfull()->DefaultColumnFamily(),
-                                            &ranges, true /* include_end */));
-  }
-  listener->VerifyMatchedCount(cf_meta.levels[4].files.size());
-
-  int num_keys = 0;
-  std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
-  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    num_keys++;
-  }
-  ASSERT_OK(iter->status());
-
-  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
-  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
-  ASSERT_GE(NumTableFilesAtLevel(3), 1);
-  ASSERT_EQ(NumTableFilesAtLevel(4), 0);
-
-  ASSERT_GT(SizeAtLevel(0) + SizeAtLevel(3), num_keys * 4000U + num_keys * 10U);
-}
-
-TEST_F(DBTest, DynamicLevelCompressionPerLevel2) {
-  if (!Snappy_Supported() || !LZ4_Supported() || !Zlib_Supported()) {
-    return;
-  }
-  const int kNKeys = 500;
-  int keys[kNKeys];
-  for (int i = 0; i < kNKeys; i++) {
-    keys[i] = i;
-  }
-  RandomShuffle(std::begin(keys), std::end(keys));
-
-  Random rnd(301);
-  Options options;
-  options.create_if_missing = true;
-  options.db_write_buffer_size = 6000000;
-  options.write_buffer_size = 600000;
-  options.max_write_buffer_number = 2;
-  options.level0_file_num_compaction_trigger = 2;
-  options.level0_slowdown_writes_trigger = 2;
-  options.level0_stop_writes_trigger = 2;
-  options.soft_pending_compaction_bytes_limit = 1024 * 1024;
-  options.target_file_size_base = 20;
-  options.env = env_;
-  options.level_compaction_dynamic_level_bytes = true;
-  options.max_bytes_for_level_base = 200;
-  options.max_bytes_for_level_multiplier = 8;
-  options.max_background_compactions = 1;
-  options.num_levels = 5;
-  std::shared_ptr<mock::MockTableFactory> mtf(new mock::MockTableFactory);
-  options.table_factory = mtf;
-
-  options.compression_per_level.resize(3);
-  options.compression_per_level[0] = kNoCompression;
-  options.compression_per_level[1] = kLZ4Compression;
-  options.compression_per_level[2] = kZlibCompression;
-
-  DestroyAndReopen(options);
-  // When base level is L4, L4 is LZ4.
-  std::atomic<int> num_zlib(0);
-  std::atomic<int> num_lz4(0);
-  std::atomic<int> num_no(0);
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = static_cast<Compaction*>(arg);
-        if (compaction->output_level() == 4) {
-          ASSERT_TRUE(compaction->output_compression() == kLZ4Compression);
-          num_lz4.fetch_add(1);
-        }
-      });
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "FlushJob::WriteLevel0Table:output_compression", [&](void* arg) {
-        auto* compression = static_cast<CompressionType*>(arg);
-        ASSERT_TRUE(*compression == kNoCompression);
-        num_no.fetch_add(1);
-      });
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
-
-  for (int i = 0; i < 100; i++) {
-    std::string value = rnd.RandomString(200);
-    ASSERT_OK(Put(Key(keys[i]), value));
-    if (i % 25 == 24) {
-      ASSERT_OK(Flush());
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    }
-  }
-
-  ASSERT_OK(Flush());
-  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
-  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
-  ASSERT_EQ(NumTableFilesAtLevel(3), 0);
-  ASSERT_GT(NumTableFilesAtLevel(4), 0);
-  ASSERT_GT(num_no.load(), 2);
-  ASSERT_GT(num_lz4.load(), 0);
-  int prev_num_files_l4 = NumTableFilesAtLevel(4);
-
-  // After base level turn L4->L3, L3 becomes LZ4 and L4 becomes Zlib
-  num_lz4.store(0);
-  num_no.store(0);
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
-        Compaction* compaction = static_cast<Compaction*>(arg);
-        if (compaction->output_level() == 4 && compaction->start_level() == 3) {
-          ASSERT_TRUE(compaction->output_compression() == kZlibCompression);
-          num_zlib.fetch_add(1);
-        } else {
-          ASSERT_TRUE(compaction->output_compression() == kLZ4Compression);
-          num_lz4.fetch_add(1);
-        }
-      });
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "FlushJob::WriteLevel0Table:output_compression", [&](void* arg) {
-        auto* compression = static_cast<CompressionType*>(arg);
-        ASSERT_TRUE(*compression == kNoCompression);
-        num_no.fetch_add(1);
-      });
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
-
-  for (int i = 101; i < 500; i++) {
-    std::string value = rnd.RandomString(200);
-    ASSERT_OK(Put(Key(keys[i]), value));
-    if (i % 100 == 99) {
-      ASSERT_OK(Flush());
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    }
-  }
-
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
-  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
-  ASSERT_GT(NumTableFilesAtLevel(3), 0);
-  ASSERT_GT(NumTableFilesAtLevel(4), prev_num_files_l4);
-  ASSERT_GT(num_no.load(), 2);
-  ASSERT_GT(num_lz4.load(), 0);
-  ASSERT_GT(num_zlib.load(), 0);
-}
-
 TEST_F(DBTest, DynamicCompactionOptions) {
  // minimum write buffer size is enforced at 64KB
  const uint64_t k32KB = 1 << 15;
@@ -7349,27 +7078,6 @@ TEST_F(DBTest, LastWriteBufferDelay) {
 }
 #endif  // !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION)

-TEST_F(DBTest, FailWhenCompressionNotSupportedTest) {
-  CompressionType compressions[] = {kZlibCompression, kBZip2Compression,
-                                    kLZ4Compression, kLZ4HCCompression,
-                                    kXpressCompression};
-  for (auto comp : compressions) {
-    if (!CompressionTypeSupported(comp)) {
-      // not supported, we should fail the Open()
-      Options options = CurrentOptions();
-      options.compression = comp;
-      ASSERT_TRUE(!TryReopen(options).ok());
-      // Try if CreateColumnFamily also fails
-      options.compression = kNoCompression;
-      ASSERT_OK(TryReopen(options));
-      ColumnFamilyOptions cf_options(options);
-      cf_options.compression = comp;
-      ColumnFamilyHandle* handle;
-      ASSERT_TRUE(!db_->CreateColumnFamily(cf_options, "name", &handle).ok());
-    }
-  }
-}
-
 TEST_F(DBTest, CreateColumnFamilyShouldFailOnIncompatibleOptions) {
  Options options = CurrentOptions();
  options.max_open_files = 100;
@@ -452,6 +452,10 @@ class SpecialEnv : public EnvWrapper {
        return s;
      }

+      Status GetFileSize(uint64_t* s) override {
+        return target_->GetFileSize(s);
+      }
+
     private:
      std::unique_ptr<RandomAccessFile> target_;
      anon::AtomicCounter* counter_;
@@ -478,6 +482,10 @@ class SpecialEnv : public EnvWrapper {
        return target_->Prefetch(offset, n);
      }

+      Status GetFileSize(uint64_t* s) override {
+        return target_->GetFileSize(s);
+      }
+
     private:
      std::unique_ptr<RandomAccessFile> target_;
      std::atomic<uint64_t>* fail_cnt_;
@@ -2400,7 +2400,6 @@ class DataVisibilityTest : public DBBasicTestWithTimestampBase {
    }
  }
 };
-constexpr int DataVisibilityTest::kTestDataSize;

 // Application specifies timestamp but not snapshot.
 //           reader              writer
@@ -156,6 +156,14 @@ Status ExternalSstFileIngestionJob::Prepare(
        // It is unsafe to assume application had sync the file and file
        // directory before ingest the file. For integrity of RocksDB we need
        // to sync the file.
+
+        // TODO(xingbo), We should in general be moving away from production
+        // uses of ReuseWritableFile (except explicitly for WAL recycling),
+        // ReopenWritableFile, and NewRandomRWFile. We should create a
+        // FileSystem::SyncFile/FsyncFile API that by default does the
+        // re-open+sync+close combo but can (a) be reused easily, and (b) be
+        // overridden to do that more cleanly, e.g. in EncryptedEnv.
+        // https://github.com/facebook/rocksdb/issues/13741
        std::unique_ptr<FSWritableFile> file_to_sync;
        Status s = fs_->ReopenWritableFile(path_inside_db, env_options_,
                                           &file_to_sync, nullptr);
@@ -689,10 +697,9 @@ void ExternalSstFileIngestionJob::CreateEquivalentFileIngestingCompactions() {
        0 /* max_subcompaction, not applicable */,
        {} /* grandparents, not applicable */,
        std::nullopt /* earliest_snapshot */, nullptr /* snapshot_checker */,
-        false /* is manual */, "" /* trim_ts */, -1 /* score, not applicable */,
-        false /* is deletion compaction, not applicable */,
-        files_overlap_ /* l0_files_might_overlap, not applicable */,
-        CompactionReason::kExternalSstIngestion));
+        CompactionReason::kExternalSstIngestion, "" /* trim_ts */,
+        -1 /* score, not applicable */,
+        files_overlap_ /* l0_files_might_overlap, not applicable */));
  }
 }

@@ -95,7 +95,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
  uint64_t prospective_record_offset = 0;

  Slice fragment;
-  while (true) {
+  for (;;) {
    uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
    size_t drop_size = 0;
    const uint8_t record_type =
@@ -140,7 +140,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
        prospective_record_offset = physical_record_offset;
        scratch->assign(fragment.data(), fragment.size());
        in_fragmented_record = true;
-        break;
+        break;  // switch

      case kMiddleType:
      case kRecyclableMiddleType:
@@ -153,7 +153,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
          }
          scratch->append(fragment.data(), fragment.size());
        }
-        break;
+        break;  // switch

      case kLastType:
      case kRecyclableLastType:
@@ -171,7 +171,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
          first_record_read_ = true;
          return true;
        }
-        break;
+        break;  // switch

      case kSetCompressionType: {
        if (compression_type_record_read_) {
@@ -193,7 +193,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
        } else {
          InitCompression(compression_record);
        }
-        break;
+        break;  // switch
      }
      case kPredecessorWALInfoType:
      case kRecyclePredecessorWALInfoType: {
@@ -210,7 +210,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
          MaybeVerifyPredecessorWALInfo(wal_recovery_mode, fragment,
                                        recorded_predecessor_wal_info);
        }
-        break;
+        break;  // switch
      }
      case kUserDefinedTimestampSizeType:
      case kRecyclableUserDefinedTimestampSizeType: {
@@ -235,7 +235,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
            ReportCorruption(fragment.size(), s.getState());
          }
        }
-        break;
+        break;  // switch
      }

      case kBadHeader:
@@ -304,7 +304,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
          in_fragmented_record = false;
          scratch->clear();
        }
-        break;
+        break;  // switch

      case kBadRecordLen:
        if (eof_) {
@@ -337,7 +337,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
          in_fragmented_record = false;
          scratch->clear();
        }
-        break;
+        break;  // switch

      default: {
        if ((record_type & kRecordTypeSafeIgnoreMask) == 0) {
@@ -349,11 +349,11 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
        }
        in_fragmented_record = false;
        scratch->clear();
-        break;
+        break;  // switch
      }
    }
  }
-  return false;
+  // unreachable
 }

 void Reader::MaybeVerifyPredecessorWALInfo(
@@ -250,7 +250,7 @@ class MergeHelper {
  // Parallel with keys_; stores the operands
  mutable MergeContext merge_context_;

-  StopWatchNano filter_timer_;
+  StopWatchNano<> filter_timer_;
  uint64_t total_filter_time_;
  Statistics* stats_;

@@ -0,0 +1,70 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "rocksdb/db.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+using MultiScanIterator = MultiScan::MultiScanIterator;
+
+MultiScan::MultiScan(const ReadOptions& read_options,
+                     const std::vector<ScanOptions>& scan_opts, DB* db,
+                     ColumnFamilyHandle* cfh)
+    : read_options_(read_options), scan_opts_(scan_opts), db_(db), cfh_(cfh) {
+  bool slow_path = false;
+  // Setup read_options with iterate_uuper_bound based on the first scan.
+  // Subsequent scans will update and allocate a new DB iterator as necessary
+  if (scan_opts[0].range.limit) {
+    upper_bound_ = *scan_opts[0].range.limit;
+    read_options_.iterate_upper_bound = &upper_bound_;
+  } else {
+    read_options_.iterate_upper_bound = nullptr;
+  }
+  for (auto opts : scan_opts) {
+    // Check that all the ScanOptions either specify an upper bound or not. If
+    // its mixed we take the slow path which avoids calling Prepare: we have to
+    // reallocate the Iterator with updated read_options everytime we switch
+    // between upper bound or no upper bound, which complicates Prepare.
+    if (opts.range.limit.has_value() != scan_opts[0].range.limit.has_value()) {
+      slow_path = true;
+      break;
+    }
+  }
+  db_iter_.reset(db->NewIterator(read_options_, cfh));
+  if (!slow_path) {
+    db_iter_->Prepare(scan_opts);
+  }
+}
+
+MultiScanIterator& MultiScanIterator::operator++() {
+  if (idx_ >= scan_opts_.size()) {
+    throw std::logic_error("Index out of range");
+  }
+  idx_++;
+  if (idx_ < scan_opts_.size()) {
+    // Check if we need to update read_options_
+    if (scan_opts_[idx_].range.limit.has_value() !=
+        (read_options_.iterate_upper_bound != nullptr)) {
+      if (scan_opts_[idx_].range.limit) {
+        *upper_bound_ = *scan_opts_[idx_].range.limit;
+        read_options_.iterate_upper_bound = upper_bound_;
+      } else {
+        read_options_.iterate_upper_bound = nullptr;
+      }
+      db_iter_.reset(db_->NewIterator(read_options_, cfh_));
+      scan_.Reset(db_iter_.get());
+    } else if (scan_opts_[idx_].range.limit) {
+      *upper_bound_ = *scan_opts_[idx_].range.limit;
+    }
+    db_iter_->Seek(*scan_opts_[idx_].range.start);
+    status_ = db_iter_->status();
+    if (!status_.ok()) {
+      throw MultiScanException(status_);
+    }
+  }
+  return *this;
+}
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -26,6 +26,7 @@ static const std::map<PeriodicTaskType, uint64_t> kDefaultPeriodSeconds = {
    {PeriodicTaskType::kPersistStats, kInvalidPeriodSec},
    {PeriodicTaskType::kFlushInfoLog, 10},
    {PeriodicTaskType::kRecordSeqnoTime, kInvalidPeriodSec},
+    {PeriodicTaskType::kTriggerCompaction, 12 * 60 * 60}  // 12 hours
 };

 static const std::map<PeriodicTaskType, std::string> kPeriodicTaskTypeNames = {
@@ -33,16 +34,20 @@ static const std::map<PeriodicTaskType, std::string> kPeriodicTaskTypeNames = {
    {PeriodicTaskType::kPersistStats, "pst_st"},
    {PeriodicTaskType::kFlushInfoLog, "flush_info_log"},
    {PeriodicTaskType::kRecordSeqnoTime, "record_seq_time"},
+    {PeriodicTaskType::kTriggerCompaction, "trigger_compaction"},
 };

 Status PeriodicTaskScheduler::Register(PeriodicTaskType task_type,
-                                       const PeriodicTaskFunc& fn) {
-  return Register(task_type, fn, kDefaultPeriodSeconds.at(task_type));
+                                       const PeriodicTaskFunc& fn,
+                                       bool run_immediately) {
+  return Register(task_type, fn, kDefaultPeriodSeconds.at(task_type),
+                  run_immediately);
 }

 Status PeriodicTaskScheduler::Register(PeriodicTaskType task_type,
                                       const PeriodicTaskFunc& fn,
-                                       uint64_t repeat_period_seconds) {
+                                       uint64_t repeat_period_seconds,
+                                       bool run_immediately) {
  MutexLock l(&timer_mutex);
  static std::atomic<uint64_t> initial_delay(0);

@@ -65,10 +70,13 @@ Status PeriodicTaskScheduler::Register(PeriodicTaskType task_type,
  std::string unique_id =
      kPeriodicTaskTypeNames.at(task_type) + std::to_string(id_++);

-  bool succeeded = timer_->Add(
-      fn, unique_id,
-      (initial_delay.fetch_add(1) % repeat_period_seconds) * kMicrosInSecond,
-      repeat_period_seconds * kMicrosInSecond);
+  uint64_t initial_delay_micros =
+      (initial_delay.fetch_add(1) % repeat_period_seconds) * kMicrosInSecond;
+  if (!run_immediately) {
+    initial_delay_micros += repeat_period_seconds * kMicrosInSecond;
+  }
+  bool succeeded = timer_->Add(fn, unique_id, initial_delay_micros,
+                               repeat_period_seconds * kMicrosInSecond);
  if (!succeeded) {
    return Status::Aborted("Failed to register periodic task");
  }
@@ -21,6 +21,7 @@ enum class PeriodicTaskType : uint8_t {
  kPersistStats,
  kFlushInfoLog,
  kRecordSeqnoTime,
+  kTriggerCompaction,
  kMax,
 };

@@ -42,13 +43,16 @@ class PeriodicTaskScheduler {
  PeriodicTaskScheduler& operator=(PeriodicTaskScheduler&&) = delete;

  // Register a task with its default repeat period. Thread safe call.
-  Status Register(PeriodicTaskType task_type, const PeriodicTaskFunc& fn);
+  // @param run_immediately If true, the task will run soon after it's
+  // scheduled, instead of waiting for the repeat period.
+  Status Register(PeriodicTaskType task_type, const PeriodicTaskFunc& fn,
+                  bool run_immediately);

  // Register a task with specified repeat period. 0 is an invalid argument
  // (kInvalidPeriodSec). To stop the task, please use Unregister().
  // Thread safe call.
  Status Register(PeriodicTaskType task_type, const PeriodicTaskFunc& fn,
-                  uint64_t repeat_period_seconds);
+                  uint64_t repeat_period_seconds, bool run_immediately);

  // Unregister the task. Thread safe call.
  Status Unregister(PeriodicTaskType task_type);
@@ -56,6 +56,12 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
  SyncPoint::GetInstance()->SetCallBack(
      "DBImpl::FlushInfoLog:StartRunning",
      [&](void*) { flush_info_log_counter++; });
+
+  int trigger_compaction_counter = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::TriggerPeriodicCompaction:StartRunning",
+      [&](void*) { trigger_compaction_counter++; });
+
  SyncPoint::GetInstance()->EnableProcessing();

  Reopen(options);
@@ -70,7 +76,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {

  const PeriodicTaskScheduler& scheduler =
      dbfull()->TEST_GetPeriodicTaskScheduler();
-  ASSERT_EQ(3, scheduler.TEST_GetValidTaskNum());
+  ASSERT_EQ((int)PeriodicTaskType::kMax - 1, scheduler.TEST_GetValidTaskNum());

  ASSERT_EQ(1, dump_st_counter);
  ASSERT_EQ(1, pst_st_counter);
@@ -103,14 +109,14 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
  ASSERT_EQ(3, pst_st_counter);
  ASSERT_EQ(4, flush_info_log_counter);

-  ASSERT_EQ(1u, scheduler.TEST_GetValidTaskNum());
+  ASSERT_EQ(2u, scheduler.TEST_GetValidTaskNum());

  // Re-enable one task
  ASSERT_OK(dbfull()->SetDBOptions({{"stats_dump_period_sec", "5"}}));
  ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec);
  ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_persist_period_sec);

-  ASSERT_EQ(2, scheduler.TEST_GetValidTaskNum());
+  ASSERT_EQ(3, scheduler.TEST_GetValidTaskNum());

  dbfull()->TEST_WaitForPeriodicTaskRun(
      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); });
@@ -118,6 +124,16 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
  ASSERT_EQ(3, pst_st_counter);
  ASSERT_EQ(5, flush_info_log_counter);

+  ASSERT_EQ(0, trigger_compaction_counter);
+  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
+    mock_clock_->MockSleepForSeconds(static_cast<int>(12 * 60 * 60));
+  });
+  ASSERT_EQ(1, trigger_compaction_counter);
+  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
+    mock_clock_->MockSleepForSeconds(static_cast<int>(12 * 60 * 60));
+  });
+  ASSERT_EQ(2, trigger_compaction_counter);
+
  Close();
 }

@@ -150,7 +166,9 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) {
  auto dbi = static_cast_with_check<DBImpl>(dbs[kInstanceNum - 1]);

  const PeriodicTaskScheduler& scheduler = dbi->TEST_GetPeriodicTaskScheduler();
-  ASSERT_EQ(kInstanceNum * 3, scheduler.TEST_GetValidTaskNum());
+  // kRecordSeqnoTime is not registered since the feature is not enabled
+  ASSERT_EQ(kInstanceNum * ((int)PeriodicTaskType::kMax - 1),
+            scheduler.TEST_GetValidTaskNum());

  int expected_run = kInstanceNum;
  dbi->TEST_WaitForPeriodicTaskRun(
@@ -95,6 +95,8 @@ namespace ROCKSDB_NAMESPACE {

 namespace {

+using ScanOptionsMap = std::unordered_map<size_t, std::vector<ScanOptions>>;
+
 // Find File in LevelFilesBrief data structure
 // Within an index range defined by left and right
 int FindFileInRange(const InternalKeyComparator& icmp,
@@ -1100,9 +1102,47 @@ class LevelIterator final : public InternalIterator {
  }

  void Prepare(const std::vector<ScanOptions>* scan_opts) override {
+    // We assume here that scan_opts is sorted such that
+    // scan_opts[0].range.start < scan_opts[1].range.start, and non overlapping
    scan_opts_ = scan_opts;
-    if (file_iter_.iter()) {
-      file_iter_.Prepare(scan_opts_);
+    if (scan_opts_ == nullptr) {
+      return;
+    }
+
+    file_to_scan_opts_ = std::make_unique<ScanOptionsMap>();
+    for (size_t k = 0; k < scan_opts_->size(); k++) {
+      const ScanOptions& opt = scan_opts_->at(k);
+      auto start = opt.range.start;
+      auto end = opt.range.limit;
+
+      if (!start.has_value()) {
+        continue;
+      }
+
+      // We can capture this case in the future, but for now lets skip this.
+      if (!end.has_value()) {
+        continue;
+      }
+
+      InternalKey istart(start.value(), kMaxSequenceNumber, kValueTypeForSeek);
+      InternalKey iend(end.value(), 0, kValueTypeForSeekForPrev);
+
+      // TODO: This needs to be optimized, right now we iterate twice, which
+      // we dont need to. We can do this in N rather than 2N.
+      size_t fstart = FindFile(icomparator_, *flevel_, istart.Encode());
+      size_t fend = FindFile(icomparator_, *flevel_, iend.Encode());
+
+      // We need to check the relevant cases
+      // Cases:
+      // 1. [  S        E  ]
+      // 2. [  S  ]  [  E  ]
+      // 3. [  S  ] ...... [  E  ]
+      for (auto i = fstart; i <= fend; i++) {
+        if (i < flevel_->num_files) {
+          (*file_to_scan_opts_)[i].emplace_back(start.value(), end.value());
+          (*file_to_scan_opts_)[i].back().property_bag = opt.property_bag;
+        }
+      }
    }
  }

@@ -1233,6 +1273,9 @@ class LevelIterator final : public InternalIterator {
  bool to_return_sentinel_ = false;
  const std::vector<ScanOptions>* scan_opts_;

+  // Our stored scan_opts for each prefix
+  std::unique_ptr<ScanOptionsMap> file_to_scan_opts_ = nullptr;
+
  // Sets flags for if we should return the sentinel key next.
  // The condition for returning sentinel is reaching the end of current
  // file_iter_: !Valid() && status.().ok().
@@ -1494,7 +1537,18 @@ bool LevelIterator::SkipEmptyFileForward() {
    // LevelIterator::Seek*, it should also call Seek* into the corresponding
    // range tombstone iterator.
    if (file_iter_.iter() != nullptr) {
-      file_iter_.SeekToFirst();
+      // If we are doing prepared scan opts then we should seek to the values
+      // specified by the scan opts
+      if (scan_opts_ && (*file_to_scan_opts_)[file_index_].size()) {
+        const ScanOptions& opts = file_to_scan_opts_->at(file_index_).front();
+        if (opts.range.start.has_value()) {
+          InternalKey target(*opts.range.start.AsPtr(), kMaxSequenceNumber,
+                             kValueTypeForSeek);
+          file_iter_.Seek(target.Encode());
+        }
+      } else {
+        file_iter_.SeekToFirst();
+      }
      if (range_tombstone_iter_) {
        if (*range_tombstone_iter_) {
          (*range_tombstone_iter_)->SeekToFirst();
@@ -1542,10 +1596,15 @@ void LevelIterator::SetFileIterator(InternalIterator* iter) {
  }

  InternalIterator* old_iter = file_iter_.Set(iter);
-  // Since this is a new table iterator, no need to call Prepare() if
-  // scan_opts_ is null
  if (iter && scan_opts_) {
-    file_iter_.Prepare(scan_opts_);
+    if (file_to_scan_opts_.get() &&
+        file_to_scan_opts_->find(file_index_) != file_to_scan_opts_->end()) {
+      const std::vector<ScanOptions>& opts =
+          file_to_scan_opts_->at(file_index_);
+      file_iter_.Prepare(&opts);
+    } else {
+      file_iter_.Prepare(scan_opts_);
+    }
  }

  // Update the read pattern for PrefetchBuffer.
@@ -1582,6 +1641,7 @@ void LevelIterator::InitFileIterator(size_t new_file_index) {
    }
  }
 }
+
 }  // anonymous namespace

 Status Version::GetTableProperties(const ReadOptions& read_options,
@@ -2741,9 +2801,10 @@ void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
          RecordTick(db_statistics_, MULTIGET_COROUTINE_COUNT,
                     mget_tasks.size());
          // Collect all results so far
-          std::vector<Status> statuses = folly::coro::blockingWait(
-              folly::coro::collectAllRange(std::move(mget_tasks))
-                  .scheduleOn(&range->context()->executor()));
+          std::vector<Status> statuses =
+              folly::coro::blockingWait(co_withExecutor(
+                  &range->context()->executor(),
+                  folly::coro::collectAllRange(std::move(mget_tasks))));
          if (s.ok()) {
            for (Status stat : statuses) {
              if (!stat.ok()) {
@@ -3028,9 +3089,10 @@ Status Version::MultiGetAsync(
        assert(waiting.size());
        RecordTick(db_statistics_, MULTIGET_COROUTINE_COUNT, mget_tasks.size());
        // Collect all results so far
-        std::vector<Status> statuses = folly::coro::blockingWait(
-            folly::coro::collectAllRange(std::move(mget_tasks))
-                .scheduleOn(&range->context()->executor()));
+        std::vector<Status> statuses =
+            folly::coro::blockingWait(co_withExecutor(
+                &range->context()->executor(),
+                folly::coro::collectAllRange(std::move(mget_tasks))));
        mget_tasks.clear();
        if (s.ok()) {
          for (Status stat : statuses) {
@@ -3507,8 +3569,13 @@ void VersionStorageInfo::ComputeCompactionScore(
      }

      if (compaction_style_ == kCompactionStyleFIFO) {
-        score = static_cast<double>(total_size) /
-                mutable_cf_options.compaction_options_fifo.max_table_files_size;
+        auto max_table_files_size =
+            mutable_cf_options.compaction_options_fifo.max_table_files_size;
+        if (max_table_files_size == 0) {
+          // avoid divide 0
+          max_table_files_size = 1;
+        }
+        score = static_cast<double>(total_size) / max_table_files_size;
        if (score < 1 &&
            mutable_cf_options.compaction_options_fifo.allow_compaction) {
          score = std::max(
@@ -876,6 +876,10 @@ Status StressTest::CommitTxn(Transaction& txn, ThreadState* thread) {
    return Status::InvalidArgument("CommitTxn when FLAGS_use_txn is not set");
  }
  Status s = Status::OK();
+  // We don't issue write to transaction's underlying WriteBatch in stress test
+  assert(txn.GetWriteBatch()->GetWriteBatch()->Count());
+  assert(txn.GetWriteBatch()->GetWBWIOpCount() ==
+         txn.GetWriteBatch()->GetWriteBatch()->Count());
  if (FLAGS_use_optimistic_txn) {
    assert(optimistic_txn_db_);
    s = txn.Commit();
@@ -100,6 +100,10 @@ class CompositeRandomAccessFileWrapper : public RandomAccessFile {
    return target_->InvalidateCache(offset, length);
  }

+  Status GetFileSize(uint64_t* size) override {
+    return target_->GetFileSize(size);
+  }
+
 private:
  std::unique_ptr<FSRandomAccessFile> target_;
 };
@@ -186,6 +186,10 @@ class LegacyRandomAccessFileWrapper : public FSRandomAccessFile {
  IOStatus InvalidateCache(size_t offset, size_t length) override {
    return status_to_io_status(target_->InvalidateCache(offset, length));
  }
+  IOStatus GetFileSize(uint64_t* result) override {
+    auto status = target_->GetFileSize(result);
+    return status_to_io_status(std::move(status));
+  }

 private:
  std::unique_ptr<RandomAccessFile> target_;
@@ -665,17 +665,52 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
                              std::unique_ptr<FSWritableFile>* result,
                              IODebugContext* dbg) override {
    result->reset();
-    if (options.use_mmap_writes) {
+    if (options.use_mmap_reads || options.use_mmap_writes) {
      return IOStatus::InvalidArgument();
    }
+
+    size_t prefix_length = 0;
+    std::unique_ptr<BlockAccessCipherStream> stream;
+
    // Open file using underlying Env implementation
    std::unique_ptr<FSWritableFile> underlying;
-    IOStatus status =
+    auto status =
        FileSystemWrapper::ReopenWritableFile(fname, options, &underlying, dbg);
    if (!status.ok()) {
      return status;
    }
-    return CreateWritableEncryptedFile(fname, underlying, options, result, dbg);
+
+    if (underlying->GetFileSize(options.io_options, dbg) != 0) {
+      // read the cipher stream from file for non-empty file
+      std::unique_ptr<FSRandomAccessFile> underlying_file_reader;
+      status = FileSystemWrapper::NewRandomAccessFile(
+          fname, options, &underlying_file_reader, dbg);
+      if (!status.ok()) {
+        return status;
+      }
+
+      status = CreateRandomReadCipherStream(
+          fname, underlying_file_reader, options, &prefix_length, &stream, dbg);
+
+      if (!status.ok()) {
+        return status;
+      }
+    } else {
+      // create cipher stream for new or empty file
+      status = CreateWritableCipherStream(fname, underlying, options,
+                                          &prefix_length, &stream, dbg);
+      if (!status.ok()) {
+        return status;
+      }
+    }
+
+    if (stream) {
+      result->reset(new EncryptedWritableFile(
+          std::move(underlying), std::move(stream), prefix_length));
+    } else {
+      result->reset(underlying.release());
+    }
+    return status;
  }

  IOStatus ReuseWritableFile(const std::string& fname,
@@ -3467,7 +3467,6 @@ class ReadAsyncRandomAccessFile : public FSRandomAccessFileOwnerWrapper {

 private:
  ReadAsyncFS& fs_;
-  std::unique_ptr<FSRandomAccessFile> file_;
  int counter = 0;
 };

@@ -3657,6 +3656,42 @@ TEST(EnvTestMisc, StaticDestruction) {
  static_destruction_tester.activated = true;
 }

+// Test GetFileSize API
+class TestGetFileSize : public testing::Test {
+ public:
+  TestGetFileSize() { env_ = Env::Default(); }
+  Env* env_;
+};
+
+// Validate GetFileSize API returns the right value.
+// Use the default implementation from env
+TEST_F(TestGetFileSize, GetFileSize) {
+  EnvOptions soptions;
+  auto fs = env_->GetFileSystem();
+
+  std::string fname = test::PerThreadDBPath(env_, "getFileSizeTestfile");
+
+  // randomize file size
+  auto rnd = Random::GetTLSInstance();
+  auto expectedFileSize = rnd->Uniform(256 * 1024) + 1;
+  auto content = rnd->RandomBinaryString(static_cast<int>(expectedFileSize));
+
+  ASSERT_OK(CreateFile(fs.get(), fname, content, false));
+
+  std::unique_ptr<FSRandomAccessFile> file;
+  ASSERT_OK(fs->NewRandomAccessFile(fname, FileOptions(), &file, nullptr));
+
+  uint64_t fileSizeFromFileSystemAPI;
+  ASSERT_OK(
+      fs->GetFileSize(fname, IOOptions(), &fileSizeFromFileSystemAPI, nullptr));
+  ASSERT_EQ(fileSizeFromFileSystemAPI, expectedFileSize);
+
+  uint64_t fileSizeFromFsRandomAccessFileAPI;
+  ASSERT_OK(file->GetFileSize(&fileSizeFromFsRandomAccessFileAPI));
+
+  ASSERT_EQ(fileSizeFromFsRandomAccessFileAPI, expectedFileSize);
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -243,7 +243,7 @@ class PosixFileSystem : public FileSystem {
      // Use mmap when virtual address-space is plentiful.
      uint64_t size;
      IOOptions opts;
-      s = GetFileSize(fname, opts, &size, nullptr);
+      s = GetFileSizeOnOpenedFile(fd, fname, &size);
      if (s.ok()) {
        void* base = mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0);
        if (base != MAP_FAILED) {
@@ -324,7 +324,7 @@ class PosixFileSystem : public FileSystem {
    }
    uint64_t initial_file_size = 0;
    if (reopen) {
-      s = GetFileSize(fname, IOOptions(), &initial_file_size, nullptr);
+      s = GetFileSizeOnOpenedFile(fd, fname, &initial_file_size);
      if (!s.ok()) {
        close(fd);
        return s;
@@ -509,7 +509,7 @@ class PosixFileSystem : public FileSystem {
    uint64_t size;
    if (status.ok()) {
      IOOptions opts;
-      status = GetFileSize(fname, opts, &size, nullptr);
+      status = GetFileSizeOnOpenedFile(fd, fname, &size);
    }
    void* base = nullptr;
    if (status.ok()) {
@@ -671,7 +671,7 @@ class PosixFileSystem : public FileSystem {

  IOStatus GetFileSize(const std::string& fname, const IOOptions& /*opts*/,
                       uint64_t* size, IODebugContext* /*dbg*/) override {
-    struct stat sbuf;
+    struct stat sbuf {};
    if (stat(fname.c_str(), &sbuf) != 0) {
      *size = 0;
      return IOError("while stat a file for size", fname, errno);
@@ -974,6 +974,22 @@ class PosixFileSystem : public FileSystem {
 private:
  bool forceMmapOff_ = false;  // do we override Env options?

+  // This is a faster API comparing to the public method that uses stat to get
+  // file size. However this API only works on opened file.
+  IOStatus GetFileSizeOnOpenedFile(const int fd, const std::string& name,
+                                   uint64_t* size) {
+    struct stat sb {};
+    *size = 0;
+    // Get file information using fstat
+    if (fstat(fd, &sb) == -1) {
+      return IOError(
+          "while fstat a file for size with fd " + std::to_string(fd), name,
+          errno);
+    }
+    *size = sb.st_size;
+    return IOStatus::OK();
+  }
+
 #ifdef OS_LINUX
  // Get the minimum "linux system limit" (i.e, the largest I/O size that the OS
  // can issue to block devices under a directory, also known as
@@ -607,6 +607,17 @@ PosixRandomAccessFile::PosixRandomAccessFile(

 PosixRandomAccessFile::~PosixRandomAccessFile() { close(fd_); }

+IOStatus PosixRandomAccessFile::GetFileSize(uint64_t* result) {
+  struct stat sbuf {};
+  if (fstat(fd_, &sbuf) != 0) {
+    *result = 0;
+    return IOError("While fstat with fd " + std::to_string(fd_), filename_,
+                   errno);
+  }
+  *result = sbuf.st_size;
+  return IOStatus::OK();
+}
+
 IOStatus PosixRandomAccessFile::Read(uint64_t offset, size_t n,
                                     const IOOptions& /*opts*/, Slice* result,
                                     char* scratch,
@@ -1056,6 +1067,11 @@ IOStatus PosixMmapReadableFile::InvalidateCache(size_t offset, size_t length) {
 #endif
 }

+IOStatus PosixMmapReadableFile::GetFileSize(uint64_t* result) {
+  *result = length_;
+  return IOStatus::OK();
+}
+
 /*
 * PosixMmapFile
 *
@@ -352,6 +352,8 @@ class PosixRandomAccessFile : public FSRandomAccessFile {
                             void* cb_arg, void** io_handle,
                             IOHandleDeleter* del_fn,
                             IODebugContext* dbg) override;
+
+  virtual IOStatus GetFileSize(uint64_t* result) override;
 };

 class PosixWritableFile : public FSWritableFile {
@@ -437,6 +439,7 @@ class PosixMmapReadableFile : public FSRandomAccessFile {
                char* scratch, IODebugContext* dbg) const override;
  void Hint(AccessPattern pattern) override;
  IOStatus InvalidateCache(size_t offset, size_t length) override;
+  virtual IOStatus GetFileSize(uint64_t* result) override;
 };

 class PosixMmapFile : public FSWritableFile {
@@ -322,6 +322,11 @@ class MockRandomAccessFile : public FSRandomAccessFile {
    }
  }

+  IOStatus GetFileSize(uint64_t* size) override {
+    *size = file_->Size();
+    return IOStatus::OK();
+  }
+
 private:
  MemFile* file_;
  bool use_direct_io_;
@@ -108,6 +108,10 @@ class ReadaheadRandomAccessFile : public FSRandomAccessFile {

  bool use_direct_io() const override { return file_->use_direct_io(); }

+  IOStatus GetFileSize(uint64_t* result) override {
+    return file_->GetFileSize(result);
+  }
+
 private:
  // Tries to read from buffer_ n bytes starting at offset. If anything was read
  // from the cache, it sets cached_len to the number of bytes actually read,
@@ -162,7 +162,6 @@ class SstFileManagerImpl : public SstFileManager {
  void Close();

  void SetStatisticsPtr(const std::shared_ptr<Statistics>& stats) override {
-    stats_ = stats;
    delete_scheduler_.SetStatisticsPtr(stats);
  }

@@ -216,7 +215,6 @@ class SstFileManagerImpl : public SstFileManager {
  std::list<ErrorHandler*> error_handler_list_;
  // Pointer to ErrorHandler instance that is currently processing recovery
  ErrorHandler* cur_instance_;
-  std::shared_ptr<Statistics> stats_;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -50,6 +50,10 @@ struct FilterBuildingContext;
 // a number of built-in CompressionTypes that ignore any dictionary block in
 // the file; therefore they cannot accommodate dictionary compression in the
 // future without a schema change / extension.)
+//
+// Exceptions MUST NOT propagate out of overridden functions into RocksDB,
+// because RocksDB is not exception-safe. This could cause undefined behavior
+// including data loss, unreported corruption, deadlocks, and more.
 class Compressor {
 public:
  Compressor() = default;
@@ -134,15 +138,15 @@ class Compressor {
  // EXTENSIBLE or reinterpret_cast-able by custom Compressor implementations
  struct WorkingArea {};

- protected:
  // To allow for flexible re-use / reclaimation, we have explicit Get and
  // Release functions, and usually wrap in a special RAII smart pointer.
  // For example, a WorkingArea could be saved/recycled in thread-local or
  // core-local storage, or heap managed, etc., though an explicit WorkingArea
  // is only advised for repeated compression (by a single thread).
+  // ReleaseWorkingArea() in not intended to be called directly, but used by
+  // ManagedWorkingArea.
  virtual void ReleaseWorkingArea(WorkingArea*) {}

- public:
  using ManagedWorkingArea =
      ManagedPtr<WorkingArea, Compressor, &Compressor::ReleaseWorkingArea>;

@@ -221,6 +225,10 @@ class Compressor {
 // decompressed into part of a single buffer allocated to hold a block's
 // uncompressed contents along with an in-memory object representation of the
 // block (to reduce fragmentation and other overheads of separate objects).
+//
+// Exceptions MUST NOT propagate out of overridden functions into RocksDB,
+// because RocksDB is not exception-safe. This could cause undefined behavior
+// including data loss, unreported corruption, deadlocks, and more.
 class Decompressor {
 public:
  Decompressor() = default;
@@ -278,6 +286,9 @@ class Decompressor {
  // supported for this kind of Decompressor. Corruption - dictionary is
  // malformed (though many implementations will accept any data as a
  // dictionary)
+  //
+  // RocksDB promises not to call this function with an empty dictionary slice
+  // (equivalent to no dictionary).
  virtual Status MaybeCloneForDict(const Slice& /*serialized_dict*/,
                                   std::unique_ptr<Decompressor>* /*out*/) {
    return Status::NotSupported(
@@ -339,6 +350,10 @@ class Decompressor {
 //   (because that would break backward compatibility, potential quiet
 //   corruption)
 // TODO: consider adding optional streaming compression support (low priority)
+//
+// Exceptions MUST NOT propagate out of overridden functions into RocksDB,
+// because RocksDB is not exception-safe. This could cause undefined behavior
+// including data loss, unreported corruption, deadlocks, and more.
 class CompressionManager
    : public std::enable_shared_from_this<CompressionManager>,
      public Customizable {
@@ -466,6 +481,10 @@ class CompressorWrapper : public Compressor {
    return wrapped_->ObtainWorkingArea();
  }

+  // NOTE: Don't need to override ReleaseWorkingArea() here because
+  // ManagedWorkingArea takes care of calling it on the Compressor that created
+  // the WorkingArea.
+
  Status CompressBlock(Slice uncompressed_data, std::string* compressed_output,
                       CompressionType* out_compression_type,
                       ManagedWorkingArea* working_area) override {
@@ -491,6 +510,10 @@ class DecompressorWrapper : public Decompressor {
    wrapped_->ReleaseWorkingArea(wa);
  }

+  // NOTE: Don't need to override ReleaseWorkingArea() here because
+  // ManagedWorkingArea takes care of calling it on the Decompressor that
+  // created the WorkingArea.
+
  ManagedWorkingArea ObtainWorkingArea(CompressionType preferred) override {
    return wrapped_->ObtainWorkingArea(preferred);
  }
@@ -589,10 +612,14 @@ const std::shared_ptr<CompressionManager>& GetBuiltinV2CompressionManager();
 // lead to unexpected schema changes for user CompressionManagers building on
 // the built-in schema, in the unlikely/rare case of a new built-in schema.

-// Gets CompressionManager designed for the automated compression strategy.
+// Creates CompressionManager designed for the automated compression strategy.
 // This may include deciding to compress or not.
-// In future should be able to select compression algorithm based on the CPU
-// utilization and IO constraints.
+// EXPERIMENTAL
 std::shared_ptr<CompressionManagerWrapper> CreateAutoSkipCompressionManager(
-    std::shared_ptr<CompressionManager> wrapped);
+    std::shared_ptr<CompressionManager> wrapped = nullptr);
+// Creates CompressionManager designed for the CPU and IO cost aware compression
+// strategy
+// EXPERIMENTAL
+std::shared_ptr<CompressionManagerWrapper> CreateCostAwareCompressionManager(
+    std::shared_ptr<CompressionManager> wrapped = nullptr);
 }  // namespace ROCKSDB_NAMESPACE
@@ -1171,6 +1171,8 @@ extern ROCKSDB_LIBRARY_API uint64_t
 rocksdb_flushjobinfo_smallest_seqno(const rocksdb_flushjobinfo_t*);
 extern ROCKSDB_LIBRARY_API void rocksdb_reset_status(
    rocksdb_status_ptr_t* status_ptr);
+extern ROCKSDB_LIBRARY_API uint32_t
+rocksdb_flushjobinfo_flush_reason(const rocksdb_flushjobinfo_t* info);

 /* Compaction job info */
 extern ROCKSDB_LIBRARY_API void rocksdb_compactionjobinfo_status(
@@ -1221,6 +1223,9 @@ extern ROCKSDB_LIBRARY_API int rocksdb_subcompactionjobinfo_base_input_level(
    const rocksdb_subcompactionjobinfo_t*);
 extern ROCKSDB_LIBRARY_API int rocksdb_subcompactionjobinfo_output_level(
    const rocksdb_subcompactionjobinfo_t*);
+extern ROCKSDB_LIBRARY_API uint32_t
+rocksdb_subcompactionjobinfo_compaction_reason(
+    const rocksdb_subcompactionjobinfo_t* info);

 /* External file ingestion info */
 extern ROCKSDB_LIBRARY_API const char*
@@ -57,6 +57,7 @@ struct WaitForCompactOptions;
 class Env;
 class EventListener;
 class FileSystem;
+class MultiScan;
 class Replayer;
 class StatsHistoryIterator;
 class TraceReader;
@@ -1092,7 +1093,30 @@ class DB {

  // Get an iterator that scans multiple key ranges. The scan ranges should
  // be in increasing order of start key. See multi_scan_iterator.h for more
-  // details.
+  // details. For optimal performance, ensure that either all entries in
+  // scan_opts specify the range limit, or none of them do.
+  //
+  // NOTE: iterate_upper_bound in ReadOptions will be ignored. Instead, the
+  // range.limit in ScanOptions is consulted to determine the upper bound key,
+  // if specified.
+  //
+  // Example usage -
+  //  std::vector<ScanOptions> scans{{.start = Slice("bar")},
+  //                              {.start = Slice("foo")}};
+  //  std::unique_ptr<MultiScan> iter.reset(
+  //                                      db->NewMultiScan());
+  //  try {
+  //    for (auto scan : *iter) {
+  //      for (auto it : scan) {
+  //        // Do something with key - it.first
+  //        // Do something with value - it.second
+  //      }
+  //    }
+  //  } catch (MultiScanException& ex) {
+  //    // Check ex.status()
+  //  } catch (std::logic_error& ex) {
+  //    // Check ex.what()
+  //  }
  virtual std::unique_ptr<MultiScan> NewMultiScan(
      const ReadOptions& /*options*/, ColumnFamilyHandle* /*column_family*/,
      const std::vector<ScanOptions>& /*scan_opts*/) {
@@ -866,6 +866,13 @@ class RandomAccessFile {
        "RandomAccessFile::InvalidateCache not supported.");
  }

+  // The default implementation returns "not supported" so that user
+  // implementations of FSRandomAccessFile do not need to immediately implement
+  // this function.
+  virtual Status GetFileSize(uint64_t* /*result*/) {
+    return Status::NotSupported("RandomAccessFile::GetFileSize not supported.");
+  }
+
  // If you're adding methods here, remember to add them to
  // RandomAccessFileWrapper too.
 };
@@ -1750,6 +1757,9 @@ class RandomAccessFileWrapper : public RandomAccessFile {
  Status InvalidateCache(size_t offset, size_t length) override {
    return target_->InvalidateCache(offset, length);
  }
+  Status GetFileSize(uint64_t* file_size) override {
+    return target_->GetFileSize(file_size);
+  }

 private:
  RandomAccessFile* target_;
@@ -240,6 +240,15 @@ class EncryptedRandomAccessFile : public FSRandomAccessFile {
  size_t GetRequiredBufferAlignment() const override;

  IOStatus InvalidateCache(size_t offset, size_t length) override;
+
+  // Intentionally leave GetFileSize not overridden here, so that it inherits
+  // the default implementation from its parent class, which is Not Supported.
+  //
+  // As GetFileSize API is not required to be implemented yet, we use encrypted
+  // file system in unit test to validate the rest of the system could continue
+  // working with the Not Supported behavior.
+  //
+  // IOStatus GetFileSize(uint64_t* /*result*/) override;
 };

 class EncryptedWritableFile : public FSWritableFile {
@@ -1051,6 +1051,14 @@ class FSRandomAccessFile {
  // open.
  virtual Temperature GetTemperature() const { return Temperature::kUnknown; }

+  // Get the file size on an open-for-reading file without re-seeking the file's
+  // path in the filesystem. The default implementation returns "not supported"
+  // so that user implementations of FSRandomAccessFile do not need to
+  // immediately implement this function.
+  virtual IOStatus GetFileSize(uint64_t* /*result*/) {
+    return IOStatus::NotSupported("GetFileSize Not Supported");
+  }
+
  // If you're adding methods here, remember to add them to
  // RandomAccessFileWrapper too.
 };
@@ -1772,6 +1780,10 @@ class FSRandomAccessFileWrapper : public FSRandomAccessFile {
    return target_->GetTemperature();
  }

+  virtual IOStatus GetFileSize(uint64_t* result) override {
+    return target_->GetFileSize(result);
+  }
+
 private:
  std::unique_ptr<FSRandomAccessFile> guard_;
  FSRandomAccessFile* target_;
@@ -95,14 +95,18 @@ class Iterator : public IteratorBase {
    return Slice();
  }

-  // RocksDB Internal - DO NOT USE
-  // Prepare the iterator to scan the ranges specified in scan_opts. The
-  // upper bound and other table specific limits may be specified. This will
-  // typically be followed by Seeks to the start keys in the order they're
-  // specified in scan_opts. If the user does a Seek to some other target key,
-  // the iterator should disregard the scan_opts from that point onwards and
-  // behave like a normal iterator. Its the user's responsibility to again
-  // call Prepare().
+  // Prepare the iterator to scan the ranges specified in scan_opts. This
+  // includes prefetching relevant blocks from disk. The upper bound and
+  // other table specific limits should be specified for each
+  // scan for best results. If an upper bound is not specified, Prepare may
+  // skip prefetching as it cannot accurately determine how much to prefetch.
+  //
+  // Prepare should typically be followed by Seeks to the start keys in the
+  // order they're specified in scan_opts. If the user does a Seek to some
+  // other target key, the iterator should disregard the scan_opts from that
+  // point onwards and behave like a normal iterator. Its the user's
+  // responsibility to again call Prepare().
+  //
  // If Prepare() is called, it overrides the iterate_upper_bound in
  // ReadOptions
  virtual void Prepare(const std::vector<ScanOptions>& /*scan_opts*/) {}
@@ -5,6 +5,7 @@

 #pragma once

+#include "rocksdb/db.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/options.h"

@@ -72,6 +73,8 @@ class Scan {

  explicit Scan(Iterator* db_iter) : db_iter_(db_iter) {}

+  void Reset(Iterator* db_iter) { db_iter_ = db_iter; }
+
  ScanIterator begin() { return ScanIterator(db_iter_); }

  std::nullptr_t end() { return nullptr; }
@@ -149,9 +152,9 @@ class Scan {
 // A Status exception is thrown if there is an error.
 class MultiScan {
 public:
-  MultiScan(const std::vector<ScanOptions>& scan_opts,
-            std::unique_ptr<Iterator>&& db_iter)
-      : scan_opts_(scan_opts), db_iter_(std::move(db_iter)) {}
+  MultiScan(const ReadOptions& read_options,
+            const std::vector<ScanOptions>& scan_opts, DB* db,
+            ColumnFamilyHandle* cfh);

  explicit MultiScan(std::unique_ptr<Iterator>&& db_iter)
      : db_iter_(std::move(db_iter)) {}
@@ -168,9 +171,17 @@ class MultiScan {
    using difference_type = int;
    using iterator_category = std::input_iterator_tag;

-    MultiScanIterator(const std::vector<ScanOptions>& scan_opts,
-                      Iterator* db_iter)
-        : scan_opts_(scan_opts), idx_(0), db_iter_(db_iter), scan_(db_iter_) {
+    MultiScanIterator(const std::vector<ScanOptions>& scan_opts, DB* db,
+                      ColumnFamilyHandle* cfh, ReadOptions& read_options,
+                      Slice* upper_bound, std::unique_ptr<Iterator>& db_iter)
+        : scan_opts_(scan_opts),
+          db_(db),
+          cfh_(cfh),
+          read_options_(read_options),
+          upper_bound_(upper_bound),
+          idx_(0),
+          db_iter_(db_iter),
+          scan_(db_iter_.get()) {
      if (scan_opts_.empty()) {
        throw std::logic_error("Zero scans in multi-scan");
      }
@@ -181,28 +192,9 @@ class MultiScan {
      }
    }

-    explicit MultiScanIterator(const std::vector<ScanOptions>& scan_opts)
-        : scan_opts_(scan_opts),
-          idx_(scan_opts_.size()),
-          db_iter_(nullptr),
-          scan_(nullptr) {}
-
    ~MultiScanIterator() { assert(status_.ok()); }

-    MultiScanIterator& operator++() {
-      if (idx_ >= scan_opts_.size()) {
-        throw std::logic_error("Index out of range");
-      }
-      idx_++;
-      if (idx_ < scan_opts_.size()) {
-        db_iter_->Seek(*scan_opts_[idx_].range.start);
-        status_ = db_iter_->status();
-        if (!status_.ok()) {
-          throw MultiScanException(status_);
-        }
-      }
-      return *this;
-    }
+    MultiScanIterator& operator++();

    bool operator==(std::nullptr_t /*other*/) const {
      return idx_ >= scan_opts_.size();
@@ -217,20 +209,29 @@ class MultiScan {

   private:
    const std::vector<ScanOptions>& scan_opts_;
+    DB* db_;
+    ColumnFamilyHandle* cfh_;
+    ReadOptions& read_options_;
+    Slice* upper_bound_;
    size_t idx_;
-    Iterator* db_iter_;
+    std::unique_ptr<Iterator>& db_iter_;
    Status status_;
    Scan scan_;
  };

  MultiScanIterator begin() {
-    return MultiScanIterator(scan_opts_, db_iter_.get());
+    return MultiScanIterator(scan_opts_, db_, cfh_, read_options_,
+                             &upper_bound_, db_iter_);
  }

  std::nullptr_t end() { return nullptr; }

 private:
+  ReadOptions read_options_;
  const std::vector<ScanOptions> scan_opts_;
+  DB* db_;
+  ColumnFamilyHandle* cfh_;
+  Slice upper_bound_;
  std::unique_ptr<Iterator> db_iter_;
 };

@@ -57,6 +57,7 @@ class Statistics;
 class InternalKeyComparator;
 class WalFilter;
 class FileSystem;
+class UserDefinedIndexFactory;

 struct Options;
 struct DbPath;
@@ -608,6 +609,13 @@ struct DBOptions {
  // checksums. True also enters a read-only mode when a DB write fails;
  // see DB::Resume().
  //
+  // When set to true, the DB will fail to open if any SST files fail to open
+  // e.g. due to incorrect file size or corrupted footer.
+  //
+  // When set to false, when there are files corrupted, the DB will still be
+  // opened, and the healthy ones could still be accessed, while corrupted one
+  // will not
+  //
  // As most workloads value data correctness over availability, this option
  // is on by default. Note that the name of this old option is potentially
  // misleading, and other options and operations go further in proactive
@@ -1297,12 +1305,13 @@ struct DBOptions {
  // Default: false
  bool skip_stats_update_on_db_open = false;

-  // If true, then DB::Open() will not fetch and check sizes of all sst files.
-  // This may significantly speed up startup if there are many sst files,
-  // especially when using non-default Env with expensive GetFileSize().
-  // We'll still check that all required sst files exist.
-  // If paranoid_checks is false, this option is ignored, and sst files are
-  // not checked at all.
+  // This option is deprecated and marked as no-op. Kept for backward
+  // compatibility until usage is fully removed.
+  // File size check will be performed through a thread
+  // pool during DB Open, when max_open_files is set to -1.
+  // Therefore, the concern of DB Open slowness is eliminated.
+  // Note that when max_open_files is not set to -1, only a subset of files will
+  // be opened and checked during DB Open.
  //
  // Default: false
  bool skip_checking_sst_file_sizes_on_db_open = false;
@@ -2061,6 +2070,17 @@ struct ReadOptions {
  // Default: false
  bool auto_refresh_iterator_with_snapshot = false;

+  // EXPERIMENTAL
+  //
+  // Specify an alternate index to use in the SST files instead of the native
+  // block based table index. The table_factory used for the column family
+  // must support building/reading this index.
+  //
+  // Currently, only forward scans are supported. For forward scans, only Seek()
+  // is supported. SeekToFirst() is not supported. If the caller wishes to scan
+  // from start to end, the native index must be used.
+  const UserDefinedIndexFactory* table_index_factory = nullptr;
+
  // *** END options only relevant to iterators or scans ***

  // *** BEGIN options for RocksDB internal use only ***
@@ -44,6 +44,7 @@ class TableReader;
 class WritableFileWriter;
 struct ConfigOptions;
 struct EnvOptions;
+class UserDefinedIndexFactory;

 // Types of checksums to use for checking integrity of logical blocks within
 // files. All checksums currently use 32 bits of checking power (1 in 4B
@@ -492,8 +493,16 @@ struct BlockBasedTableOptions {
  // Because filters only impact performance and are not data-critical, an
  // SST file can be opened and used without filters if (a) the filter
  // policy name or schema is unrecognized, or (b) filter_policy is nullptr.
+  // See filter_policy regarding filters.
  std::shared_ptr<const FilterPolicy> filter_policy = nullptr;

+  // EXPERIMENTAL
+  //
+  // If non-nullptr, use the specified factory to build user-defined index.
+  // This allows users to define their own index format and build the index
+  // during table building.
+  std::shared_ptr<UserDefinedIndexFactory> user_defined_index_factory = nullptr;
+
  // If true, place whole keys in the filter (not just prefixes).
  // This must generally be true for gets to be efficient.
  bool whole_key_filtering = true;
@@ -69,6 +69,7 @@ class ToolHooks {
  virtual Status Open(const Options& options,
                      const blob_db::BlobDBOptions& bdb_options,
                      const std::string& dbname, blob_db::BlobDB** blob_db) = 0;
+  virtual void Exit(int status) = 0;
 };

 class DefaultHooks : public ToolHooks {
@@ -117,6 +118,8 @@ class DefaultHooks : public ToolHooks {
                      const blob_db::BlobDBOptions& bdb_options,
                      const std::string& dbname,
                      blob_db::BlobDB** blob_db) override;
+
+  virtual void Exit(int status) override { exit(status); }
 };

 extern DefaultHooks defaultHooks;
@@ -0,0 +1,142 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+//  *****************************************************************
+//  EXPERIMENTAL - subject to change while under development
+//  *****************************************************************
+
+#pragma once
+
+#include <string>
+
+#include "rocksdb/advanced_iterator.h"
+#include "rocksdb/customizable.h"
+#include "rocksdb/options.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// Prefix for user-defined index block names
+inline const std::string kUserDefinedIndexPrefix =
+    "rocksdb.user_defined_index.";
+
+// This is a public API for user-defined index builders.
+// It allows users to define their own index format and build custom
+// indexes during table building. Currently, only a monolithic index
+// block is supported (no partitioned index).
+
+// The interface for building user-defined index.
+class UserDefinedIndexBuilder {
+ public:
+  // Right now, we only support Puts. In the future, we may support merges,
+  // deletions etc.
+  enum ValueType {
+    kValue,
+    kTypeMax,
+  };
+
+  // File offset and size of the data block
+  struct BlockHandle {
+    uint64_t offset;
+    uint64_t size;
+  };
+
+  virtual ~UserDefinedIndexBuilder() = default;
+
+  // Add a new index entry to index block. The key for the new index entry
+  // should be >= last_key_in_current_block and < first_key_in_next_block.
+  // The previous index entry key and the new index entry key cover
+  // all the keys in the data block associated with the new index entry.
+  //
+  // Called before the OnKeyAdded() call for first_key_in_next_block.
+  // @last_key_in_current_block: The last key in the current data block
+  // @first_key_in_next_block: it will be nullptr if the entry being added is
+  //                           the last one in the table
+  // @block_handle: offset/size of the data block referenced by this index
+  //                entry. This should be stored along with the index entry
+  //                key
+  // @separator_scratch: a scratch buffer to back a computed separator between
+  //                     those, as needed. May be modified on each call.
+  // @return: the key or separator stored in the index, which could be
+  //          last_key_in_current_block or a computed separator backed by
+  //          separator_scratch.
+  virtual Slice AddIndexEntry(const Slice& last_key_in_current_block,
+                              const Slice* first_key_in_next_block,
+                              const BlockHandle& block_handle,
+                              std::string* separator_scratch) = 0;
+
+  // This method will be called whenever a key is added. The subclasses may
+  // override OnKeyAdded() if they need to collect additional information.
+  // The type argument indicates whether the value is a full value or partial.
+  // At the moment, only full values are supported.
+  virtual void OnKeyAdded(const Slice& /*key*/, ValueType /*type*/,
+                          const Slice& /*value*/) {}
+
+  // Finish building the index.
+  // Returns a Status and the serialized index contents.
+  // The memory backing the contents should not be freed until this builder
+  // object is destructed.
+  virtual Status Finish(Slice* index_contents) = 0;
+};
+
+// The interface for iterating the user defined index. This will be
+// instantiated and used by a scan to iterate through the index entries
+// covered by the scan.
+class UserDefinedIndexIterator {
+ public:
+  virtual ~UserDefinedIndexIterator() = default;
+
+  // Prepare the iterator for a series of scans. The iterator should use
+  // this as an opportunity to do any prefetching and buffering of results.
+  virtual void Prepare(const ScanOptions scan_opts[], size_t num_opts) = 0;
+
+  // Given the target key, position the index iterator at the index entry
+  // with the smallest key >= target. The result must be updated with the
+  // index key, and the bound_check_result. The bound_check_result should
+  // be set to kOutOfBound if no block satisfies the target key and
+  // termination criteria, kInbound if the data block is definitely fully
+  // within bounds, or kUnknown if the data block could be partially
+  // within bounds.
+  virtual Status SeekAndGetResult(const Slice& target,
+                                  IterateResult* result) = 0;
+
+  // Advance to the next index entry. The result must be populated similar
+  // to SeekAndGetResult.
+  virtual Status NextAndGetResult(IterateResult* result) = 0;
+
+  // Return the BlockHandle in the current index entry
+  virtual UserDefinedIndexBuilder::BlockHandle value() = 0;
+};
+
+// A reader interface for the user defined index
+class UserDefinedIndexReader {
+ public:
+  virtual ~UserDefinedIndexReader() = default;
+
+  // Allocate an iterator that will be used by RocksDB to perform scans
+  virtual std::unique_ptr<UserDefinedIndexIterator> NewIterator(
+      const ReadOptions& read_options) = 0;
+
+  // The memory usage of the index, including the size of the raw contents and
+  // any other heap data structures allocated by the reader
+  virtual size_t ApproximateMemoryUsage() const = 0;
+};
+
+// Factory for creating user-defined index builders.
+class UserDefinedIndexFactory : public Customizable {
+ public:
+  virtual ~UserDefinedIndexFactory() = default;
+
+  // Create a new builder for user-defined index.
+  virtual UserDefinedIndexBuilder* NewBuilder() const = 0;
+
+  // Create a new user defined index reader given the contents of the index
+  // block
+  virtual std::unique_ptr<UserDefinedIndexReader> NewReader(
+      Slice& index_block) const = 0;
+};
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -63,6 +63,8 @@ class DBWithTTL : public StackableDB {

  virtual void SetTtl(ColumnFamilyHandle* h, int32_t ttl) = 0;

+  virtual Status GetTtl(ColumnFamilyHandle* h, int32_t* ttl) = 0;
+
 protected:
  explicit DBWithTTL(DB* db) : StackableDB(db) {}
 };
@@ -396,9 +396,9 @@ struct TransactionOptions {
  // due to too many memtables.
  // Note that the ingestion relies on the transaction's underlying index,
  // (WriteBatchWithIndex), so updates that are added to the transaction
-  // without indexing (e.g. added directly to the transaction underlying
+  // without indexing (i.e. added directly to the transaction underlying
  // write batch through Transaction::GetWriteBatch()->GetWriteBatch())
-  // are not supported. They will not be applied to the DB.
+  // are not supported, and the optimization will not apply in that case.
  //
  // NOTE: since WBWI keep track of the most recent update per key, a Put
  // followed by a SingleDelete will be written to DB as a SingleDelete. This
@@ -379,6 +379,8 @@ class WriteBatchWithIndex : public WriteBatchBase {
  };
  const std::unordered_map<uint32_t, CFStat>& GetCFStats() const;

+  // The total number of operations issued into this WBWI.
+  size_t GetWBWIOpCount() const;
  bool GetOverwriteKey() const;

 private:
@@ -12,7 +12,7 @@
 // NOTE: in 'main' development branch, this should be the *next*
 // minor or major version number planned for release.
 #define ROCKSDB_MAJOR 10
-#define ROCKSDB_MINOR 4
+#define ROCKSDB_MINOR 6
 #define ROCKSDB_PATCH 0

 // Do not use these. We made the mistake of declaring macros starting with
@@ -30,6 +30,14 @@ public class NativeLibraryLoader {
  private static final String tempFilePrefix = "librocksdbjni";
  private static final String tempFileSuffix = Environment.getJniLibraryExtension();

+  /**
+   * If you set the System Property ROCKS_JAVA_DEBUG_NLL can be to true
+   * messages about attempts to load the native library will be printed
+   * to std out.
+   */
+  private static boolean DEBUG_LOADING =
+      "true".equals(System.getProperty("ROCKS_JAVA_DEBUG_NLL", "false"));
+
  /**
   * Get a reference to the NativeLibraryLoader
   *
@@ -55,7 +63,7 @@ public class NativeLibraryLoader {
   *
   * @throws java.io.IOException if a filesystem operation fails.
   */
-  @SuppressWarnings("PMD.EmptyCatchBlock")
+  @SuppressWarnings({"PMD.EmptyCatchBlock", "PMD.SystemPrintln"})
  public synchronized void loadLibrary(final String tmpDir) throws IOException {
    try {
      // try dynamic library
@@ -63,6 +71,9 @@ public class NativeLibraryLoader {
      return;
    } catch (final UnsatisfiedLinkError ule) {
      // ignore - try from static library
+      if (DEBUG_LOADING) {
+        System.out.println("Unable to load shared dynamic library: " + sharedLibraryName);
+      }
    }

    try {
@@ -71,6 +82,9 @@ public class NativeLibraryLoader {
      return;
    } catch (final UnsatisfiedLinkError ule) {
      // ignore - then try static library fallback or from jar
+      if (DEBUG_LOADING) {
+        System.out.println("Unable to load shared static library: " + jniLibraryName);
+      }
    }

    if (fallbackJniLibraryName != null) {
@@ -80,6 +94,10 @@ public class NativeLibraryLoader {
        return;
      } catch (final UnsatisfiedLinkError ule) {
        // ignore - then try from jar
+        if (DEBUG_LOADING) {
+          System.out.println(
+              "Unable to load shared static fallback library: " + fallbackJniLibraryName);
+        }
      }
    }

@@ -137,18 +155,23 @@ public class NativeLibraryLoader {
    }
  }

-  @SuppressWarnings({"PMD.UseProperClassLoader", "PMD.UseTryWithResources"})
+  @SuppressWarnings({"PMD.UseProperClassLoader", "PMD.UseTryWithResources", "PMD.SystemPrintln"})
  File loadLibraryFromJarToTemp(final String tmpDir) throws IOException {
    try (InputStream is = getClass().getClassLoader().getResourceAsStream(jniLibraryFileName)) {
      if (is != null) {
        final File temp = createTemp(tmpDir, jniLibraryFileName);
        Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
        return temp;
+      } else {
+        if (DEBUG_LOADING) {
+          System.out.println("Unable to find: " + jniLibraryFileName + " on the classpath");
+        }
      }
    }

    if (fallbackJniLibraryFileName == null) {
-      throw new RuntimeException(fallbackJniLibraryFileName + " was not found inside JAR.");
+      throw new RuntimeException(
+          jniLibraryFileName + " was not found inside JAR, and there is no fallback.");
    }

    try (InputStream is =
@@ -157,10 +180,16 @@ public class NativeLibraryLoader {
        final File temp = createTemp(tmpDir, fallbackJniLibraryFileName);
        Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
        return temp;
+      } else {
+        if (DEBUG_LOADING) {
+          System.out.println(
+              "Unable to find fallback: " + fallbackJniLibraryFileName + " on the classpath");
+        }
      }
    }

-    throw new RuntimeException(jniLibraryFileName + " was not found inside JAR.");
+    throw new RuntimeException("Neither " + jniLibraryFileName + " or " + fallbackJniLibraryFileName
+        + " were found inside the JAR, and there is no fallback.");
  }

  /**
@@ -84,13 +84,7 @@ public class RocksDB extends RocksObject {
      return;
    }

-    while (libraryLoaded.get() == LibraryState.LOADING) {
-      try {
-        Thread.sleep(10);
-      } catch(final InterruptedException e) {
-        //ignore
-      }
-    }
+    waitForLibraryToBeLoaded();
  }

  /**
@@ -146,12 +140,28 @@ public class RocksDB extends RocksObject {
      return;
    }

-    while (libraryLoaded.get() == LibraryState.LOADING) {
-      try {
-        Thread.sleep(10);
-      } catch(final InterruptedException e) {
-        //ignore
+    waitForLibraryToBeLoaded();
+  }
+
+  private static void waitForLibraryToBeLoaded() {
+    final long wait = 10; // Time to wait before re-checking if another thread loaded the library
+    final long timeout =
+        10 * 1000; // Maximum time to wait for another thread to load the library (10 seconds)
+    long waited = 0;
+    try {
+      while (libraryLoaded.get() == LibraryState.LOADING) {
+        Thread.sleep(wait);
+        waited += wait;
+
+        if (waited >= timeout) {
+          throw new RuntimeException(
+              "Exceeded timeout whilst trying to load the RocksDB shared library");
+        }
      }
+    } catch (final InterruptedException e) {
+      // restore interrupted status
+      Thread.currentThread().interrupt();
+      throw new RuntimeException("Interrupted whilst trying to load the RocksDB shared library", e);
    }
  }

@@ -72,6 +72,9 @@ std::unique_ptr<Configurable> CFOptionsAsConfigurable(
 Status StringToMap(const std::string& opts_str,
                   std::unordered_map<std::string, std::string>* opts_map);

+Status GetStringFromCompressionType(std::string* compression_str,
+                                    CompressionType compression_type);
+
 struct OptionsHelper {
  static const std::string kCFOptionsName /*= "ColumnFamilyOptions"*/;
  static const std::string kDBOptionsName /*= "DBOptions" */;
@@ -129,6 +129,8 @@ TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) {
       sizeof(CacheUsageOptions)},
      {offsetof(struct BlockBasedTableOptions, filter_policy),
       sizeof(std::shared_ptr<const FilterPolicy>)},
+      {offsetof(struct BlockBasedTableOptions, user_defined_index_factory),
+       sizeof(std::shared_ptr<UserDefinedIndexFactory>)},
  };

  // In this test, we catch a new option of BlockBasedTableOptions that is not
@@ -242,6 +242,16 @@ size_t WinMmapReadableFile::GetUniqueId(char* id, size_t max_size) const {
  return GetUniqueIdFromFile(hFile_, id, max_size);
 }

+IOStatus WinMmapReadableFile::GetFileSize(uint64_t* size) {
+  LARGE_INTEGER fileSize;
+  if (GetFileSizeEx(hFile_, &fileSize)) {
+    *size = fileSize.QuadPart;
+    return IOStatus::OK();
+  } else {
+    return IOStatus::IOError("Failed to get file size", filename_);
+  }
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 /// WinMmapFile

@@ -735,6 +745,16 @@ size_t WinRandomAccessFile::GetRequiredBufferAlignment() const {
  return GetAlignment();
 }

+IOStatus WinRandomAccessFile::GetFileSize(uint64_t* size) {
+  LARGE_INTEGER fileSize;
+  if (GetFileSizeEx(hFile_, &fileSize)) {
+    *size = fileSize.QuadPart;
+    return IOStatus::OK();
+  } else {
+    return IOStatus::IOError("Failed to get file size", filename_);
+  }
+}
+
 /////////////////////////////////////////////////////////////////////////////
 // WinWritableImpl
 //
@@ -152,6 +152,8 @@ class WinMmapReadableFile : private WinFileData, public FSRandomAccessFile {
  IOStatus InvalidateCache(size_t offset, size_t length) override;

  size_t GetUniqueId(char* id, size_t max_size) const override;
+
+  IOStatus GetFileSize(uint64_t* file_size) override;
 };

 // We preallocate and use memcpy to append new
@@ -292,6 +294,8 @@ class WinRandomAccessFile
  IOStatus InvalidateCache(size_t offset, size_t length) override;

  size_t GetRequiredBufferAlignment() const override;
+
+  IOStatus GetFileSize(uint64_t* file_size) override;
 };

 // This is a sequential write class. It has been mimicked (as others) after
@@ -80,6 +80,7 @@ LIB_SOURCES =                                                   \
  db/memtable_list.cc                                           \
  db/merge_helper.cc                                            \
  db/merge_operator.cc                                          \
+  db/multi_scan.cc						\
  db/output_validator.cc                                        \
  db/periodic_task_scheduler.cc                                 \
  db/range_del_aggregator.cc                                    \
@@ -237,13 +238,13 @@ LIB_SOURCES =                                                   \
  trace_replay/block_cache_tracer.cc                            \
  trace_replay/io_tracer.cc                                     \
  util/async_file_reader.cc					                            \
+  util/auto_tune_compressor.cc                                           \
  util/build_version.cc                                         \
  util/cleanable.cc                                             \
  util/coding.cc                                                \
  util/compaction_job_stats_impl.cc                             \
  util/comparator.cc                                            \
  util/compression.cc                                           \
-  util/auto_skip_compressor.cc                                           \
  util/compression_context_cache.cc                             \
  util/concurrent_task_limiter_impl.cc                          \
  util/crc32c.cc                                                \
@@ -1016,10 +1016,10 @@ bool IndexBlockIter::PrefixSeek(const Slice& target, uint32_t* index,
 }

 uint32_t Block::NumRestarts() const {
-  assert(size_ >= 2 * sizeof(uint32_t));
-  uint32_t block_footer = DecodeFixed32(data_ + size_ - sizeof(uint32_t));
+  assert(size() >= 2 * sizeof(uint32_t));
+  uint32_t block_footer = DecodeFixed32(data() + size() - sizeof(uint32_t));
  uint32_t num_restarts = block_footer;
-  if (size_ > kMaxBlockSizeSupportedByHashIndex) {
+  if (size() > kMaxBlockSizeSupportedByHashIndex) {
    // In BlockBuilder, we have ensured a block with HashIndex is less than
    // kMaxBlockSizeSupportedByHashIndex (64KiB).
    //
@@ -1038,12 +1038,12 @@ uint32_t Block::NumRestarts() const {
 }

 BlockBasedTableOptions::DataBlockIndexType Block::IndexType() const {
-  assert(size_ >= 2 * sizeof(uint32_t));
-  if (size_ > kMaxBlockSizeSupportedByHashIndex) {
+  assert(size() >= 2 * sizeof(uint32_t));
+  if (size() > kMaxBlockSizeSupportedByHashIndex) {
    // The check is for the same reason as that in NumRestarts()
    return BlockBasedTableOptions::kDataBlockBinarySearch;
  }
-  uint32_t block_footer = DecodeFixed32(data_ + size_ - sizeof(uint32_t));
+  uint32_t block_footer = DecodeFixed32(data() + size() - sizeof(uint32_t));
  uint32_t num_restarts = block_footer;
  BlockBasedTableOptions::DataBlockIndexType index_type;
  UnPackIndexTypeAndNumRestarts(block_footer, &index_type, &num_restarts);
@@ -1059,54 +1059,51 @@ Block::~Block() {

 Block::Block(BlockContents&& contents, size_t read_amp_bytes_per_bit,
             Statistics* statistics)
-    : contents_(std::move(contents)),
-      data_(contents_.data.data()),
-      size_(contents_.data.size()),
-      restart_offset_(0),
-      num_restarts_(0) {
+    : contents_(std::move(contents)), restart_offset_(0), num_restarts_(0) {
  TEST_SYNC_POINT("Block::Block:0");
-  if (size_ < sizeof(uint32_t)) {
-    size_ = 0;  // Error marker
+  auto& size = contents_.data.size_;
+  if (size < sizeof(uint32_t)) {
+    size = 0;  // Error marker
  } else {
    // Should only decode restart points for uncompressed blocks
    num_restarts_ = NumRestarts();
    switch (IndexType()) {
      case BlockBasedTableOptions::kDataBlockBinarySearch:
-        restart_offset_ = static_cast<uint32_t>(size_) -
+        restart_offset_ = static_cast<uint32_t>(size) -
                          (1 + num_restarts_) * sizeof(uint32_t);
-        if (restart_offset_ > size_ - sizeof(uint32_t)) {
+        if (restart_offset_ > size - sizeof(uint32_t)) {
          // The size is too small for NumRestarts() and therefore
          // restart_offset_ wrapped around.
-          size_ = 0;
+          size = 0;
        }
        break;
      case BlockBasedTableOptions::kDataBlockBinaryAndHash:
-        if (size_ < sizeof(uint32_t) /* block footer */ +
-                        sizeof(uint16_t) /* NUM_BUCK */) {
-          size_ = 0;
+        if (size < sizeof(uint32_t) /* block footer */ +
+                       sizeof(uint16_t) /* NUM_BUCK */) {
+          size = 0;
          break;
        }

        uint16_t map_offset;
        data_block_hash_index_.Initialize(
-            data_, static_cast<uint16_t>(size_ - sizeof(uint32_t)), /*chop off
-                                                                NUM_RESTARTS*/
-            &map_offset);
+            contents_.data.data(),
+            /* chop off NUM_RESTARTS */
+            static_cast<uint16_t>(size - sizeof(uint32_t)), &map_offset);

        restart_offset_ = map_offset - num_restarts_ * sizeof(uint32_t);

        if (restart_offset_ > map_offset) {
          // map_offset is too small for NumRestarts() and
          // therefore restart_offset_ wrapped around.
-          size_ = 0;
+          size = 0;
          break;
        }
        break;
      default:
-        size_ = 0;  // Error marker
+        size = 0;  // Error marker
    }
  }
-  if (read_amp_bytes_per_bit != 0 && statistics && size_ != 0) {
+  if (read_amp_bytes_per_bit != 0 && statistics && size != 0) {
    read_amp_bitmap_.reset(new BlockReadAmpBitmap(
        restart_offset_, read_amp_bytes_per_bit, statistics));
  }
@@ -1148,7 +1145,7 @@ void Block::InitializeDataBlockProtectionInfo(uint8_t protection_bytes_per_key,
      assert(!iter->status().ok() || i == num_keys * protection_bytes_per_key);
    }
    if (!iter->status().ok()) {
-      size_ = 0;  // Error marker
+      contents_.data.size_ = 0;  // Error marker
      return;
    }
    protection_bytes_per_key_ = protection_bytes_per_key;
@@ -1197,7 +1194,7 @@ void Block::InitializeIndexBlockProtectionInfo(uint8_t protection_bytes_per_key,
      assert(!iter->status().ok() || i == num_keys * protection_bytes_per_key);
    }
    if (!iter->status().ok()) {
-      size_ = 0;  // Error marker
+      contents_.data.size_ = 0;  // Error marker
      return;
    }
    protection_bytes_per_key_ = protection_bytes_per_key;
@@ -1231,7 +1228,7 @@ void Block::InitializeMetaIndexBlockProtectionInfo(
      assert(!iter->status().ok() || i == num_keys * protection_bytes_per_key);
    }
    if (!iter->status().ok()) {
-      size_ = 0;  // Error marker
+      contents_.data.size_ = 0;  // Error marker
      return;
    }
    protection_bytes_per_key_ = protection_bytes_per_key;
@@ -1240,14 +1237,14 @@ void Block::InitializeMetaIndexBlockProtectionInfo(

 MetaBlockIter* Block::NewMetaIterator(bool block_contents_pinned) {
  MetaBlockIter* iter = new MetaBlockIter();
-  if (size_ < 2 * sizeof(uint32_t)) {
+  if (size() < 2 * sizeof(uint32_t)) {
    iter->Invalidate(Status::Corruption("bad block contents"));
    return iter;
  } else if (num_restarts_ == 0) {
    // Empty block.
    iter->Invalidate(Status::OK());
  } else {
-    iter->Initialize(data_, restart_offset_, num_restarts_,
+    iter->Initialize(data(), restart_offset_, num_restarts_,
                     block_contents_pinned, protection_bytes_per_key_,
                     kv_checksum_, block_restart_interval_);
  }
@@ -1265,7 +1262,7 @@ DataBlockIter* Block::NewDataIterator(const Comparator* raw_ucmp,
  } else {
    ret_iter = new DataBlockIter;
  }
-  if (size_ < 2 * sizeof(uint32_t)) {
+  if (size() < 2 * sizeof(uint32_t)) {
    ret_iter->Invalidate(Status::Corruption("bad block contents"));
    return ret_iter;
  }
@@ -1275,7 +1272,7 @@ DataBlockIter* Block::NewDataIterator(const Comparator* raw_ucmp,
    return ret_iter;
  } else {
    ret_iter->Initialize(
-        raw_ucmp, data_, restart_offset_, num_restarts_, global_seqno,
+        raw_ucmp, data(), restart_offset_, num_restarts_, global_seqno,
        read_amp_bitmap_.get(), block_contents_pinned,
        user_defined_timestamps_persisted,
        data_block_hash_index_.Valid() ? &data_block_hash_index_ : nullptr,
@@ -1303,7 +1300,7 @@ IndexBlockIter* Block::NewIndexIterator(
  } else {
    ret_iter = new IndexBlockIter;
  }
-  if (size_ < 2 * sizeof(uint32_t)) {
+  if (size() < 2 * sizeof(uint32_t)) {
    ret_iter->Invalidate(Status::Corruption("bad block contents"));
    return ret_iter;
  }
@@ -1315,7 +1312,7 @@ IndexBlockIter* Block::NewIndexIterator(
    BlockPrefixIndex* prefix_index_ptr =
        total_order_seek ? nullptr : prefix_index;
    ret_iter->Initialize(
-        raw_ucmp, data_, restart_offset_, num_restarts_, global_seqno,
+        raw_ucmp, data(), restart_offset_, num_restarts_, global_seqno,
        prefix_index_ptr, have_first_key, key_includes_seq, value_is_full,
        block_contents_pinned, user_defined_timestamps_persisted,
        protection_bytes_per_key_, kv_checksum_, block_restart_interval_);
@@ -163,8 +163,8 @@ class Block {

  ~Block();

-  size_t size() const { return size_; }
-  const char* data() const { return data_; }
+  size_t size() const { return contents_.data.size(); }
+  const char* data() const { return contents_.data.data(); }
  // The additional memory space taken by the block data.
  size_t usable_size() const { return contents_.usable_size(); }
  uint32_t NumRestarts() const;
@@ -277,8 +277,6 @@ class Block {

 private:
  BlockContents contents_;
-  const char* data_;         // contents_.data.data()
-  size_t size_;              // contents_.data.size()
  uint32_t restart_offset_;  // Offset in data_ of restart array
  uint32_t num_restarts_;
  std::unique_ptr<BlockReadAmpBitmap> read_amp_bitmap_;
@@ -46,6 +46,7 @@
 #include "table/block_based/filter_policy_internal.h"
 #include "table/block_based/full_filter_block.h"
 #include "table/block_based/partitioned_filter_block.h"
+#include "table/block_based/user_defined_index_wrapper.h"
 #include "table/format.h"
 #include "table/meta_blocks.h"
 #include "table/table_builder.h"
@@ -408,7 +409,15 @@ struct BlockBasedTableBuilder::ParallelCompressionRep {
    }
  }

-  ~ParallelCompressionRep() { block_rep_pool.finish(); }
+  ~ParallelCompressionRep() {
+    block_rep_pool.finish();
+#ifndef NDEBUG
+    // Silence ASSERT_STATUS_CHECKED warnings
+    for (auto& block_rep : block_rep_buf) {
+      assert(block_rep.status.ok());
+    }
+#endif
+  }

  // Make a block prepared to be emitted to compression thread
  // Used in non-buffered mode
@@ -889,6 +898,27 @@ struct BlockBasedTableBuilder::Rep {
          &this->internal_prefix_transform, use_delta_encoding_for_index_values,
          table_options, ts_sz, persist_user_defined_timestamps));
    }
+
+    // If user_defined_index_factory is provided, wrap the index builder with
+    // UserDefinedIndexWrapper
+    if (table_options.user_defined_index_factory != nullptr) {
+      if (tbo.moptions.compression_opts.parallel_threads > 1 ||
+          tbo.moptions.bottommost_compression_opts.parallel_threads > 1) {
+        SetStatus(
+            Status::InvalidArgument("user_defined_index_factory not supported "
+                                    "with parallel compression"));
+      } else {
+        std::unique_ptr<UserDefinedIndexBuilder> user_defined_index_builder(
+            table_options.user_defined_index_factory->NewBuilder());
+        if (user_defined_index_builder != nullptr) {
+          index_builder.reset(new UserDefinedIndexBuilderWrapper(
+              std::string(table_options.user_defined_index_factory->Name()),
+              std::move(index_builder), std::move(user_defined_index_builder),
+              &internal_comparator, ts_sz, persist_user_defined_timestamps));
+        }
+      }
+    }
+
    if (ioptions.optimize_filters_for_hits && tbo.is_bottommost) {
      // Apply optimize_filters_for_hits setting here when applicable by
      // skipping filter generation
@@ -1192,7 +1222,7 @@ void BlockBasedTableBuilder::Add(const Slice& ikey, const Slice& value) {
      // `Finish()` once compression dictionary has been finalized.
    } else {
      if (!r->IsParallelCompressionEnabled()) {
-        r->index_builder->OnKeyAdded(ikey);
+        r->index_builder->OnKeyAdded(ikey, value);
      }
    }
    // TODO offset passed in is not accurate for parallel compression case
@@ -1634,7 +1664,7 @@ void BlockBasedTableBuilder::BGWorkWriteMaybeCompressedBlock() {
        r->filter_builder->AddWithPrevKey(key_no_ts, prev_key_no_ts);
        prev_key_no_ts = key_no_ts;
      }
-      r->index_builder->OnKeyAdded(key);
+      r->index_builder->OnKeyAdded(key, {});
    }
    if (r->filter_builder != nullptr) {
      prev_block_last_key_no_ts.assign(prev_key_no_ts.data(),
@@ -1808,7 +1838,13 @@ void BlockBasedTableBuilder::WriteIndexBlock(
  if (ok()) {
    for (const auto& item : index_blocks.meta_blocks) {
      BlockHandle block_handle;
-      WriteBlock(item.second, &block_handle, BlockType::kIndex);
+      if (item.second.first == BlockType::kIndex) {
+        WriteBlock(item.second.second, &block_handle, item.second.first);
+      } else {
+        assert(item.second.first == BlockType::kUserDefinedIndex);
+        WriteMaybeCompressedBlock(item.second.second, kNoCompression,
+                                  &block_handle, item.second.first);
+      }
      if (!ok()) {
        break;
      }
@@ -1854,8 +1890,8 @@ void BlockBasedTableBuilder::WriteIndexBlock(
    }
  }
  // If success and need to record in metaindex rather than footer...
-  if (!FormatVersionUsesIndexHandleInFooter(
-          rep_->table_options.format_version)) {
+  if (ok() && !FormatVersionUsesIndexHandleInFooter(
+                  rep_->table_options.format_version)) {
    meta_index_builder->Add(kIndexBlockName, *index_block_handle);
  }
 }
@@ -2184,7 +2220,7 @@ void BlockBasedTableBuilder::EnterUnbuffered() {
          r->filter_builder->Add(
              ExtractUserKeyAndStripTimestamp(key, r->ts_sz));
        }
-        r->index_builder->OnKeyAdded(key);
+        r->index_builder->OnKeyAdded(key, iter->value());
      }
      WriteBlock(Slice(data_block), &r->pending_handle, BlockType::kData);
      if (ok() && i + 1 < r->data_block_buffers.size()) {
@@ -690,6 +690,12 @@ Status BlockBasedTableFactory::ValidateOptions(
        "data_block_hash_table_util_ratio should be greater than 0 when "
        "data_block_index_type is set to kDataBlockBinaryAndHash");
  }
+  if (table_options_.user_defined_index_factory &&
+      (cf_opts.compression_opts.parallel_threads > 1 ||
+       cf_opts.bottommost_compression_opts.parallel_threads > 1)) {
+    return Status::InvalidArgument(
+        "user_defined_index_factory not supported with parallel compression");
+  }
  if (db_opts.unordered_write && cf_opts.max_successive_merges > 0) {
    // TODO(myabandeh): support it
    return Status::InvalidArgument(
@@ -37,6 +37,14 @@ void BlockBasedTableIterator::SeekImpl(const Slice* target,
                                       bool async_prefetch) {
  // TODO(hx235): set `seek_key_prefix_for_readahead_trimming_`
  // even when `target == nullptr` that is when `SeekToFirst()` is called
+  if (multi_scan_) {
+    if (SeekMultiScan(target)) {
+      return;
+    }
+  }
+
+  assert(!multi_scan_);
+
  if (target != nullptr && prefix_extractor_ &&
      read_options_.prefix_same_as_start) {
    const Slice& seek_user_key = ExtractUserKey(*target);
@@ -56,7 +64,7 @@ void BlockBasedTableIterator::SeekImpl(const Slice* target,
  ResetBlockCacheLookupVar();

  bool autotune_readaheadsize =
-      is_first_pass && read_options_.auto_readahead_size &&
+      read_options_.auto_readahead_size &&
      (read_options_.iterate_upper_bound || read_options_.prefix_same_as_start);

  if (autotune_readaheadsize &&
@@ -181,6 +189,7 @@ void BlockBasedTableIterator::SeekImpl(const Slice* target,
 }

 void BlockBasedTableIterator::SeekForPrev(const Slice& target) {
+  multi_scan_.reset();
  direction_ = IterDirection::kBackward;
  ResetBlockCacheLookupVar();
  is_out_of_bound_ = false;
@@ -255,6 +264,7 @@ void BlockBasedTableIterator::SeekForPrev(const Slice& target) {
 }

 void BlockBasedTableIterator::SeekToLast() {
+  multi_scan_.reset();
  direction_ = IterDirection::kBackward;
  ResetBlockCacheLookupVar();
  is_out_of_bound_ = false;
@@ -278,7 +288,9 @@ void BlockBasedTableIterator::SeekToLast() {
 }

 void BlockBasedTableIterator::Next() {
+  assert(Valid());
  if (is_at_first_key_from_index_ && !MaterializeCurrentBlock()) {
+    assert(!multi_scan_);
    return;
  }
  assert(block_iter_points_to_real_block_);
@@ -299,7 +311,9 @@ bool BlockBasedTableIterator::NextAndGetResult(IterateResult* result) {
 }

 void BlockBasedTableIterator::Prev() {
-  if (readahead_cache_lookup_ && !IsIndexAtCurr()) {
+  assert(!multi_scan_);
+  if ((readahead_cache_lookup_ && !IsIndexAtCurr()) || multi_scan_) {
+    multi_scan_.reset();
    // In case of readahead_cache_lookup_, index_iter_ has moved forward. So we
    // need to reseek the index_iter_ to point to current block by using
    // block_iter_'s key.
@@ -566,6 +580,10 @@ void BlockBasedTableIterator::FindKeyForward() {
 }

 void BlockBasedTableIterator::FindBlockForward() {
+  if (multi_scan_) {
+    FindBlockForwardInMultiScan();
+    return;
+  }
  // TODO the while loop inherits from two-level-iterator. We don't know
  // whether a block can be empty so it can be replaced by an "if".
  do {
@@ -901,4 +919,351 @@ void BlockBasedTableIterator::BlockCacheLookupForReadAheadSize(
  ResetPreviousBlockOffset();
 }

+// Note:
+// - Iterator should not be reused for multiple multiscans or mixing
+// multiscan with regular iterator usage.
+// - scan ranges should be non-overlapping, and have increasing start keys.
+// If a scan range's limit is not set, then there should only be one scan range.
+// - After Prepare(), the iterator expects Seek to be called on the start key
+// of each ScanOption in order. If any other seek is done, the optimization here
+// is aborted and fall back to vanilla iterator.
+// FIXME: DBIter and MergingIterator may
+// internally do Seek() on child iterators, e.g. due to
+// ReadOptions::max_skippable_internal_keys or reseeking into range deletion
+// end key. So these Seeks can cause iterator to fall back to normal
+// (non-prepared) iterator and ignore the optimizations done in Prepare().
+void BlockBasedTableIterator::Prepare(
+    const std::vector<ScanOptions>* scan_opts) {
+  index_iter_->Prepare(scan_opts);
+
+  assert(!multi_scan_);
+  if (multi_scan_) {
+    multi_scan_.reset();
+    return;
+  }
+  if (scan_opts == nullptr || scan_opts->empty()) {
+    return;
+  }
+  const bool has_limit = scan_opts->front().range.limit.has_value();
+  if (!has_limit && scan_opts->size() > 1) {
+    // Abort: overlapping ranges
+    return;
+  }
+
+  // Validate scan ranges to be increasing and with limit.
+  for (size_t i = 0; i < scan_opts->size(); ++i) {
+    const auto& scan_range = (*scan_opts)[i].range;
+    if (!scan_range.start.has_value()) {
+      // Abort: no start key
+      return;
+    }
+
+    // Assume for each scan range start <= limit.
+    if (scan_range.limit.has_value()) {
+      assert(user_comparator_.Compare(scan_range.start.value(),
+                                      scan_range.limit.value()) <= 0);
+    }
+
+    if (i > 0) {
+      if (!scan_range.limit.has_value()) {
+        // multiple no limit scan ranges
+        return;
+      }
+
+      const auto& last_end_key = (*scan_opts)[i - 1].range.limit.value();
+      if (user_comparator_.Compare(scan_range.start.value(), last_end_key) <
+          0) {
+        // Abort: overlapping ranges
+        return;
+      }
+    }
+  }
+
+  // Gather all relevant data block handles
+  std::vector<BlockHandle> blocks_to_prepare;
+  Status s;
+  std::vector<std::tuple<size_t, size_t>> block_ranges_per_scan;
+  for (const auto& scan_opt : *scan_opts) {
+    size_t num_blocks = 0;
+    // Current scan overlap the last block of the previous scan.
+    bool check_overlap = !blocks_to_prepare.empty();
+
+    // Scan range is specified in user key, here we seek to the minimum internal
+    // key with this user key.
+    InternalKey start_key(scan_opt.range.start.value(), kMaxSequenceNumber,
+                          kValueTypeForSeek);
+    index_iter_->Seek(start_key.Encode());
+    while (index_iter_->Valid() &&
+           (!scan_opt.range.limit.has_value() ||
+            user_comparator_.CompareWithoutTimestamp(
+                index_iter_->user_key(),
+                /*a_has_ts*/ true, *scan_opt.range.limit,
+                /*b_has_ts=*/false) <= 0)) {
+      if (check_overlap &&
+          blocks_to_prepare.back() == index_iter_->value().handle) {
+        // Skip the current block since it's already in the list
+      } else {
+        blocks_to_prepare.push_back(index_iter_->value().handle);
+      }
+      ++num_blocks;
+      index_iter_->Next();
+      check_overlap = false;
+    }
+    // Stop until index->key > limit
+    // Include the current block since it can still contain keys <= limit
+    if (index_iter_->Valid()) {
+      if (check_overlap &&
+          blocks_to_prepare.back() == index_iter_->value().handle) {
+        // Skip adding the current block since it's already in the list
+      } else {
+        blocks_to_prepare.push_back(index_iter_->value().handle);
+      }
+      ++num_blocks;
+    }
+
+    if (!index_iter_->status().ok()) {
+      // Abort: index iterator error
+      return;
+    }
+
+    block_ranges_per_scan.emplace_back(blocks_to_prepare.size() - num_blocks,
+                                       blocks_to_prepare.size());
+  }
+
+  // blocks_to_prepare has all the blocks that need to be read.
+  // Look up entries in cache and pin if exist.
+  // Store indices of blocks to read.
+  std::vector<size_t> blocks_to_read;
+  std::vector<CachableEntry<Block>> pinned_data_blocks_guard;
+  pinned_data_blocks_guard.resize(blocks_to_prepare.size());
+  for (size_t i = 0; i < blocks_to_prepare.size(); ++i) {
+    const auto& data_block_handle = blocks_to_prepare[i];
+    s = table_->LookupAndPinBlocksInCache<Block_kData>(
+        read_options_, data_block_handle,
+        &pinned_data_blocks_guard[i].As<Block_kData>());
+
+    if (!s.ok()) {
+      // Abort: block cache look up failed.
+      return;
+    }
+    if (!pinned_data_blocks_guard[i].GetValue()) {
+      // Block not in cache, will read it below.
+      blocks_to_read.emplace_back(i);
+    }
+  }
+
+  // Coalesce IOs
+  // TODO: limit prefetching size to bound memory usage.
+  if (!blocks_to_read.empty()) {
+    // Each vector correspond to blocks to read in a single read request.
+    // Each member in the vector is an index into blocks_to_prepare.
+    std::vector<std::vector<size_t>> collapsed_blocks_to_read(1);
+
+    // TODO: make this threshold configurable
+    constexpr size_t kCoalesceThreshold = 16 << 10;  // 16KB
+
+    for (const auto& block_idx : blocks_to_read) {
+      if (!collapsed_blocks_to_read.back().empty()) {
+        // Check if we can coalesce.
+        const auto& last_block =
+            blocks_to_prepare[collapsed_blocks_to_read.back().back()];
+        uint64_t last_block_end =
+            last_block.offset() +
+            BlockBasedTable::BlockSizeWithTrailer(last_block);
+        uint64_t current_start = blocks_to_prepare[block_idx].offset();
+
+        if (current_start > last_block_end + kCoalesceThreshold) {
+          // new IO
+          collapsed_blocks_to_read.emplace_back();
+        }
+      }
+      collapsed_blocks_to_read.back().emplace_back(block_idx);
+    }
+
+    // do IO
+    IOOptions io_opts;
+    s = table_->get_rep()->file->PrepareIOOptions(read_options_, io_opts);
+    if (!s.ok()) {
+      // Abort: PrepareIOOptions failed
+      return;
+    }
+
+    // Init read requests for Multi-Read
+    std::vector<FSReadRequest> read_reqs;
+    read_reqs.reserve(collapsed_blocks_to_read.size());
+    size_t total_len = 0;
+    for (const auto& blocks : collapsed_blocks_to_read) {
+      assert(blocks.size());
+      const auto& first_block = blocks_to_prepare[blocks[0]];
+      const auto& last_block = blocks_to_prepare[blocks.back()];
+
+      const auto start_offset = first_block.offset();
+      const auto end_offset = last_block.offset() +
+                              BlockBasedTable::BlockSizeWithTrailer(last_block);
+      assert(end_offset > start_offset);
+      FSReadRequest read_req;
+      read_req.offset = start_offset;
+      read_req.len = end_offset - start_offset;
+      total_len += read_req.len;
+      read_reqs.emplace_back(std::move(read_req));
+    }
+
+    // Init buffer for read
+    std::unique_ptr<char[]> buf;
+    const bool direct_io = table_->get_rep()->file->use_direct_io();
+    if (direct_io) {
+      for (auto& read_req : read_reqs) {
+        read_req.scratch = nullptr;
+      }
+    } else {
+      // TODO: optimize if FSSupportedOps::kFSBuffer is supported.
+      buf.reset(new char[total_len]);
+      size_t offset = 0;
+      for (auto& read_req : read_reqs) {
+        read_req.scratch = buf.get() + offset;
+        offset += read_req.len;
+      }
+    }
+
+    AlignedBuf aligned_buf;
+    s = table_->get_rep()->file.get()->MultiRead(
+        io_opts, read_reqs.data(), read_reqs.size(),
+        direct_io ? &aligned_buf : nullptr);
+    if (!s.ok()) {
+      return;
+    }
+    for (auto& req : read_reqs) {
+      if (!req.status.ok()) {
+        return;
+      }
+    }
+
+    // Init blocks and pin them in block cache.
+    MemoryAllocator* memory_allocator =
+        table_->get_rep()->table_options.block_cache->memory_allocator();
+    for (size_t i = 0; i < collapsed_blocks_to_read.size(); i++) {
+      const auto& blocks = collapsed_blocks_to_read[i];
+      const auto& read_req = read_reqs[i];
+      for (const auto& block_idx : blocks) {
+        const auto& block = blocks_to_prepare[block_idx];
+        const auto block_size_with_trailer =
+            BlockBasedTable::BlockSizeWithTrailer(block);
+        const auto block_offset_in_buffer = block.offset() - read_req.offset;
+
+        CacheAllocationPtr data =
+            AllocateBlock(block_size_with_trailer, memory_allocator);
+        memcpy(data.get(), read_req.result.data() + block_offset_in_buffer,
+               block_size_with_trailer);
+        BlockContents tmp_contents(std::move(data), block.size());
+
+#ifndef NDEBUG
+        tmp_contents.has_trailer =
+            table_->get_rep()->footer.GetBlockTrailerSize() > 0;
+#endif
+        assert(pinned_data_blocks_guard[block_idx].IsEmpty());
+        s = table_->CreateAndPinBlockInCache<Block_kData>(
+            read_options_, block, &tmp_contents,
+            &(pinned_data_blocks_guard[block_idx].As<Block_kData>()));
+        if (!s.ok()) {
+          // Abort: failed to create and pin block in cache
+          return;
+        }
+      }
+    }
+  }
+
+  // Successful Prepare, init related states so the iterator reads from prepared
+  // blocks
+  multi_scan_.reset(new MultiScanState(scan_opts,
+                                       std::move(pinned_data_blocks_guard),
+                                       std::move(block_ranges_per_scan)));
+  is_index_at_curr_block_ = false;
+  block_iter_points_to_real_block_ = false;
+}
+
+bool BlockBasedTableIterator::SeekMultiScan(const Slice* target) {
+  assert(multi_scan_);
+  // This is a MultiScan and Preapre() has been called.
+  //
+  // Validate seek key with scan options
+  if (multi_scan_->next_scan_idx >= multi_scan_->scan_opts->size()) {
+    multi_scan_.reset();
+  } else if (!target) {
+    // start key must be set for multi-scan
+    multi_scan_.reset();
+  } else if (user_comparator_.CompareWithoutTimestamp(
+                 ExtractUserKey(*target), /*a_has_ts=*/true,
+                 (*multi_scan_->scan_opts)[multi_scan_->next_scan_idx]
+                     .range.start.value(),
+                 /*b_has_ts=*/false) != 0) {
+    // Unexpected seek key
+    multi_scan_.reset();
+  } else {
+    auto [cur_scan_start_idx, cur_scan_end_idx] =
+        multi_scan_->block_ranges_per_scan[multi_scan_->next_scan_idx];
+    // We should have the data block already loaded
+    ++multi_scan_->next_scan_idx;
+    if (cur_scan_start_idx >= cur_scan_end_idx) {
+      is_out_of_bound_ = true;
+      assert(!Valid());
+      return true;
+    } else {
+      is_out_of_bound_ = false;
+    }
+
+    if (!block_iter_points_to_real_block_ ||
+        multi_scan_->cur_data_block_idx != cur_scan_start_idx) {
+      if (block_iter_points_to_real_block_) {
+        // Should be scan in increasing key range.
+        // All blocks before cur_data_block_idx_ are not pinned anymore.
+        assert(multi_scan_->cur_data_block_idx < cur_scan_start_idx);
+      }
+
+      ResetDataIter();
+      // Note that the block_iter_ takes ownership of the pinned data block
+      // TODO: we can delegate the clean up like with pinned_iters_mgr_ if
+      // need to pin blocks longer.
+      table_->NewDataBlockIterator<DataBlockIter>(
+          read_options_, multi_scan_->pinned_data_blocks[cur_scan_start_idx],
+          &block_iter_, Status::OK());
+    }
+    multi_scan_->cur_data_block_idx = cur_scan_start_idx;
+    block_iter_points_to_real_block_ = true;
+    block_iter_.Seek(*target);
+    FindKeyForward();
+    return true;
+  }
+
+  return false;
+}
+
+void BlockBasedTableIterator::FindBlockForwardInMultiScan() {
+  assert(multi_scan_);
+  assert(multi_scan_->next_scan_idx >= 1);
+  const auto cur_scan_end_idx = std::get<1>(
+      multi_scan_->block_ranges_per_scan[multi_scan_->next_scan_idx - 1]);
+  do {
+    if (!block_iter_.status().ok()) {
+      return;
+    }
+
+    if (multi_scan_->cur_data_block_idx + 1 >= cur_scan_end_idx) {
+      // We don't ResetDataIter() here since next scan might be reading from
+      // the same block. ResetDataIter() will free the underlying block cache
+      // handle and we don't want the block to be unpinned.
+      is_out_of_bound_ = true;
+      assert(!Valid());
+      return;
+    }
+    // Move to the next pinned data block
+    ResetDataIter();
+    ++multi_scan_->cur_data_block_idx;
+    table_->NewDataBlockIterator<DataBlockIter>(
+        read_options_,
+        multi_scan_->pinned_data_blocks[multi_scan_->cur_data_block_idx],
+        &block_iter_, Status::OK());
+    block_iter_points_to_real_block_ = true;
+    block_iter_.SeekToFirst();
+  } while (!block_iter_.Valid());
+}
 }  // namespace ROCKSDB_NAMESPACE
@@ -41,11 +41,11 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
            compaction_readahead_size,
            table_->get_rep()->table_options.initial_auto_readahead_size),
        allow_unprepared_value_(allow_unprepared_value),
-        block_iter_points_to_real_block_(false),
        check_filter_(check_filter),
        need_upper_bound_check_(need_upper_bound_check),
        async_read_in_progress_(false),
-        is_last_level_(table->IsLastLevel()) {}
+        is_last_level_(table->IsLastLevel()),
+        block_iter_points_to_real_block_(false) {}

  ~BlockBasedTableIterator() override { ClearBlockHandles(); }

@@ -69,6 +69,7 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
  Slice key() const override {
    assert(Valid());
    if (is_at_first_key_from_index_) {
+      assert(!multi_scan_);
      return index_iter_->value().first_internal_key;
    } else {
      return block_iter_.key();
@@ -141,10 +142,12 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
    // Prefix index set status to NotFound when the prefix does not exist.
    if (IsIndexAtCurr() && !index_iter_->status().ok() &&
        !index_iter_->status().IsNotFound()) {
+      assert(!multi_scan_);
      return index_iter_->status();
    } else if (block_iter_points_to_real_block_) {
      return block_iter_.status();
    } else if (async_read_in_progress_) {
+      assert(!multi_scan_);
      return Status::TryAgain("Async read in progress");
    } else {
      return Status::OK();
@@ -222,6 +225,8 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
    }
  }

+  void Prepare(const std::vector<ScanOptions>* scan_opts) override;
+
  FilePrefetchBuffer* prefetch_buffer() {
    return block_prefetcher_.prefetch_buffer();
  }
@@ -308,12 +313,20 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {

  BlockPrefetcher block_prefetcher_;

+  // It stores all the block handles that are lookuped in cache ahead when
+  // BlockCacheLookupForReadAheadSize is called. Since index_iter_ may point to
+  // different blocks when readahead_size is calculated in
+  // BlockCacheLookupForReadAheadSize, to avoid index_iter_ reseek,
+  // block_handles_ is used.
+  // `block_handles_` is lazily constructed to save CPU when it is unused
+  std::unique_ptr<std::deque<BlockHandleInfo>> block_handles_;
+
+  // The prefix of the key called with SeekImpl().
+  // This is for readahead trimming so no data blocks containing keys of a
+  // different prefix are prefetched
+  std::string seek_key_prefix_for_readahead_trimming_ = "";
+
  const bool allow_unprepared_value_;
-  // True if block_iter_ is initialized and points to the same block
-  // as index iterator.
-  bool block_iter_points_to_real_block_;
-  // See InternalIteratorBase::IsOutOfBound().
-  bool is_out_of_bound_ = false;
  // How current data block's boundary key with the next block is compared with
  // iterate upper bound.
  BlockUpperBound block_upper_bound_check_ = BlockUpperBound::kUnknown;
@@ -333,18 +346,6 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
  // size based on cache hit and miss.
  bool readahead_cache_lookup_ = false;

-  // It stores all the block handles that are lookuped in cache ahead when
-  // BlockCacheLookupForReadAheadSize is called. Since index_iter_ may point to
-  // different blocks when readahead_size is calculated in
-  // BlockCacheLookupForReadAheadSize, to avoid index_iter_ reseek,
-  // block_handles_ is used.
-  // `block_handles_` is lazily constructed to save CPU when it is unused
-  std::unique_ptr<std::deque<BlockHandleInfo>> block_handles_;
-
-  // During cache lookup to find readahead size, index_iter_ is iterated and it
-  // can point to a different block. is_index_at_curr_block_ keeps track of
-  // that.
-  bool is_index_at_curr_block_ = true;
  bool is_index_out_of_bound_ = false;

  // Used in case of auto_readahead_size to disable the block_cache lookup if
@@ -353,10 +354,48 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
  // is used to disable the lookup.
  IterDirection direction_ = IterDirection::kForward;

-  // The prefix of the key called with SeekImpl().
-  // This is for readahead trimming so no data blocks containing keys of a
-  // different prefix are prefetched
-  std::string seek_key_prefix_for_readahead_trimming_ = "";
+  //*** BEGIN States used by both regular scan and multiscan
+
+  // True if block_iter_ is initialized and points to the same block
+  // as index iterator.
+  bool block_iter_points_to_real_block_;
+  // See InternalIteratorBase::IsOutOfBound().
+  bool is_out_of_bound_ = false;
+  // During cache lookup to find readahead size, index_iter_ is iterated and it
+  // can point to a different block.
+  // If Prepare() is called, index_iter_ is used to prefetch data blocks for the
+  // multiscan, so is_index_at_curr_block_ will be false.
+  // Whether index is expected to match the current data_block_iter_.
+  bool is_index_at_curr_block_ = true;
+
+  // *** END States used by both regular scan and multiscan
+
+  // *** BEGIN MultiScan related states ***
+  struct MultiScanState {
+    // bool prepared_ = false;
+    const std::vector<ScanOptions>* scan_opts;
+    std::vector<CachableEntry<Block>> pinned_data_blocks;
+
+    // Indicies into multiscan_pinned_data_blocks_ for data blocks that are
+    // relevant for each scan range.
+    // inclusive start, exclusive end
+    std::vector<std::tuple<size_t, size_t>> block_ranges_per_scan;
+    size_t next_scan_idx;
+    size_t cur_data_block_idx;
+
+    MultiScanState(
+        const std::vector<ScanOptions>* _scan_opts,
+        std::vector<CachableEntry<Block>>&& _pinned_data_blocks,
+        std::vector<std::tuple<size_t, size_t>>&& _block_ranges_per_scan)
+        : scan_opts(_scan_opts),
+          pinned_data_blocks(std::move(_pinned_data_blocks)),
+          block_ranges_per_scan(std::move(_block_ranges_per_scan)),
+          next_scan_idx(0),
+          cur_data_block_idx(0) {}
+  };
+
+  std::unique_ptr<MultiScanState> multi_scan_;
+  // *** END MultiScan related APIs and states ***

  void SeekSecondPass(const Slice* target);

@@ -472,5 +511,12 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
                                    uint64_t& end_updated_offset,
                                    size_t& prev_handles_size);
  // *** END APIs relevant to auto tuning of readahead_size ***
+
+  // *** BEGIN APIs relevant to multiscan ***
+  // Returns true iff seek is successful.
+  bool SeekMultiScan(const Slice* target);
+
+  void FindBlockForwardInMultiScan();
+  // *** END APIs relevant to multiscan ***
 };
 }  // namespace ROCKSDB_NAMESPACE
@@ -46,6 +46,7 @@
 #include "rocksdb/table.h"
 #include "rocksdb/table_properties.h"
 #include "rocksdb/trace_record.h"
+#include "rocksdb/user_defined_index.h"
 #include "table/block_based/binary_search_index_reader.h"
 #include "table/block_based/block.h"
 #include "table/block_based/block_based_table_factory.h"
@@ -58,6 +59,7 @@
 #include "table/block_based/hash_index_reader.h"
 #include "table/block_based/partitioned_filter_block.h"
 #include "table/block_based/partitioned_index_reader.h"
+#include "table/block_based/user_defined_index_wrapper.h"
 #include "table/block_fetcher.h"
 #include "table/format.h"
 #include "table/get_context.h"
@@ -104,7 +106,11 @@ CacheAllocationPtr CopyBufferToHeap(MemoryAllocator* allocator, Slice& buf) {
      bool use_block_cache_for_lookup) const;                                  \
  template Status BlockBasedTable::LookupAndPinBlocksInCache<T>(               \
      const ReadOptions& ro, const BlockHandle& handle,                        \
-      CachableEntry<T>* out_parsed_block) const;
+      CachableEntry<T>* out_parsed_block) const;                               \
+  template Status BlockBasedTable::CreateAndPinBlockInCache<T>(                \
+      const ReadOptions& ro, const BlockHandle& handle,                        \
+      BlockContents* block_contents, CachableEntry<T>* out_parsed_block)       \
+      const;

 INSTANTIATE_BLOCKLIKE_TEMPLATES(ParsedFullFilterBlock);
 INSTANTIATE_BLOCKLIKE_TEMPLATES(DecompressorDict);
@@ -113,6 +119,7 @@ INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kIndex);
 INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kFilterPartitionIndex);
 INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kRangeDeletion);
 INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kMetaIndex);
+INSTANTIATE_BLOCKLIKE_TEMPLATES(Block_kUserDefinedIndex);

 }  // namespace ROCKSDB_NAMESPACE

@@ -1318,6 +1325,34 @@ Status BlockBasedTable::PrefetchIndexAndFilterBlocks(
  if (!s.ok()) {
    return s;
  }
+  if (table_options.user_defined_index_factory != nullptr) {
+    std::string udi_name(table_options.user_defined_index_factory->Name());
+    BlockHandle udi_block_handle;
+
+    // Should we use FindOptionalMetaBlock here?
+    s = FindMetaBlock(meta_iter, kUserDefinedIndexPrefix + udi_name,
+                      &udi_block_handle);
+    if (!s.ok()) {
+      return s;
+    }
+    // Read the block, and allocate on heap or pin in cache. The UDI block is
+    // not compressed. RetrieveBlock will verify the checksum.
+    s = RetrieveBlock(prefetch_buffer, ro, udi_block_handle,
+                      rep_->decompressor.get(), &rep_->udi_block,
+                      /*get_context=*/nullptr, lookup_context,
+                      /*for_compaction=*/false, use_cache, /*async_read=*/false,
+                      /*use_block_cache_for_lookup=*/false);
+    if (!s.ok()) {
+      return s;
+    }
+    assert(!rep_->udi_block.IsEmpty());
+
+    std::unique_ptr<UserDefinedIndexReader> udi_reader =
+        table_options.user_defined_index_factory->NewReader(
+            rep_->udi_block.GetValue()->data);
+    index_reader = std::make_unique<UserDefinedIndexReaderWrapper>(
+        udi_name, std::move(index_reader), std::move(udi_reader));
+  }

  rep_->index_reader = std::move(index_reader);

@@ -1704,6 +1739,17 @@ Status BlockBasedTable::LookupAndPinBlocksInCache(
  return s;
 }

+template <typename TBlocklike>
+Status BlockBasedTable::CreateAndPinBlockInCache(
+    const ReadOptions& ro, const BlockHandle& handle, BlockContents* contents,
+    CachableEntry<TBlocklike>* out_parsed_block) const {
+  return MaybeReadBlockAndLoadToCache(
+      nullptr, ro, handle, rep_->decompressor.get(),
+      /*for_compaction=*/false, out_parsed_block, nullptr, nullptr, contents,
+      /*async_read=*/false,
+      /*use_block_cache_for_lookup=*/true);
+}
+
 // If contents is nullptr, this function looks up the block caches for the
 // data block referenced by handle, and read the block from disk if necessary.
 // If contents is non-null, it skips the cache lookup and disk read, since
@@ -1765,8 +1811,7 @@ BlockBasedTable::MaybeReadBlockAndLoadToCache(
        ro.fill_cache) {
      Statistics* statistics = rep_->ioptions.stats;
      const bool maybe_compressed =
-          TBlocklike::kBlockType != BlockType::kFilter &&
-          TBlocklike::kBlockType != BlockType::kCompressionDictionary &&
+          BlockTypeMaybeCompressed(TBlocklike::kBlockType) &&
          rep_->decompressor;
      // This flag, if true, tells BlockFetcher to return the uncompressed
      // block when ReadBlockContents() is called.
@@ -1910,6 +1955,7 @@ BlockBasedTable::SaveLookupContextOrTraceRecord(
      trace_block_type = TraceType::kBlockTraceRangeDeletionBlock;
      break;
    case BlockType::kIndex:
+    case BlockType::kUserDefinedIndex:
      trace_block_type = TraceType::kBlockTraceIndexBlock;
      break;
    default:
@@ -2002,9 +2048,7 @@ WithBlocklikeCheck<Status, TBlocklike> BlockBasedTable::RetrieveBlock(
  }

  const bool maybe_compressed =
-      TBlocklike::kBlockType != BlockType::kFilter &&
-      TBlocklike::kBlockType != BlockType::kCompressionDictionary &&
-      rep_->decompressor;
+      BlockTypeMaybeCompressed(TBlocklike::kBlockType) && rep_->decompressor;
  std::unique_ptr<TBlocklike> block;

  {
@@ -2747,6 +2791,10 @@ BlockType BlockBasedTable::GetBlockTypeForMetaBlockByName(
    return BlockType::kIndex;
  }

+  if (meta_block_name.starts_with(kUserDefinedIndexPrefix)) {
+    return BlockType::kUserDefinedIndex;
+  }
+
  if (meta_block_name.starts_with(kObsoleteFilterBlockPrefix)) {
    // Obsolete but possible in old files
    return BlockType::kInvalid;
@@ -228,11 +228,15 @@ class BlockBasedTable : public TableReader {

    // Create an iterator for index access. If iter is null, then a new object
    // is created on the heap, and the callee will have the ownership.
-    // If a non-null iter is passed in, it will be used, and the returned value
-    // is either the same as iter or a new on-heap object that
-    // wraps the passed iter. In the latter case the return value points
-    // to a different object then iter, and the callee has the ownership of the
-    // returned object.
+    // If a non-null iter is passed in, it may be used, and the returned value
+    // is either the same as iter or a new on-heap object.
+    // In the latter case the return value points to a different object then
+    // iter, and the callee has the ownership of the returned object.
+    //
+    // Under all circumstances, the caller MUST use the returned iterator
+    // for further operations. If the returned iterator != iter, then the
+    // caller MUST ensure that iter stays in scope until the returned
+    // iterator is destroyed.
    virtual InternalIteratorBase<IndexValue>* NewIterator(
        const ReadOptions& read_options, bool disable_prefix_seek,
        IndexBlockIter* iter, GetContext* get_context,
@@ -295,11 +299,21 @@ class BlockBasedTable : public TableReader {
  Status GetKVPairsFromDataBlocks(const ReadOptions& read_options,
                                  std::vector<KVPairBlock>* kv_pair_blocks);

+  // Look up the block cache for the specified block.
+  // out_parsed_block is set to nullptr if the block is not found in the cache.
  template <typename TBlocklike>
  Status LookupAndPinBlocksInCache(
      const ReadOptions& ro, const BlockHandle& handle,
      CachableEntry<TBlocklike>* out_parsed_block) const;

+  // Create the block given in `block_contents` and insert it into block cache.
+  // `out_parsed_block` points to the inserted block if successful.
+  template <typename TBlocklike>
+  Status CreateAndPinBlockInCache(
+      const ReadOptions& ro, const BlockHandle& handle,
+      BlockContents* block_contents,
+      CachableEntry<TBlocklike>* out_parsed_block) const;
+
  struct Rep;

  Rep* get_rep() { return rep_; }
@@ -544,6 +558,12 @@ class BlockBasedTable : public TableReader {

  bool TimestampMayMatch(const ReadOptions& read_options) const;

+  bool BlockTypeMaybeCompressed(BlockType type) const {
+    return type != BlockType::kFilter &&
+           type != BlockType::kCompressionDictionary &&
+           type != BlockType::kUserDefinedIndex;
+  }
+
  // A cumulative data block file read in MultiGet lower than this size will
  // use a stack buffer
  static constexpr size_t kMultiGetReadStackBufSize = 8192;
@@ -689,6 +709,8 @@ struct BlockBasedTable::Rep {
  std::unique_ptr<CacheReservationManager::CacheReservationHandle>
      table_reader_cache_res_handle = nullptr;

+  CachableEntry<Block_kUserDefinedIndex> udi_block;
+
  SequenceNumber get_global_seqno(BlockType block_type) const {
    return (block_type == BlockType::kFilterPartitionIndex ||
            block_type == BlockType::kCompressionDictionary)
@@ -173,7 +173,7 @@ class BlockBasedTableReaderBaseTest : public testing::Test {
        0 /* _tail_size */, user_defined_timestamps_persisted);

    std::unique_ptr<RandomAccessFileReader> file;
-    NewFileReader(table_name, foptions, &file);
+    NewFileReader(table_name, foptions, &file, ioptions.statistics.get());

    uint64_t file_size = 0;
    ASSERT_OK(env_->GetFileSize(Path(table_name), &file_size));
@@ -222,12 +222,15 @@ class BlockBasedTableReaderBaseTest : public testing::Test {
  }

  void NewFileReader(const std::string& filename, const FileOptions& opt,
-                     std::unique_ptr<RandomAccessFileReader>* reader) {
+                     std::unique_ptr<RandomAccessFileReader>* reader,
+                     Statistics* stats = nullptr) {
    std::string path = Path(filename);
    std::unique_ptr<FSRandomAccessFile> f;
    ASSERT_OK(fs_->NewRandomAccessFile(path, opt, &f, nullptr));
    reader->reset(new RandomAccessFileReader(std::move(f), path,
-                                             env_->GetSystemClock().get()));
+                                             env_->GetSystemClock().get(),
+                                             /*io_tracer=*/nullptr,
+                                             /*stats=*/stats));
  }
 };

@@ -990,6 +993,167 @@ TEST_P(BlockBasedTableReaderTestVerifyChecksum, ChecksumMismatch) {
  ASSERT_EQ(s.code(), Status::kCorruption);
 }

+TEST_P(BlockBasedTableReaderTest, MultiScanPrepare) {
+  Options options;
+  options.statistics = CreateDBStatistics();
+  ReadOptions read_opts;
+  size_t ts_sz = options.comparator->timestamp_size();
+  std::vector<std::pair<std::string, std::string>> kv =
+      BlockBasedTableReaderBaseTest::GenerateKVMap(
+          100 /* num_block */,
+          true /* mixed_with_human_readable_string_value */, ts_sz);
+
+  std::string table_name = "BlockBasedTableReaderTest_NewIterator" +
+                           CompressionTypeToString(compression_type_);
+
+  ImmutableOptions ioptions(options);
+  CreateTable(table_name, ioptions, compression_type_, kv,
+              compression_parallel_threads_, compression_dict_bytes_);
+
+  std::unique_ptr<BlockBasedTable> table;
+  FileOptions foptions;
+  foptions.use_direct_reads = true;
+  InternalKeyComparator comparator(options.comparator);
+  NewBlockBasedTableReader(foptions, ioptions, comparator, table_name, &table,
+                           true /* bool prefetch_index_and_filter_in_cache */,
+                           nullptr /* status */, persist_udt_);
+
+  std::unique_ptr<InternalIterator> iter;
+  iter.reset(table->NewIterator(
+      read_opts, options_.prefix_extractor.get(), /*arena=*/nullptr,
+      /*skip_filters=*/false, TableReaderCaller::kUncategorized));
+
+  // Should coalesce into a single I/O
+  std::vector<ScanOptions> scan_options(
+      {ScanOptions(ExtractUserKey(kv[0].first),
+                   ExtractUserKey(kv[kEntriesPerBlock].first)),
+       ScanOptions(ExtractUserKey(kv[2 * kEntriesPerBlock].first),
+                   ExtractUserKey(kv[3 * kEntriesPerBlock].first))});
+
+  auto read_count_before =
+      options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT);
+  iter->Prepare(&scan_options);
+  auto read_count_after =
+      options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT);
+  ASSERT_EQ(read_count_before + 1, read_count_after);
+  iter->Seek(kv[0].first);
+  for (size_t i = 0; i < kEntriesPerBlock + 1; ++i) {
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key().ToString(), kv[i].first);
+    iter->Next();
+  }
+  // Iter may still be valid after scan range. Upper layer (DBIter) handles
+  // exact upper bound checking. So we don't check !iter->Valid() here.
+  ASSERT_OK(iter->status());
+  iter->Seek(kv[2 * kEntriesPerBlock].first);
+  for (size_t i = 2 * kEntriesPerBlock; i < 3 * kEntriesPerBlock; ++i) {
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key().ToString(), kv[i].first);
+    iter->Next();
+  }
+  ASSERT_OK(iter->status());
+
+  iter.reset(table->NewIterator(
+      read_opts, options_.prefix_extractor.get(), /*arena=*/nullptr,
+      /*skip_filters=*/false, TableReaderCaller::kUncategorized));
+  // No IO coalesce, should do MultiRead with 2 read requests.
+  scan_options = {ScanOptions(ExtractUserKey(kv[70 * kEntriesPerBlock].first),
+                              ExtractUserKey(kv[75 * kEntriesPerBlock].first)),
+                  ScanOptions(ExtractUserKey(kv[90 * kEntriesPerBlock].first),
+                              ExtractUserKey(kv[95 * kEntriesPerBlock].first))};
+  read_count_before =
+      options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT);
+  iter->Prepare(&scan_options);
+  read_count_after =
+      options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT);
+  ASSERT_EQ(read_count_before + 2, read_count_after);
+
+  iter->Seek(kv[70 * kEntriesPerBlock].first);
+  for (size_t i = 70 * kEntriesPerBlock; i < 75 * kEntriesPerBlock; ++i) {
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key().ToString(), kv[i].first);
+    iter->Next();
+  }
+  ASSERT_OK(iter->status());
+  iter->Seek(kv[90 * kEntriesPerBlock].first);
+  for (size_t i = 90 * kEntriesPerBlock; i < 95 * kEntriesPerBlock; ++i) {
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key().ToString(), kv[i].first);
+    iter->Next();
+  }
+  ASSERT_OK(iter->status());
+
+  iter.reset(table->NewIterator(
+      read_opts, options_.prefix_extractor.get(), /*arena=*/nullptr,
+      /*skip_filters=*/false, TableReaderCaller::kUncategorized));
+  // Should do two I/Os since blocks 80-81 and 90-95 are already in block cache,
+  // reads from blocks 50-79 and 82-.. are co
+  scan_options = {ScanOptions(ExtractUserKey(kv[50 * kEntriesPerBlock].first))};
+  read_count_before =
+      options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT);
+  iter->Prepare(&scan_options);
+  read_count_after =
+      options.statistics->getTickerCount(NON_LAST_LEVEL_READ_COUNT);
+  ASSERT_EQ(read_count_before + 3, read_count_after);
+  iter->Seek(kv[50 * kEntriesPerBlock].first);
+  for (size_t i = 50 * kEntriesPerBlock; i < 100 * kEntriesPerBlock; ++i) {
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key().ToString(), kv[i].first);
+    iter->Next();
+  }
+  ASSERT_FALSE(iter->Valid());
+  ASSERT_OK(iter->status());
+
+  // Check cases when Seek key does not match start key in ScanOptions
+  iter.reset(table->NewIterator(
+      read_opts, options_.prefix_extractor.get(), /*arena=*/nullptr,
+      /*skip_filters=*/false, TableReaderCaller::kUncategorized));
+  scan_options = {ScanOptions(ExtractUserKey(kv[10 * kEntriesPerBlock].first),
+                              ExtractUserKey(kv[20 * kEntriesPerBlock].first)),
+                  ScanOptions(ExtractUserKey(kv[30 * kEntriesPerBlock].first),
+                              ExtractUserKey(kv[40 * kEntriesPerBlock].first))};
+  iter->Prepare(&scan_options);
+  // Match start key
+  iter->Seek(kv[10 * kEntriesPerBlock].first);
+  for (size_t i = 10 * kEntriesPerBlock; i < 20 * kEntriesPerBlock; ++i) {
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key().ToString(), kv[i].first);
+    iter->Next();
+  }
+  ASSERT_OK(iter->status());
+  // Does not match start key of the second ScanOptions.
+  iter->Seek(kv[50 * kEntriesPerBlock + 1].first);
+  for (size_t i = 50 * kEntriesPerBlock + 1; i < 100 * kEntriesPerBlock; ++i) {
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key().ToString(), kv[i].first);
+    iter->Next();
+  }
+  ASSERT_FALSE(iter->Valid());
+  ASSERT_OK(iter->status());
+
+  iter.reset(table->NewIterator(
+      read_opts, options_.prefix_extractor.get(), /*arena=*/nullptr,
+      /*skip_filters=*/false, TableReaderCaller::kUncategorized));
+  scan_options = {ScanOptions(ExtractUserKey(kv[10 * kEntriesPerBlock].first)),
+                  ScanOptions(ExtractUserKey(kv[11 * kEntriesPerBlock].first))};
+  iter->Prepare(&scan_options);
+  // Does not match the first ScanOptions.
+  iter->SeekToFirst();
+  for (size_t i = 0; i < kEntriesPerBlock; ++i) {
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key().ToString(), kv[i].first);
+    iter->Next();
+  }
+  ASSERT_OK(iter->status());
+  iter->Seek(kv[10 * kEntriesPerBlock].first);
+  for (size_t i = 10 * kEntriesPerBlock; i < 12 * kEntriesPerBlock; ++i) {
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key().ToString(), kv[i].first);
+    iter->Next();
+  }
+  ASSERT_OK(iter->status());
+}
+
 // Param 1: compression type
 // Param 2: whether to use direct reads
 // Param 3: Block Based Table Index type, partitioned filters are also enabled
@@ -21,15 +21,19 @@
 // An entry for a particular key-value pair has the form:
 //     shared_bytes: varint32
 //     unshared_bytes: varint32
-//     value_length: varint32
+//     value_length: varint32 (NOTE1)
 //     key_delta: char[unshared_bytes]
 //     value: char[value_length]
-// shared_bytes == 0 for restart points.
+// shared_bytes == 0 (explicitly stored) for restart points.
 //
 // The trailer of the block has the form:
 //     restarts: uint32[num_restarts]
 //     num_restarts: uint32
 // restarts[i] contains the offset within the block of the ith restart point.
+//
+// NOTE1: omitted for format_version >= 4 index blocks, because the value is
+// composed of one (shared_bytes > 0) or two (shared_bytes == 0) varints, whose
+// length is self-describing.

 #include "table/block_based/block_builder.h"

@@ -46,6 +46,12 @@ void BlockCreateContext::Create(std::unique_ptr<Block_kMetaIndex>* parsed_out,
      protection_bytes_per_key);
 }

+void BlockCreateContext::Create(
+    std::unique_ptr<Block_kUserDefinedIndex>* parsed_out,
+    BlockContents&& block) {
+  parsed_out->reset(new Block_kUserDefinedIndex(std::move(block)));
+}
+
 void BlockCreateContext::Create(
    std::unique_ptr<ParsedFullFilterBlock>* parsed_out, BlockContents&& block) {
  parsed_out->reset(new ParsedFullFilterBlock(
@@ -67,6 +67,16 @@ class Block_kMetaIndex : public Block {
  static constexpr BlockType kBlockType = BlockType::kMetaIndex;
 };

+class Block_kUserDefinedIndex : public BlockContents {
+ public:
+  static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kIndexBlock;
+  static constexpr BlockType kBlockType = BlockType::kUserDefinedIndex;
+
+  explicit Block_kUserDefinedIndex(BlockContents&& other)
+      : BlockContents(std::move(other)) {}
+  const Slice& ContentSlice() const { return data; }
+};
+
 struct BlockCreateContext : public Cache::CreateContext {
  BlockCreateContext() {}
  BlockCreateContext(const BlockBasedTableOptions* _table_options,
@@ -126,6 +136,8 @@ struct BlockCreateContext : public Cache::CreateContext {
              BlockContents&& block);
  void Create(std::unique_ptr<Block_kMetaIndex>* parsed_out,
              BlockContents&& block);
+  void Create(std::unique_ptr<Block_kUserDefinedIndex>* parsed_out,
+              BlockContents&& block);
  void Create(std::unique_ptr<ParsedFullFilterBlock>* parsed_out,
              BlockContents&& block);
  void Create(std::unique_ptr<DecompressorDict>* parsed_out,
@@ -27,6 +27,7 @@ enum class BlockType : uint8_t {
  kHashIndexMetadata,
  kMetaIndex,
  kIndex,
+  kUserDefinedIndex,
  // Note: keep kInvalid the last value when adding new enum values.
  kInvalid
 };
@@ -1012,9 +1012,6 @@ class Standard128RibbonBitsBuilder : public XXPH3FilterBitsBuilder {
  FastLocalBloomBitsBuilder bloom_fallback_;
 };

-// for the linker, at least with DEBUG_LEVEL=2
-constexpr uint32_t Standard128RibbonBitsBuilder::kMaxRibbonEntries;
-
 class Standard128RibbonBitsReader : public BuiltinFilterBitsReader {
 public:
  Standard128RibbonBitsReader(const char* data, size_t len_bytes,
@@ -46,7 +46,7 @@ class IndexBuilder {
  //     primary index.
  struct IndexBlocks {
    Slice index_block_contents;
-    std::unordered_map<std::string, Slice> meta_blocks;
+    std::unordered_map<std::string, std::pair<BlockType, Slice>> meta_blocks;
  };
  IndexBuilder(const InternalKeyComparator* comparator, size_t ts_sz,
               bool persist_user_defined_timestamps)
@@ -78,7 +78,8 @@ class IndexBuilder {

  // This method will be called whenever a key is added. The subclasses may
  // override OnKeyAdded() if they need to collect additional information.
-  virtual void OnKeyAdded(const Slice& /*key*/) {}
+  virtual void OnKeyAdded(const Slice& /*key*/,
+                          const std::optional<Slice>& /*value*/) {}

  // Inform the index builder that all entries has been written. Block builder
  // may therefore perform any operation required for block finalization.
@@ -180,7 +181,8 @@ class ShortenedIndexBuilder : public IndexBuilder {
    seperator_is_key_plus_seq_ = (format_version <= 2);
  }

-  void OnKeyAdded(const Slice& key) override {
+  void OnKeyAdded(const Slice& key,
+                  const std::optional<Slice>& /*value*/) override {
    if (include_first_key_ && current_block_first_internal_key_.empty()) {
      current_block_first_internal_key_.assign(key.data(), key.size());
    }
@@ -358,7 +360,8 @@ class HashIndexBuilder : public IndexBuilder {
        separator_scratch);
  }

-  void OnKeyAdded(const Slice& key) override {
+  void OnKeyAdded(const Slice& key,
+                  const std::optional<Slice>& /*value*/) override {
    auto key_prefix = hash_key_extractor_->Transform(key);
    bool is_first_entry = pending_block_num_ == 0;

@@ -393,9 +396,9 @@ class HashIndexBuilder : public IndexBuilder {
    Status s = primary_index_builder_.Finish(index_blocks,
                                             last_partition_block_handle);
    index_blocks->meta_blocks.insert(
-        {kHashIndexPrefixesBlock.c_str(), prefix_block_});
-    index_blocks->meta_blocks.insert(
-        {kHashIndexPrefixesMetadataBlock.c_str(), prefix_meta_block_});
+        {kHashIndexPrefixesBlock.c_str(), {BlockType::kIndex, prefix_block_}});
+    index_blocks->meta_blocks.insert({kHashIndexPrefixesMetadataBlock.c_str(),
+                                      {BlockType::kIndex, prefix_meta_block_}});
    return s;
  }

@@ -0,0 +1,244 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
+#include "rocksdb/user_defined_index.h"
+#include "table/block_based/block_based_table_reader.h"
+#include "table/block_based/block_type.h"
+#include "table/block_based/cachable_entry.h"
+#include "table/block_based/index_builder.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// UserDefinedIndexWrapper wraps around the existing index types in block based
+// table, and supports plugging in an additional user defined index. The wrapper
+// class forwards calls to both the wrapped internal index, and a user defined
+// index builder.
+class UserDefinedIndexBuilderWrapper : public IndexBuilder {
+ public:
+  UserDefinedIndexBuilderWrapper(
+      const std::string& name,
+      std::unique_ptr<IndexBuilder> internal_index_builder,
+      std::unique_ptr<UserDefinedIndexBuilder> user_defined_index_builder,
+      const InternalKeyComparator* comparator, size_t ts_sz,
+      bool persist_user_defined_timestamps)
+      : IndexBuilder(comparator, ts_sz, persist_user_defined_timestamps),
+        name_(name),
+        internal_index_builder_(std::move(internal_index_builder)),
+        user_defined_index_builder_(std::move(user_defined_index_builder)) {}
+
+  ~UserDefinedIndexBuilderWrapper() override = default;
+
+  Slice AddIndexEntry(const Slice& last_key_in_current_block,
+                      const Slice* first_key_in_next_block,
+                      const BlockHandle& block_handle,
+                      std::string* separator_scratch) override {
+    UserDefinedIndexBuilder::BlockHandle handle;
+    handle.offset = block_handle.offset();
+    handle.size = block_handle.size();
+    // Forward the call to both index builders
+    user_defined_index_builder_->AddIndexEntry(last_key_in_current_block,
+                                               first_key_in_next_block, handle,
+                                               separator_scratch);
+    return internal_index_builder_->AddIndexEntry(
+        last_key_in_current_block, first_key_in_next_block, block_handle,
+        separator_scratch);
+  }
+
+  void OnKeyAdded(const Slice& key,
+                  const std::optional<Slice>& value) override {
+    if (status_.ok()) {
+      if (!value.has_value()) {
+        status_ = Status::InvalidArgument(
+            "user_defined_index_factory not supported with parallel "
+            "compression");
+      } else {
+        ParsedInternalKey pkey;
+        status_ = ParseInternalKey(key, &pkey, /*lof_err_key*/ false);
+        if (status_.ok() && pkey.type != ValueType::kTypeValue) {
+          status_ = Status::InvalidArgument(
+              "user_defined_index_factory only supported with Puts");
+        }
+      }
+    }
+    if (!status_.ok()) {
+      return;
+    }
+
+    // Forward the call to both index builders
+    internal_index_builder_->OnKeyAdded(key, value);
+    user_defined_index_builder_->OnKeyAdded(
+        key, UserDefinedIndexBuilder::ValueType::kValue, value.value());
+  }
+
+  Status Finish(IndexBlocks* index_blocks,
+                const BlockHandle& last_partition_block_handle) override {
+    if (!status_.ok()) {
+      return status_;
+    }
+
+    // Finish the internal index builder
+    status_ = internal_index_builder_->Finish(index_blocks,
+                                              last_partition_block_handle);
+    if (!status_.ok()) {
+      return status_;
+    }
+
+    // Finish the user defined index builder
+    Slice user_index_contents;
+    status_ = user_defined_index_builder_->Finish(&user_index_contents);
+    if (!status_.ok()) {
+      return status_;
+    }
+
+    // Add the user defined index to the meta blocks
+    std::string block_name = kUserDefinedIndexPrefix + name_;
+    index_blocks->meta_blocks.insert(
+        {block_name, {BlockType::kUserDefinedIndex, user_index_contents}});
+
+    index_size_ = internal_index_builder_->IndexSize();
+    return status_;
+  }
+
+  size_t IndexSize() const override { return index_size_; }
+
+  bool seperator_is_key_plus_seq() override {
+    return internal_index_builder_->seperator_is_key_plus_seq();
+  }
+
+ private:
+  const std::string name_;
+  std::unique_ptr<IndexBuilder> internal_index_builder_;
+  std::unique_ptr<UserDefinedIndexBuilder> user_defined_index_builder_;
+  Status status_;
+};
+
+class UserDefinedIndexIteratorWrapper
+    : public InternalIteratorBase<IndexValue> {
+ public:
+  explicit UserDefinedIndexIteratorWrapper(
+      std::unique_ptr<UserDefinedIndexIterator>&& udi_iter)
+      : udi_iter_(std::move(udi_iter)), valid_(false) {}
+
+  bool Valid() const override { return valid_; }
+
+  void SeekToFirst() override {
+    status_ = Status::NotSupported("SeekToFirst not supported");
+  }
+
+  void SeekToLast() override {
+    status_ = Status::NotSupported("SeekToLast not supported");
+  }
+
+  void Seek(const Slice& target) override {
+    ParsedInternalKey pkey;
+    status_ = ParseInternalKey(target, &pkey, /*log_err_key=*/false);
+    if (status_.ok()) {
+      status_ = udi_iter_->SeekAndGetResult(pkey.user_key, &result_);
+      valid_ = status_.ok() &&
+               result_.bound_check_result == IterBoundCheck::kInbound;
+    }
+  }
+
+  void Next() override {
+    status_ = udi_iter_->NextAndGetResult(&result_);
+    valid_ =
+        status_.ok() && result_.bound_check_result == IterBoundCheck::kInbound;
+  }
+
+  bool NextAndGetResult(IterateResult* result) override {
+    status_ = udi_iter_->NextAndGetResult(&result_);
+    valid_ =
+        status_.ok() && result_.bound_check_result == IterBoundCheck::kInbound;
+    if (status_.ok()) {
+      *result = result_;
+    }
+    return valid_;
+  }
+
+  void SeekForPrev(const Slice& /*target*/) override {
+    status_ = Status::NotSupported("SeekForPrev not supported");
+  }
+
+  void Prev() override { status_ = Status::NotSupported("Prev not supported"); }
+
+  Slice key() const override { return result_.key; }
+
+  IndexValue value() const override {
+    auto handle = udi_iter_->value();
+    IndexValue val(BlockHandle(handle.offset, handle.size), Slice());
+    return val;
+  }
+
+  Status status() const override { return status_; }
+
+  void Prepare(const std::vector<ScanOptions>* scan_opts) override {
+    udi_iter_->Prepare(scan_opts->data(), scan_opts->size());
+  }
+
+ private:
+  std::unique_ptr<UserDefinedIndexIterator> udi_iter_;
+  IterateResult result_;
+  Status status_;
+  bool valid_;
+};
+
+class UserDefinedIndexReaderWrapper : public BlockBasedTable::IndexReader {
+ public:
+  UserDefinedIndexReaderWrapper(
+      const std::string& name,
+      std::unique_ptr<BlockBasedTable::IndexReader>&& reader,
+      std::unique_ptr<UserDefinedIndexReader>&& udi_reader)
+      : name_(name),
+        reader_(std::move(reader)),
+        udi_reader_(std::move(udi_reader)) {}
+
+  virtual InternalIteratorBase<IndexValue>* NewIterator(
+      const ReadOptions& read_options, bool disable_prefix_seek,
+      IndexBlockIter* iter, GetContext* get_context,
+      BlockCacheLookupContext* lookup_context) override {
+    if (!read_options.table_index_factory) {
+      return reader_->NewIterator(read_options, disable_prefix_seek, iter,
+                                  get_context, lookup_context);
+    }
+    if (name_ != read_options.table_index_factory->Name()) {
+      return NewErrorInternalIterator<IndexValue>(Status::InvalidArgument(
+          "Bad index name" +
+          std::string(read_options.table_index_factory->Name()) +
+          ". Only supported UDI is " + name_));
+    }
+    std::unique_ptr<UserDefinedIndexIterator> udi_iter =
+        udi_reader_->NewIterator(read_options);
+    return new UserDefinedIndexIteratorWrapper(std::move(udi_iter));
+  }
+
+  virtual Status CacheDependencies(
+      const ReadOptions& ro, bool pin,
+      FilePrefetchBuffer* tail_prefetch_buffer) override {
+    return reader_->CacheDependencies(ro, pin, tail_prefetch_buffer);
+  }
+
+  size_t ApproximateMemoryUsage() const override {
+    return reader_->ApproximateMemoryUsage();
+  }
+
+  virtual void EraseFromCacheBeforeDestruction(
+      uint32_t uncache_aggressiveness) override {
+    reader_->EraseFromCacheBeforeDestruction(uncache_aggressiveness);
+  }
+
+ private:
+  std::string name_;
+  std::unique_ptr<BlockBasedTable::IndexReader> reader_;
+  std::unique_ptr<UserDefinedIndexReader> udi_reader_;
+};
+}  // namespace ROCKSDB_NAMESPACE
@@ -303,7 +303,7 @@ class ExternalTableBuilderAdapter : public TableBuilder {
        properties_.num_entries++;
        properties_.raw_key_size += key.size();
        properties_.raw_value_size += value.size();
-        NotifyCollectTableCollectorsOnAdd(key, value, /*offset=*/0,
+        NotifyCollectTableCollectorsOnAdd(key, value, /*file_size=*/0,
                                          table_properties_collectors_,
                                          ioptions_.logger);
      }
@@ -482,15 +482,42 @@ bool& TEST_AllowUnsupportedFormatVersion() {
  return allow;
 }

-static Status ReadFooterFromFileInternal(const IOOptions& opts,
-                                         RandomAccessFileReader* file,
-                                         FileSystem& fs,
-                                         FilePrefetchBuffer* prefetch_buffer,
-                                         uint64_t file_size, Footer* footer,
-                                         uint64_t enforce_table_magic_number) {
-  if (file_size < Footer::kMinEncodedLength) {
+static Status ReadFooterFromFileInternal(
+    const IOOptions& opts, RandomAccessFileReader* file, FileSystem& fs,
+    FilePrefetchBuffer* prefetch_buffer, uint64_t expected_file_size,
+    Footer* footer, uint64_t enforce_table_magic_number) {
+  uint64_t file_size_from_file_system = 0;
+  Status s;
+  s = file->file()->GetFileSize(&file_size_from_file_system);
+  if (!s.ok()) {
+    auto corrupted_status =
+        Status::Corruption("Failed to get file size: " + s.ToString() +
+                           " for file " + file->file_name());
+    if (s.IsNotSupported()) {
+      // If file handle does not support GetFileSize, try File System API
+      s = fs.GetFileSize(file->file_name(), IOOptions(),
+                         &file_size_from_file_system, nullptr);
+      if (!s.ok()) {
+        return corrupted_status;
+      }
+    } else {
+      return corrupted_status;
+    }
+  }
+
+  if (expected_file_size != file_size_from_file_system) {
+    // When file is opened during DB Open, the expected file size is from
+    // manifest. Otherwise it is not guaranteed.
+    return Status::Corruption("Sst file size mismatch between expected " +
+                              std::to_string(expected_file_size) +
+                              " and file system " +
+                              std::to_string(file_size_from_file_system) +
+                              " sstable: " + file->file_name());
+  }
+
+  if (expected_file_size < Footer::kMinEncodedLength) {
    return Status::Corruption("file is too short (" +
-                              std::to_string(file_size) +
+                              std::to_string(expected_file_size) +
                              " bytes) to be an "
                              "sstable: " +
                              file->file_name());
@@ -499,10 +526,9 @@ static Status ReadFooterFromFileInternal(const IOOptions& opts,
  std::array<char, Footer::kMaxEncodedLength + 1> footer_buf;
  AlignedBuf internal_buf;
  Slice footer_input;
-  uint64_t read_offset = (file_size > Footer::kMaxEncodedLength)
-                             ? file_size - Footer::kMaxEncodedLength
+  uint64_t read_offset = (expected_file_size > Footer::kMaxEncodedLength)
+                             ? expected_file_size - Footer::kMaxEncodedLength
                             : 0;
-  Status s;
  // TODO: Need to pass appropriate deadline to TryReadFromCache(). Right now,
  // there is no readahead for point lookups, so TryReadFromCache will fail if
  // the required data is not in the prefetch buffer. Once deadline is enabled
@@ -527,23 +553,14 @@ static Status ReadFooterFromFileInternal(const IOOptions& opts,

  TEST_SYNC_POINT_CALLBACK("ReadFooterFromFileInternal:0", &footer_input);

-  // Check that we actually read the whole footer from the file. It may be
-  // that size isn't correct.
+  // Check that we actually read the whole footer from the file.
  if (footer_input.size() < Footer::kMinEncodedLength) {
-    uint64_t size_on_disk = 0;
-    if (fs.GetFileSize(file->file_name(), IOOptions(), &size_on_disk, nullptr)
-            .ok()) {
-      // Similar to CheckConsistency message, but not completely sure the
-      // expected size always came from manifest.
-      return Status::Corruption("Sst file size mismatch: " + file->file_name() +
-                                ". Expected " + std::to_string(file_size) +
-                                ", actual size " +
-                                std::to_string(size_on_disk) + "\n");
-    } else {
-      return Status::Corruption(
-          "Missing SST footer data in file " + file->file_name() +
-          " File too short? Expected size: " + std::to_string(file_size));
-    }
+    return Status::Corruption(
+        "The number of bytes read for Footer input " +
+        std::to_string(footer_input.size()) +
+        " is smaller than minimum footer encoded length: " +
+        std::to_string(Footer::kMinEncodedLength) + " for file " +
+        file->file_name() + "\n");
  }

  s = footer->DecodeFrom(footer_input, read_offset, enforce_table_magic_number);
@@ -556,20 +573,21 @@ static Status ReadFooterFromFileInternal(const IOOptions& opts,

 Status ReadFooterFromFile(const IOOptions& opts, RandomAccessFileReader* file,
                          FileSystem& fs, FilePrefetchBuffer* prefetch_buffer,
-                          uint64_t file_size, Footer* footer,
+                          uint64_t expected_file_size, Footer* footer,
                          uint64_t enforce_table_magic_number,
                          Statistics* stats) {
-  Status s =
-      ReadFooterFromFileInternal(opts, file, fs, prefetch_buffer, file_size,
-                                 footer, enforce_table_magic_number);
+  Status s = ReadFooterFromFileInternal(opts, file, fs, prefetch_buffer,
+                                        expected_file_size, footer,
+                                        enforce_table_magic_number);
  if (s.IsCorruption() &&
      CheckFSFeatureSupport(&fs, FSSupportedOps::kVerifyAndReconstructRead)) {
    IOOptions new_opts = opts;
    new_opts.verify_and_reconstruct_read = true;
    footer->Reset();
    s = ReadFooterFromFileInternal(new_opts, file, fs,
-                                   /*prefetch_buffer=*/nullptr, file_size,
-                                   footer, enforce_table_magic_number);
+                                   /*prefetch_buffer=*/nullptr,
+                                   expected_file_size, footer,
+                                   enforce_table_magic_number);
    RecordTick(stats, FILE_READ_CORRUPTION_RETRY_COUNT);
    if (s.ok()) {
      RecordTick(stats, FILE_READ_CORRUPTION_RETRY_SUCCESS_COUNT);
@@ -55,7 +55,7 @@ class BlockHandle {
  uint64_t offset() const { return offset_; }
  void set_offset(uint64_t _offset) { offset_ = _offset; }

-  // The size of the stored block
+  // The size of the stored block, this size does not include the block trailer.
  uint64_t size() const { return size_; }
  void set_size(uint64_t _size) { size_ = _size; }

@@ -260,16 +260,21 @@ Status SstFileDumper::CalculateCompressedTableSize(
 }

 Status SstFileDumper::ShowAllCompressionSizes(
-    size_t block_size,
-    const std::vector<std::pair<CompressionType, const char*>>&
-        compression_types,
+    size_t block_size, const std::vector<CompressionType>& compression_types,
    int32_t compress_level_from, int32_t compress_level_to,
    uint32_t max_dict_bytes, uint32_t zstd_max_train_bytes,
    uint64_t max_dict_buffer_bytes, bool use_zstd_dict_trainer) {
  fprintf(stdout, "Block Size: %" ROCKSDB_PRIszt "\n", block_size);
-  for (auto& i : compression_types) {
-    if (CompressionTypeSupported(i.first)) {
-      fprintf(stdout, "Compression: %-24s\n", i.second);
+  for (CompressionType ctype : compression_types) {
+    std::string cname;
+    if (!GetStringFromCompressionType(&cname, ctype).ok()) {
+      // Can produce names like "Reserved4F" for unrecognized values
+      cname = CompressionTypeToString(ctype);
+    }
+    if (options_.compression_manager
+            ? options_.compression_manager->SupportsCompressionType(ctype)
+            : CompressionTypeSupported(ctype)) {
+      fprintf(stdout, "Compression: %-24s\n", cname.c_str());
      CompressionOptions compress_opt;
      compress_opt.max_dict_bytes = max_dict_bytes;
      compress_opt.zstd_max_train_bytes = zstd_max_train_bytes;
@@ -278,13 +283,13 @@ Status SstFileDumper::ShowAllCompressionSizes(
      for (int32_t j = compress_level_from; j <= compress_level_to; j++) {
        fprintf(stdout, "Compression level: %d", j);
        compress_opt.level = j;
-        Status s = ShowCompressionSize(block_size, i.first, compress_opt);
+        Status s = ShowCompressionSize(block_size, ctype, compress_opt);
        if (!s.ok()) {
          return s;
        }
      }
    } else {
-      fprintf(stdout, "Unsupported compression type: %s.\n", i.second);
+      fprintf(stdout, "Unsupported compression type: %s.\n", cname.c_str());
    }
  }
  return Status::OK();
@@ -43,9 +43,7 @@ class SstFileDumper {
  Status getStatus() { return init_result_; }

  Status ShowAllCompressionSizes(
-      size_t block_size,
-      const std::vector<std::pair<CompressionType, const char*>>&
-          compression_types,
+      size_t block_size, const std::vector<CompressionType>& compression_types,
      int32_t compress_level_from, int32_t compress_level_to,
      uint32_t max_dict_bytes, uint32_t zstd_max_train_bytes,
      uint64_t max_dict_buffer_bytes, bool use_zstd_dict_trainer);
@@ -472,6 +472,7 @@ Status SstFileWriter::Finish(ExternalSstFileInfo* file_info) {
  }
  if (r->file_info.num_entries == 0 &&
      r->file_info.num_range_del_entries == 0) {
+    r->builder->status().PermitUncheckedError();
    return Status::InvalidArgument("Cannot create sst file with no entries");
  }

@@ -495,7 +496,10 @@ Status SstFileWriter::Finish(ExternalSstFileInfo* file_info) {
        r->file_writer->GetFileChecksumFuncName();
  }
  if (!s.ok()) {
-    r->ioptions.env->DeleteFile(r->file_info.file_path);
+    Status status = r->ioptions.env->DeleteFile(r->file_info.file_path);
+    // Silence ASSERT_STATUS_CHECKED warning, since DeleteFile may fail under
+    // some error injection, and we can just ignore the failure
+    status.PermitUncheckedError();
  }

  if (file_info != nullptr) {
@@ -29,6 +29,7 @@
 #include "db/write_batch_internal.h"
 #include "memtable/stl_wrappers.h"
 #include "monitoring/statistics_impl.h"
+#include "options/cf_options.h"
 #include "options/options_helper.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
@@ -51,6 +52,7 @@
 #include "rocksdb/table_properties.h"
 #include "rocksdb/trace_record.h"
 #include "rocksdb/unique_id.h"
+#include "rocksdb/user_defined_index.h"
 #include "rocksdb/write_buffer_manager.h"
 #include "table/block_based/block.h"
 #include "table/block_based/block_based_table_builder.h"
@@ -71,7 +73,7 @@
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
-#include "util/coding_lean.h"
+#include "util/coding.h"
 #include "util/compression.h"
 #include "util/file_checksum_helper.h"
 #include "util/random.h"
@@ -7396,6 +7398,632 @@ TEST_F(ExternalTableTest, IngestionTest) {
  ASSERT_OK(db->DestroyColumnFamilyHandle(cfh));
  ASSERT_OK(db->Close());
 }
+
+class UserDefinedIndexTest : public BlockBasedTableTestBase {
+ public:
+  class CustomFlushBlockPolicy : public FlushBlockPolicy {
+   public:
+    explicit CustomFlushBlockPolicy(int keys_per_block)
+        : keys_in_current_block_(0), keys_per_block_(keys_per_block) {}
+
+    bool Update(const Slice& /*key*/, const Slice& /*value*/) override {
+      if (keys_in_current_block_ >= keys_per_block_) {
+        keys_in_current_block_ = 1;
+        return true;
+      }
+      keys_in_current_block_++;
+      return false;
+    }
+
+   private:
+    int keys_in_current_block_;
+    int keys_per_block_;
+  };
+
+  class CustomFlushBlockPolicyFactory : public FlushBlockPolicyFactory {
+   public:
+    CustomFlushBlockPolicyFactory(int keys_per_block = 3)
+        : keys_per_block_(keys_per_block) {}
+    const char* Name() const override { return "CustomFlushBlockPolicy"; }
+    FlushBlockPolicy* NewFlushBlockPolicy(const BlockBasedTableOptions&,
+                                          const BlockBuilder&) const override {
+      return new CustomFlushBlockPolicy(keys_per_block_);
+    }
+    int keys_per_block_;
+  };
+
+ public:
+  class TestUserDefinedIndexFactory : public UserDefinedIndexFactory {
+   public:
+    const char* Name() const override { return "test_index"; }
+    UserDefinedIndexBuilder* NewBuilder() const override {
+      return new TestUserDefinedIndexBuilder();
+    }
+
+    std::unique_ptr<UserDefinedIndexReader> NewReader(
+        Slice& index_block) const override {
+      return std::make_unique<TestUserDefinedIndexReader>(index_block, this);
+    }
+
+    uint64_t seek_error_count_ = 0;
+    uint64_t next_error_count_ = 0;
+
+   private:
+    class TestUserDefinedIndexBuilder : public UserDefinedIndexBuilder {
+     public:
+      TestUserDefinedIndexBuilder() : entries_added_(0), keys_added_(0) {}
+
+      Slice AddIndexEntry(const Slice& last_key_in_current_block,
+                          const Slice* first_key_in_next_block,
+                          const BlockHandle& block_handle,
+                          std::string* separator_scratch) override {
+        // Unused parameters
+        (void)first_key_in_next_block;
+        (void)separator_scratch;
+        entries_added_++;
+        // Store the block handle for each key
+        PutFixed64(&index_data_[last_key_in_current_block.ToString()],
+                   block_handle.offset);
+        PutFixed64(&index_data_[last_key_in_current_block.ToString()],
+                   block_handle.size);
+        PutFixed32(&index_data_[last_key_in_current_block.ToString()],
+                   keys_added_);
+        keys_added_ = 0;
+        return last_key_in_current_block;
+      }
+
+      void OnKeyAdded(const Slice& /*key*/, ValueType /*value*/,
+                      const Slice& /*value*/) override {
+        // Track keys added to the index
+        keys_added_++;
+      }
+
+      Status Finish(Slice* index_contents) override {
+        // Serialize the index data
+        std::string result;
+        for (const auto& entry : index_data_) {
+          PutLengthPrefixedSlice(&result, entry.first);
+          result.append(entry.second);
+        }
+        index_contents_data_ = result;
+        *index_contents = index_contents_data_;
+        return Status::OK();
+      }
+
+      int GetEntriesAdded() const { return entries_added_; }
+
+     private:
+      int entries_added_;
+      std::map<std::string, std::string> index_data_;
+      uint32_t keys_added_;
+      std::string index_contents_data_;
+    };
+
+    class TestUserDefinedIndexReader : public UserDefinedIndexReader {
+     public:
+      explicit TestUserDefinedIndexReader(
+          Slice& index_block, const TestUserDefinedIndexFactory* factory)
+          : factory_(factory) {
+        Slice block = index_block;
+        while (!block.empty()) {
+          Slice key;
+          uint64_t offset = 0;
+          uint64_t size = 0;
+          uint32_t num_keys = 0;
+          EXPECT_TRUE(GetLengthPrefixedSlice(&block, &key));
+          EXPECT_TRUE(GetFixed64(&block, &offset));
+          EXPECT_TRUE(GetFixed64(&block, &size));
+          EXPECT_TRUE(GetFixed32(&block, &num_keys));
+
+          UserDefinedIndexBuilder::BlockHandle handle{0, 0};
+          handle.offset = offset;
+          handle.size = size;
+          index_data_[key.ToString()] =
+              std::make_pair<UserDefinedIndexBuilder::BlockHandle, uint32_t>(
+                  std::move(handle), std::move(num_keys));
+        }
+      }
+
+      std::unique_ptr<UserDefinedIndexIterator> NewIterator(
+          const ReadOptions& ro) override {
+        return std::make_unique<TestUserDefinedIndexIterator>(ro, index_data_,
+                                                              factory_);
+      }
+
+      size_t ApproximateMemoryUsage() const override { return 0; }
+
+     private:
+      class TestUserDefinedIndexIterator : public UserDefinedIndexIterator {
+       public:
+        TestUserDefinedIndexIterator(
+            const ReadOptions& ro,
+            std::map<std::string,
+                     std::pair<UserDefinedIndexBuilder::BlockHandle, uint32_t>>&
+                index,
+            const TestUserDefinedIndexFactory* factory)
+            : ro_(ro),
+              index_(index),
+              iter_(index_.end()),
+              scan_opts_(nullptr),
+              num_opts_(0),
+              target_num_keys_(0),
+              seek_error_count_(factory->seek_error_count_),
+              next_error_count_(factory->next_error_count_) {}
+
+        Status SeekAndGetResult(const Slice& key,
+                                IterateResult* result) override {
+          Status s;
+          if (seek_error_count_) {
+            seek_error_count_--;
+            s = Status::IOError();
+          }
+          if (!s.ok()) {
+            return s;
+          }
+          if (scan_opts_) {
+            if (scan_opts_[scan_idx_].range.start.value().compare(key) == 0) {
+              EXPECT_TRUE(scan_opts_[scan_idx_].property_bag.has_value());
+              target_num_keys_ = std::stoi(scan_opts_[scan_idx_]
+                                               .property_bag.value()
+                                               .find("count")
+                                               ->second);
+              scan_idx_++;
+            } else {
+              scan_opts_ = nullptr;
+            }
+          }
+          iter_ = index_.lower_bound(key.ToString());
+          if (iter_ != index_.end()) {
+            result->bound_check_result = IterBoundCheck::kInbound;
+            result->key = Slice(iter_->first);
+            if (scan_opts_ && target_num_keys_ > 0 &&
+                iter_->first.compare(key.ToString()) == 0) {
+              target_num_keys_--;
+            }
+          } else {
+            result->bound_check_result = IterBoundCheck::kOutOfBound;
+            result->key = Slice();
+          }
+          return Status::OK();
+        }
+
+        Status NextAndGetResult(IterateResult* result) override {
+          Status s;
+          if (next_error_count_) {
+            next_error_count_--;
+            s = Status::IOError();
+          }
+          if (!s.ok()) {
+            return s;
+          }
+          if (ro_.iterate_upper_bound) {
+            if (iter_->first.compare(ro_.iterate_upper_bound->ToString()) >=
+                0) {
+              result->bound_check_result = IterBoundCheck::kOutOfBound;
+              result->key = Slice();
+              return Status::OK();
+            }
+          }
+          if (scan_opts_ && target_num_keys_ == 0) {
+            result->key = Slice();
+            result->bound_check_result = IterBoundCheck::kOutOfBound;
+            return Status::OK();
+          }
+          iter_++;
+          if (iter_ != index_.end()) {
+            result->bound_check_result = IterBoundCheck::kInbound;
+            result->key = Slice(iter_->first);
+            target_num_keys_ -=
+                std::min(target_num_keys_, iter_->second.second);
+          } else {
+            // EOF
+            result->bound_check_result = IterBoundCheck::kUnknown;
+            result->key = Slice();
+          }
+          return Status::OK();
+        }
+
+        UserDefinedIndexBuilder::BlockHandle value() override {
+          UserDefinedIndexBuilder::BlockHandle handle{0, 0};
+          handle.offset = iter_->second.first.offset;
+          handle.size = iter_->second.first.size;
+          return handle;
+        }
+
+        void Prepare(const ScanOptions scan_opts[], size_t num_opts) override {
+          scan_opts_ = scan_opts;
+          num_opts_ = num_opts;
+          scan_idx_ = 0;
+        }
+
+       private:
+        const ReadOptions& ro_;
+        std::map<std::string,
+                 std::pair<UserDefinedIndexBuilder::BlockHandle, uint32_t>>&
+            index_;
+        std::map<std::string, std::pair<UserDefinedIndexBuilder::BlockHandle,
+                                        uint32_t>>::iterator iter_;
+        const ScanOptions* scan_opts_;
+        size_t num_opts_{};
+        size_t scan_idx_{};
+        uint32_t target_num_keys_;
+        uint64_t seek_error_count_;
+        uint64_t next_error_count_;
+      };
+
+      const TestUserDefinedIndexFactory* factory_;
+      std::map<std::string,
+               std::pair<UserDefinedIndexBuilder::BlockHandle, uint32_t>>
+          index_data_;
+    };
+  };
+};
+
+TEST_F(UserDefinedIndexTest, BasicTest) {
+  Options options;
+  BlockBasedTableOptions table_options;
+  std::string dbname = test::PerThreadDBPath("user_defined_index_test");
+  std::string ingest_file = dbname + "test.sst";
+
+  // Set up the user-defined index factory
+  auto user_defined_index_factory =
+      std::make_shared<TestUserDefinedIndexFactory>();
+  table_options.user_defined_index_factory = user_defined_index_factory;
+
+  // Set up custom flush block policy that flushes every 3 keys
+  table_options.flush_block_policy_factory =
+      std::make_shared<CustomFlushBlockPolicyFactory>();
+
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  std::unique_ptr<SstFileWriter> writer;
+  writer.reset(new SstFileWriter(EnvOptions(), options));
+  ASSERT_OK(writer->Open(ingest_file));
+
+  // Add 100 keys instead of just 5
+  for (int i = 0; i < 100; i++) {
+    std::stringstream ss;
+    ss << std::setw(2) << std::setfill('0') << i;
+    std::string key = "key" + ss.str();
+    std::string value = "value" + ss.str();
+    ASSERT_OK(writer->Put(key, value));
+  }
+  ASSERT_OK(writer->Finish());
+  writer.reset();
+
+  ImmutableOptions ioptions(options);
+  MutableCFOptions moptions((ColumnFamilyOptions(options)));
+  EnvOptions eoptions(options);
+  TableReaderOptions toptions(
+      ioptions, moptions.prefix_extractor,
+      /*_compression_manager=*/nullptr, eoptions, ioptions.internal_comparator,
+      moptions.block_protection_bytes_per_key,
+      /*skip_filters*/ false, /*immortal*/ false,
+      /*force_direct_prefetch*/ false, /*level*/ -1,
+      /*block_cache_tracer*/ nullptr,
+      /*max_file_size_for_l0_meta_pin*/ 0, /*cur_db_session_id*/ "",
+      /*cur_file_num*/ 0,
+      /* unique_id */ {}, /* largest_seqno */ 0,
+      /* tail_size */ 0, ioptions.persist_user_defined_timestamps);
+  // Verify that the user-defined index was created
+  std::string meta_block_name = kUserDefinedIndexPrefix + "test_index";
+  BlockHandle block_handle;
+  uint64_t file_size = 0;
+  std::unique_ptr<FSRandomAccessFile> file;
+  std::unique_ptr<RandomAccessFileReader> file_reader;
+  const auto& fs = options.env->GetFileSystem();
+  ASSERT_OK(fs->GetFileSize(ingest_file, IOOptions(), &file_size, nullptr));
+  ASSERT_OK(fs->NewRandomAccessFile(ingest_file, eoptions, &file, nullptr));
+  file_reader.reset(new RandomAccessFileReader(std::move(file), ingest_file));
+  ASSERT_OK(FindMetaBlockInFile(file_reader.get(), file_size,
+                                kBlockBasedTableMagicNumber, ioptions,
+                                ReadOptions(), meta_block_name, &block_handle));
+  file_reader.reset();
+  // With our custom flush policy that flushes every 3 keys,
+  // we expect around 34 data blocks (100/3 rounded up)
+  // Verify the number of entries in the user-defined index
+  // Each data block should have an entry in the index
+  // With our flush policy of 3 keys per block, we expect around 34 entries
+  int expected_entries = (100 + 2) / 3;  // Ceiling of 100/3
+  ASSERT_GE(block_handle.size(),
+            expected_entries);  // At least this many entries
+
+  std::unique_ptr<SstFileReader> reader(new SstFileReader(options));
+  ASSERT_OK(reader->Open(ingest_file));
+
+  ReadOptions ro;
+  std::unique_ptr<Iterator> iter(reader->NewIterator(ro));
+  ASSERT_NE(iter, nullptr);
+
+  // Test that we can read all the keys
+  int key_count = 0;
+  for (iter->SeekToFirst(); iter->Valid() && iter->status().ok();
+       iter->Next()) {
+    key_count++;
+  }
+  ASSERT_EQ(key_count, 100);  // We added 100 keys
+  ASSERT_OK(iter->status());
+  iter.reset();
+
+  ro.table_index_factory = user_defined_index_factory.get();
+  iter.reset(reader->NewIterator(ro));
+  ASSERT_NE(iter, nullptr);
+
+  // Test that we can read all the keys
+  key_count = 0;
+  for (iter->Seek("key09"); iter->Valid(); iter->Next()) {
+    key_count++;
+  }
+  ASSERT_EQ(key_count, 91);
+  ASSERT_OK(iter->status());
+
+  Slice ub("key75");
+  ro.iterate_upper_bound = &ub;
+  iter.reset(reader->NewIterator(ro));
+  ASSERT_NE(iter, nullptr);
+
+  // Test that we can read all the keys
+  key_count = 0;
+  for (iter->Seek("key09"); iter->Valid(); iter->Next()) {
+    key_count++;
+  }
+  ASSERT_EQ(key_count, 66);
+  ASSERT_OK(iter->status());
+
+  user_defined_index_factory->seek_error_count_ = 1;
+  iter.reset(reader->NewIterator(ro));
+  ASSERT_NE(iter, nullptr);
+  iter->Seek("key09");
+  ASSERT_NOK(iter->status());
+
+  user_defined_index_factory->seek_error_count_ = 0;
+  user_defined_index_factory->next_error_count_ = 1;
+  iter.reset(reader->NewIterator(ro));
+  ASSERT_NE(iter, nullptr);
+  iter->Seek("key09");
+  ASSERT_OK(iter->status());
+  iter->Next();
+  ASSERT_OK(iter->status());
+  iter->Next();
+  ASSERT_OK(iter->status());
+  iter->Next();
+  ASSERT_NOK(iter->status());
+  user_defined_index_factory->next_error_count_ = 0;
+
+  ro.iterate_upper_bound = nullptr;
+  iter.reset(reader->NewIterator(ro));
+  ASSERT_NE(iter, nullptr);
+  std::vector<ScanOptions> scan_opts({ScanOptions("key20")});
+  ;
+  scan_opts[0].property_bag.emplace().emplace("count", std::to_string(25));
+  iter->Prepare(scan_opts);
+  // Test that we can read all the keys
+  key_count = 0;
+  for (iter->Seek(scan_opts[0].range.start.value()); iter->Valid();
+       iter->Next()) {
+    key_count++;
+  }
+  ASSERT_GE(key_count, 25);
+  // The index may undercount by 2 blocks
+  ASSERT_LE(key_count, 30);
+  ASSERT_OK(iter->status());
+}
+
+TEST_F(UserDefinedIndexTest, InvalidArgumentTest1) {
+  Options options;
+  BlockBasedTableOptions table_options;
+  std::string dbname = test::PerThreadDBPath("user_defined_index_test");
+  std::string ingest_file = dbname + "test.sst";
+
+  // Set up the user-defined index factory
+  auto user_defined_index_factory =
+      std::make_shared<TestUserDefinedIndexFactory>();
+  table_options.user_defined_index_factory = user_defined_index_factory;
+
+  // Set up custom flush block policy that flushes every 3 keys
+  table_options.flush_block_policy_factory =
+      std::make_shared<CustomFlushBlockPolicyFactory>();
+
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  options.compression_opts.parallel_threads = 10;
+
+  std::unique_ptr<SstFileWriter> writer;
+  writer.reset(new SstFileWriter(EnvOptions(), options));
+  ASSERT_OK(writer->Open(ingest_file));
+
+  std::string key = "foo";
+  std::string value = "bar";
+  ASSERT_EQ(writer->Put(key, value), Status::InvalidArgument());
+  ASSERT_EQ(writer->Finish(), Status::InvalidArgument());
+  writer.reset();
+}
+
+TEST_F(UserDefinedIndexTest, InvalidArgumentTest2) {
+  Options options;
+  BlockBasedTableOptions table_options;
+  std::string dbname = test::PerThreadDBPath("user_defined_index_test");
+  std::string ingest_file = dbname + "test.sst";
+
+  // Set up the user-defined index factory
+  auto user_defined_index_factory =
+      std::make_shared<TestUserDefinedIndexFactory>();
+  table_options.user_defined_index_factory = user_defined_index_factory;
+
+  // Set up custom flush block policy that flushes every 3 keys
+  table_options.flush_block_policy_factory =
+      std::make_shared<CustomFlushBlockPolicyFactory>();
+
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  std::unique_ptr<SstFileWriter> writer;
+  writer.reset(new SstFileWriter(EnvOptions(), options));
+  ASSERT_OK(writer->Open(ingest_file));
+
+  std::string key = "foo";
+  std::string value = "bar";
+  ASSERT_OK(writer->Merge(key, value));
+  ASSERT_EQ(writer->Finish(), Status::InvalidArgument());
+  writer.reset();
+}
+
+TEST_F(UserDefinedIndexTest, IngestTest) {
+  Options options;
+  BlockBasedTableOptions table_options;
+  std::string dbname = test::PerThreadDBPath("user_defined_index_test");
+  std::string ingest_file = dbname + "test.sst";
+
+  // Set up the user-defined index factory
+  auto user_defined_index_factory =
+      std::make_shared<TestUserDefinedIndexFactory>();
+  table_options.user_defined_index_factory = user_defined_index_factory;
+
+  // Set up custom flush block policy that flushes every 3 keys
+  table_options.flush_block_policy_factory =
+      std::make_shared<CustomFlushBlockPolicyFactory>();
+
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  std::unique_ptr<SstFileWriter> writer;
+  writer.reset(new SstFileWriter(EnvOptions(), options));
+  ASSERT_OK(writer->Open(ingest_file));
+
+  // Add 100 keys instead of just 5
+  for (int i = 0; i < 100; i++) {
+    std::stringstream ss;
+    ss << std::setw(2) << std::setfill('0') << i;
+    std::string key = "key" + ss.str();
+    std::string value = "value" + ss.str();
+    ASSERT_OK(writer->Put(key, value));
+  }
+  ASSERT_OK(writer->Finish());
+  writer.reset();
+
+  std::unique_ptr<DB> db;
+  options.create_if_missing = true;
+  Status s = DB::Open(options, dbname, &db);
+  ASSERT_OK(s);
+  ASSERT_TRUE(db != nullptr);
+  ColumnFamilyHandle* cfh = nullptr;
+  ASSERT_OK(db->CreateColumnFamily(options, "new_cf", &cfh));
+
+  IngestExternalFileOptions ifo;
+  s = db->IngestExternalFile(cfh, {ingest_file}, ifo);
+  ASSERT_OK(s);
+
+  ReadOptions ro;
+  std::unique_ptr<Iterator> iter(db->NewIterator(ro, cfh));
+  ASSERT_NE(iter, nullptr);
+  ASSERT_OK(iter->status());
+
+  // Test that we can read all the keys
+  int key_count = 0;
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    key_count++;
+  }
+  ASSERT_EQ(key_count, 100);  // We added 100 keys
+  ASSERT_OK(iter->status());
+  iter.reset();
+
+  ro.table_index_factory = user_defined_index_factory.get();
+  iter.reset(db->NewIterator(ro, cfh));
+  ASSERT_NE(iter, nullptr);
+
+  // Test that we can read all the keys
+  key_count = 0;
+  for (iter->Seek("key09"); iter->Valid(); iter->Next()) {
+    key_count++;
+  }
+  ASSERT_EQ(key_count, 91);
+  ASSERT_OK(iter->status());
+
+  Slice ub("key75");
+  ro.iterate_upper_bound = &ub;
+  iter.reset(db->NewIterator(ro, cfh));
+  ASSERT_NE(iter, nullptr);
+
+  // Test that we can read all the keys
+  key_count = 0;
+  for (iter->Seek("key09"); iter->Valid(); iter->Next()) {
+    key_count++;
+  }
+  ASSERT_EQ(key_count, 66);
+  ASSERT_OK(iter->status());
+
+  ro.iterate_upper_bound = nullptr;
+  iter.reset(db->NewIterator(ro, cfh));
+  ASSERT_NE(iter, nullptr);
+  std::vector<ScanOptions> scan_opts({ScanOptions("key20")});
+  ;
+  scan_opts[0].property_bag.emplace().emplace("count", std::to_string(25));
+  iter->Prepare(scan_opts);
+  // Test that we can read all the keys
+  key_count = 0;
+  for (iter->Seek(scan_opts[0].range.start.value()); iter->Valid();
+       iter->Next()) {
+    key_count++;
+  }
+  ASSERT_GE(key_count, 25);
+  // The index may undercount by 2 blocks
+  ASSERT_LE(key_count, 30);
+  ASSERT_OK(iter->status());
+  iter.reset();
+
+  ASSERT_OK(db->DestroyColumnFamilyHandle(cfh));
+  ASSERT_OK(db->Close());
+  ASSERT_OK(DestroyDB(dbname, options));
+}
+
+// Verify that external file ingestion fails if we try to ingest an SST file
+// without the UDI and a UDI factory is configured in BlockBasedTableOptions
+TEST_F(UserDefinedIndexTest, IngestFailTest) {
+  Options options;
+  BlockBasedTableOptions table_options;
+  std::string dbname = test::PerThreadDBPath("user_defined_index_test");
+  std::string ingest_file = dbname + "test.sst";
+
+  // Set up custom flush block policy that flushes every 3 keys
+  table_options.flush_block_policy_factory =
+      std::make_shared<CustomFlushBlockPolicyFactory>();
+
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  std::unique_ptr<SstFileWriter> writer;
+  writer.reset(new SstFileWriter(EnvOptions(), options));
+  ASSERT_OK(writer->Open(ingest_file));
+
+  // Add 100 keys instead of just 5
+  for (int i = 0; i < 100; i++) {
+    std::stringstream ss;
+    ss << std::setw(2) << std::setfill('0') << i;
+    std::string key = "key" + ss.str();
+    std::string value = "value" + ss.str();
+    ASSERT_OK(writer->Put(key, value));
+  }
+  ASSERT_OK(writer->Finish());
+  writer.reset();
+
+  // Set up the user-defined index factory
+  auto user_defined_index_factory =
+      std::make_shared<TestUserDefinedIndexFactory>();
+  table_options.user_defined_index_factory = user_defined_index_factory;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  std::unique_ptr<DB> db;
+  options.create_if_missing = true;
+  Status s = DB::Open(options, dbname, &db);
+  ASSERT_OK(s);
+  ASSERT_TRUE(db != nullptr);
+  ColumnFamilyHandle* cfh = nullptr;
+  ASSERT_OK(db->CreateColumnFamily(options, "new_cf", &cfh));
+
+  IngestExternalFileOptions ifo;
+  s = db->IngestExternalFile(cfh, {ingest_file}, ifo);
+  ASSERT_NOK(s);
+
+  ASSERT_OK(db->DestroyColumnFamilyHandle(cfh));
+  ASSERT_OK(db->Close());
+  ASSERT_OK(DestroyDB(dbname, options));
+}
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {