Update HISTORY.md for PR 9273 (#9282 )

Summary: Pull Request resolved: https://github.com/facebook/rocksdb/pull/9282 Reviewed By: akankshamahajan15 Differential Revision: D33027844 Pulled By: ltamasi fbshipit-source-id: 7540d36010414311bc39610fff92a6498be1570c
Update HISTORY.md and version.h for 6.27.3
2026-11-07 03:22:34 -05:00 · 2021-12-10 14:56:20 -08:00 · 2021-12-10 10:26:22 -08:00 · 2021-12-10 09:53:08 -08:00 · 2021-12-09 13:40:03 -08:00 · 2021-12-09 13:39:49 -08:00
211 changed files with 5940 additions and 1998 deletions
@@ -48,7 +48,6 @@ commands:
            echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
            echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
            echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
-            echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV
            echo "export GTEST_COLOR=1" >> $BASH_ENV
            echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV
            echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV
@@ -152,7 +151,7 @@ jobs:
      - install-pyenv-on-macos
      - install-gflags-on-macos
      - pre-steps-macos
-      - run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
+      - run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-macos-cmake:
@@ -165,7 +164,7 @@ jobs:
      - install-cmake-on-macos
      - install-gflags-on-macos
      - pre-steps-macos
-      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32 && ctest -j10) | .circleci/cat_ignore_eagain
+      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32 && ctest -j10) 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux:
@@ -175,7 +174,7 @@ jobs:
    steps:
      - pre-steps
      - install-gflags
-      - run: make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
+      - run: make V=1 J=32 -j32 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-mem-env-librados:
@@ -186,7 +185,7 @@ jobs:
      - pre-steps
      - install-gflags
      - install-librados
-      - run: MEM_ENV=1 ROCKSDB_USE_LIBRADOS=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
+      - run: MEM_ENV=1 ROCKSDB_USE_LIBRADOS=1 make V=1 J=32 -j32 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-encrypted-env:
@@ -196,7 +195,7 @@ jobs:
    steps:
      - pre-steps
      - install-gflags
-      - run: ENCRYPTED_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
+      - run: ENCRYPTED_ENV=1 make V=1 J=32 -j32 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-shared_lib-alt_namespace-status_checked:
@@ -206,7 +205,7 @@ jobs:
    steps:
      - pre-steps
      - install-gflags
-      - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 check | .circleci/cat_ignore_eagain
+      - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-release:
@@ -215,10 +214,10 @@ jobs:
    resource_class: large
    steps:
      - checkout # check out the code in the project directory
-      - run: make V=1 -j8 release | .circleci/cat_ignore_eagain
+      - run: make V=1 -j8 release 2>&1 | .circleci/cat_ignore_eagain
      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
      - install-gflags
-      - run: make V=1 -j8 release | .circleci/cat_ignore_eagain
+      - run: make V=1 -j8 release 2>&1 | .circleci/cat_ignore_eagain
      - run: ./db_stress --version # ensure with gflags
      - post-steps

@@ -229,11 +228,11 @@ jobs:
    steps:
      - checkout # check out the code in the project directory
      - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
+      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench 2>&1 | .circleci/cat_ignore_eagain
      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
      - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
+      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench 2>&1 | .circleci/cat_ignore_eagain
      - run: ./db_stress --version # ensure with gflags

  build-linux-lite:
@@ -243,7 +242,7 @@ jobs:
    steps:
      - pre-steps
      - install-gflags
-      - run: LITE=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
+      - run: LITE=1 make V=1 J=32 -j32 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-lite-release:
@@ -252,10 +251,10 @@ jobs:
    resource_class: large
    steps:
      - checkout # check out the code in the project directory
-      - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
+      - run: LITE=1 make V=1 -j8 release 2>&1 | .circleci/cat_ignore_eagain
      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
      - install-gflags
-      - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
+      - run: LITE=1 make V=1 -j8 release 2>&1 | .circleci/cat_ignore_eagain
      - run: ./db_stress --version # ensure with gflags
      - post-steps

@@ -266,7 +265,7 @@ jobs:
    steps:
      - checkout # check out the code in the project directory
      - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev libtbb-dev
-      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all | .circleci/cat_ignore_eagain
+      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-clang10-asan:
@@ -277,7 +276,7 @@ jobs:
      - pre-steps
      - install-gflags
      - install-clang-10
-      - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
+      - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check 2>&1 | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
      - post-steps

  build-linux-clang10-mini-tsan:
@@ -288,7 +287,7 @@ jobs:
      - pre-steps
      - install-gflags
      - install-clang-10
-      - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out.
+      - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check 2>&1 | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out.
      - post-steps

  build-linux-clang10-ubsan:
@@ -299,7 +298,7 @@ jobs:
      - pre-steps
      - install-gflags
      - install-clang-10
-      - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
+      - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check 2>&1 | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
      - post-steps

  build-linux-clang10-clang-analyze:
@@ -311,7 +310,7 @@ jobs:
      - install-gflags
      - install-clang-10
      - run: sudo apt-get update -y && sudo apt-get install -y clang-tools-10
-      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
+      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze 2>&1 | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
      - post-steps

  build-linux-cmake:
@@ -322,7 +321,7 @@ jobs:
      - pre-steps
      - install-gflags
      - upgrade-cmake
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) | .circleci/cat_ignore_eagain
+      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-cmake-ubuntu-20:
@@ -333,7 +332,7 @@ jobs:
      - pre-steps
      - install-gflags
      - install-benchmark
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 && make microbench) | .circleci/cat_ignore_eagain
+      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 && make microbench) 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-unity-and-headers:
@@ -343,8 +342,8 @@ jobs:
    steps:
      - checkout # check out the code in the project directory
      - run: apt-get update -y && apt-get install -y libgflags-dev
-      - run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test | .circleci/cat_ignore_eagain
-      - run: make V=1 -j8 -k check-headers | .circleci/cat_ignore_eagain # could be moved to a different build
+      - run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test 2>&1 | .circleci/cat_ignore_eagain
+      - run: make V=1 -j8 -k check-headers 2>&1 | .circleci/cat_ignore_eagain # could be moved to a different build
      - post-steps

  build-linux-gcc-4_8-no_test_run:
@@ -354,7 +353,7 @@ jobs:
    steps:
      - pre-steps
      - run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev
-      - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
+      - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j8 all 2>&1 | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
      - post-steps

  build-linux-gcc-8-no_test_run:
@@ -364,7 +363,7 @@ jobs:
    steps:
      - pre-steps
      - run: sudo apt-get update -y && sudo apt-get install gcc-8 g++-8 libgflags-dev
-      - run: CC=gcc-8 CXX=g++-8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
+      - run: CC=gcc-8 CXX=g++-8 V=1 SKIP_LINK=1 make -j8 all 2>&1 | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
      - post-steps

  build-linux-gcc-9-no_test_run:
@@ -374,7 +373,7 @@ jobs:
    steps:
      - pre-steps
      - run: sudo apt-get update -y && sudo apt-get install gcc-9 g++-9 libgflags-dev
-      - run: CC=gcc-9 CXX=g++-9 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
+      - run: CC=gcc-9 CXX=g++-9 V=1 SKIP_LINK=1 make -j8 all 2>&1 | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
      - post-steps

  build-linux-gcc-10-cxx20-no_test_run:
@@ -384,7 +383,7 @@ jobs:
    steps:
      - pre-steps
      - run: sudo apt-get update -y && sudo apt-get install gcc-10 g++-10 libgflags-dev
-      - run: CC=gcc-10 CXX=g++-10 V=1 SKIP_LINK=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
+      - run: CC=gcc-10 CXX=g++-10 V=1 SKIP_LINK=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all 2>&1 | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
      - post-steps

  build-linux-gcc-11-no_test_run:
@@ -394,7 +393,7 @@ jobs:
    steps:
      - pre-steps
      - run: sudo apt-get update -y && sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get install gcc-11 g++-11 libgflags-dev
-      - run: CC=gcc-11 CXX=g++-11 V=1 SKIP_LINK=1 make -j16 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
+      - run: CC=gcc-11 CXX=g++-11 V=1 SKIP_LINK=1 make -j16 all 2>&1 | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
      - post-steps

  # This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing.
@@ -405,7 +404,7 @@ jobs:
    steps:
      - pre-steps
      - install-benchmark
-      - run: DEBUG_LEVEL=0 make microbench | .circleci/cat_ignore_eagain
+      - run: DEBUG_LEVEL=0 make microbench 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-windows:
@@ -494,10 +493,10 @@ jobs:
            which javac && javac -version
      - run:
          name: "Build RocksDBJava Shared Library"
-          command: make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
+          command: make V=1 J=8 -j8 rocksdbjava 2>&1 | .circleci/cat_ignore_eagain
      - run:
          name: "Test RocksDBJava"
-          command: make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
+          command: make V=1 J=8 -j8 jtest 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-java-static:
@@ -518,7 +517,7 @@ jobs:
            which javac && javac -version
      - run:
          name: "Build RocksDBJava Static Library"
-          command: make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
+          command: make V=1 J=8 -j8 rocksdbjavastatic 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-macos-java:
@@ -541,10 +540,10 @@ jobs:
            which javac && javac -version
      - run:
          name: "Build RocksDBJava Shared Library"
-          command: make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
+          command: make V=1 J=8 -j8 rocksdbjava 2>&1 | .circleci/cat_ignore_eagain
      - run:
          name: "Test RocksDBJava"
-          command: make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
+          command: make V=1 J=8 -j8 jtest 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-macos-java-static:
@@ -568,7 +567,7 @@ jobs:
            which javac && javac -version
      - run:
          name: "Build RocksDBJava Static Library"
-          command: make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
+          command: make V=1 J=8 -j8 rocksdbjavastatic 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-examples:
@@ -644,7 +643,7 @@ jobs:
    steps:
      - pre-steps
      - install-gflags
-      - run: make V=1 J=4 -j4 check | .circleci/cat_ignore_eagain
+      - run: make V=1 J=4 -j4 check 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-arm:
@@ -654,7 +653,7 @@ jobs:
    steps:
      - pre-steps
      - install-gflags
-      - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some | .circleci/cat_ignore_eagain
+      - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some 2>&1 | .circleci/cat_ignore_eagain
      - post-steps

  build-linux-arm-cmake-no_test_run:
@@ -652,6 +652,7 @@ set(SOURCES
        db/blob/blob_log_format.cc
        db/blob/blob_log_sequential_reader.cc
        db/blob/blob_log_writer.cc
+        db/blob/prefetch_buffer_collection.cc
        db/builder.cc
        db/c.cc
        db/column_family.cc
@@ -877,6 +878,7 @@ set(SOURCES
        utilities/env_timed.cc
        utilities/fault_injection_env.cc
        utilities/fault_injection_fs.cc
+        utilities/fault_injection_secondary_cache.cc
        utilities/leveldb_options/leveldb_options.cc
        utilities/memory/memory_util.cc
        utilities/merge_operators.cc
@@ -1,8 +1,27 @@
 # Rocksdb Change Log
-## Unreleased
+## 6.27.3 (2021-12-10)
+### Bug Fixes
+* Fixed a bug in TableOptions.prepopulate_block_cache which causes segmentation fault when used with TableOptions.partition_filters = true and TableOptions.cache_index_and_filter_blocks = true.
+* Fixed a bug affecting custom memtable factories which are not registered with the `ObjectRegistry`. The bug could result in failure to save the OPTIONS file.
+
+## 6.27.2 (2021-12-01)
+### Bug Fixes
+* Fixed a bug in rocksdb automatic implicit prefetching which got broken because of new feature adaptive_readahead and internal prefetching got disabled when iterator moves from one file to next.
+
+## 6.27.1 (2021-11-29)
+### Bug Fixes
+* Fixed a bug that could, with WAL enabled, cause backups, checkpoints, and `GetSortedWalFiles()` to fail randomly with an error like `IO error: 001234.log: No such file or directory`
+
+## 6.27.0 (2021-11-19)
 ### New Features
 * Added new ChecksumType kXXH3 which is faster than kCRC32c on almost all x86\_64 hardware.
 * Added a new online consistency check for BlobDB which validates that the number/total size of garbage blobs does not exceed the number/total size of all blobs in any given blob file.
+* Provided support for tracking per-sst user-defined timestamp information in MANIFEST.
+* Added new option "adaptive_readahead" in ReadOptions. For iterators, RocksDB does auto-readahead on noticing sequential reads and by enabling this option, readahead_size of current file (if reads are sequential) will be carried forward to next file instead of starting from the scratch at each level (except L0 level files). If reads are not sequential it will fall back to 8KB. This option is applicable only for RocksDB internal prefetch buffer and isn't supported with underlying file system prefetching.
+* Added the read count and read bytes related stats to Statistics for tiered storage hot, warm, and cold file reads.
+* Added an option to dynamically charge an updating estimated memory usage of block-based table building to block cache if block cache available. It currently only includes charging memory usage of constructing (new) Bloom Filter and Ribbon Filter to block cache. To enable this feature, set `BlockBasedTableOptions::reserve_table_builder_memory = true`.
+* Add a new API OnIOError in listener.h that notifies listeners when an IO error occurs during FileSystem operation along with filename, status etc.
+* Added compaction readahead support for blob files to the integrated BlobDB implementation, which can improve compaction performance when the database resides on higher-latency storage like HDDs or remote filesystems. Readahead can be configured using the column family option `blob_compaction_readahead_size`.

 ### Bug Fixes
 * Prevent a `CompactRange()` with `CompactRangeOptions::change_level == true` from possibly causing corruption to the LSM state (overlapping files within a level) when run in parallel with another manual compaction. Note that setting `force_consistency_checks == true` (the default) would cause the DB to enter read-only mode in this scenario and return `Status::Corruption`, rather than committing any corruption.
@@ -11,19 +30,30 @@
 * EventListener::OnTableFileCreated was previously called with OK status and file_size==0 in cases of no SST file contents written (because there was no content to add) and the empty file deleted before calling the listener. Now the status is Aborted.
 * Fixed a bug in CompactionIterator when write-preared transaction is used. Releasing earliest_snapshot during compaction may cause a SingleDelete to be output after a PUT of the same user key whose seq has been zeroed.
 * Added input sanitization on negative bytes passed into `GenericRateLimiter::Request`.
+* Fixed an assertion failure in CompactionIterator when write-prepared transaction is used. We prove that certain operations can lead to a Delete being followed by a SingleDelete (same user key). We can drop the SingleDelete.
+* Fixed a bug of timestamp-based GC which can cause all versions of a key under full_history_ts_low to be dropped. This bug will be triggered when some of the ikeys' timestamps are lower than full_history_ts_low, while others are newer.
+* In some cases outside of the DB read and compaction paths, SST block checksums are now checked where they were not before.
+* Explicitly check for and disallow the `BlockBasedTableOptions` if insertion into one of {`block_cache`, `block_cache_compressed`, `persistent_cache`} can show up in another of these. (RocksDB expects to be able to use the same key for different physical data among tiers.)
+* Users who configured a dedicated thread pool for bottommost compactions by explicitly adding threads to the `Env::Priority::BOTTOM` pool will no longer see RocksDB schedule automatic compactions exceeding the DB's compaction concurrency limit. For details on per-DB compaction concurrency limit, see API docs of `max_background_compactions` and `max_background_jobs`.
+* Fixed a bug of background flush thread picking more memtables to flush and prematurely advancing column family's log_number.
+* Fixed an assertion failure in ManifestTailer.

 ### Behavior Changes
 * `NUM_FILES_IN_SINGLE_COMPACTION` was only counting the first input level files, now it's including all input files.
-
-### Public Interface Change
-* When options.ttl is used with leveled compaction with compactinon priority kMinOverlappingRatio, files exceeding half of TTL value will be prioritized more, so that by the time TTL is reached, fewer extra compactions will be scheduled to clear them up. At the same time, when compacting files with data older than half of TTL, output files may be cut off based on those files' boundaries, in order for the early TTL compaction to work properly.
+* `TransactionUtil::CheckKeyForConflicts` can also perform conflict-checking based on user-defined timestamps in addition to sequence numbers.
+* Removed `GenericRateLimiter`'s minimum refill bytes per period previously enforced.

 ### Public API change
+* When options.ttl is used with leveled compaction with compactinon priority kMinOverlappingRatio, files exceeding half of TTL value will be prioritized more, so that by the time TTL is reached, fewer extra compactions will be scheduled to clear them up. At the same time, when compacting files with data older than half of TTL, output files may be cut off based on those files' boundaries, in order for the early TTL compaction to work properly.
 * Made FileSystem extend the Customizable class and added a CreateFromString method.  Implementations need to be registered with the ObjectRegistry and to implement a Name() method in order to be created via this method.
 * Clarified in API comments that RocksDB is not exception safe for callbacks and custom extensions. An exception propagating into RocksDB can lead to undefined behavior, including data loss, unreported corruption, deadlocks, and more.
 * Marked `WriteBufferManager` as `final` because it is not intended for extension.
+* Removed unimportant implementation details from table_properties.h
 * Add API `FSDirectory::FsyncWithDirOptions()`, which provides extra information like directory fsync reason in `DirFsyncOptions`. File system like btrfs is using that to skip directory fsync for creating a new file, or when renaming a file, fsync the target file instead of the directory, which improves the `DB::Open()` speed by ~20%.
 * `DB::Open()` is not going be blocked by obsolete file purge if `DBOptions::avoid_unnecessary_blocking_io` is set to true.
+* In builds where glibc provides `gettid()`, info log ("LOG" file) lines now print a system-wide thread ID from `gettid()` instead of the process-local `pthread_self()`. For all users, the thread ID format is changed from hexadecimal to decimal integer.
+* In builds where glibc provides `pthread_setname_np()`, the background thread names no longer contain an ID suffix. For example, "rocksdb:bottom7" (and all other threads in the `Env::Priority::BOTTOM` pool) are now named "rocksdb:bottom". Previously large thread pools could breach the name size limit (e.g., naming "rocksdb:bottom10" would fail).
+* Deprecating `ReadOptions::iter_start_seqnum` and `DBOptions::preserve_deletes`, please try using user defined timestamp feature instead. The options will be removed in a future release, currently it logs a warning message when using.

 ### Performance Improvements
 * Released some memory related to filter construction earlier in `BlockBasedTableBuilder` for `FullFilter` and `PartitionedFilter` case (#9070)
@@ -39,6 +69,7 @@
 * Fixed a bug where stalled writes would remain stalled forever after the user calls `WriteBufferManager::SetBufferSize()` with `new_size == 0` to dynamically disable memory limiting.
 * Make `DB::close()` thread-safe.
 * Fix a bug in atomic flush where one bg flush thread will wait forever for a preceding bg flush thread to commit its result to MANIFEST but encounters an error which is mapped to a soft error (DB not stopped).
+* Fix a bug in `BackupEngine` where some internal callers of `GenericRateLimiter::Request()` do not honor `bytes <= GetSingleBurstBytes()`.

 ### New Features
 * Print information about blob files when using "ldb list_live_files_metadata"
@@ -306,6 +306,13 @@ ifdef COMPILE_WITH_ASAN
 	EXEC_LDFLAGS += -fsanitize=address
 	PLATFORM_CCFLAGS += -fsanitize=address
 	PLATFORM_CXXFLAGS += -fsanitize=address
+ifeq ($(LIB_MODE),shared)
+ifdef USE_CLANG
+# Fix false ODR violation; see https://github.com/google/sanitizers/issues/1017
+	EXEC_LDFLAGS += -mllvm -asan-use-private-alias=1
+	PLATFORM_CXXFLAGS += -mllvm -asan-use-private-alias=1
+endif
+endif
 endif

 # TSAN doesn't work well with jemalloc. If we're compiling with TSAN, we should use regular malloc.
@@ -508,6 +515,7 @@ STRESS_OBJECTS =  $(patsubst %.cc, $(OBJ_DIR)/%.o, $(STRESS_LIB_SOURCES))
 ALL_SOURCES  = $(filter-out util/build_version.cc, $(LIB_SOURCES)) $(TEST_LIB_SOURCES) $(MOCK_LIB_SOURCES) $(GTEST_DIR)/gtest/gtest-all.cc
 ALL_SOURCES += $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(CACHE_BENCH_LIB_SOURCES) $(ANALYZER_LIB_SOURCES) $(STRESS_LIB_SOURCES)
 ALL_SOURCES += $(TEST_MAIN_SOURCES) $(TOOL_MAIN_SOURCES) $(BENCH_MAIN_SOURCES)
+ALL_SOURCES += $(ROCKSDB_PLUGIN_SOURCES)

 TESTS = $(patsubst %.cc, %, $(notdir $(TEST_MAIN_SOURCES)))
 TESTS += $(patsubst %.c, %, $(notdir $(TEST_MAIN_SOURCES_C)))
@@ -824,14 +832,11 @@ parallel_tests = $(patsubst %,parallel_%,$(PARALLEL_TEST))
 $(parallel_tests): $(PARALLEL_TEST)
 	$(AM_V_at)TEST_BINARY=$(patsubst parallel_%,%,$@); \
  TEST_NAMES=` \
-    ./$$TEST_BINARY --gtest_list_tests \
-    | perl -n \
-      -e 's/ *\#.*//;' \
-      -e '/^(\s*)(\S+)/; !$$1 and do {$$p=$$2; break};'	\
-      -e 'print qq! $$p$$2!'`; \
+    (./$$TEST_BINARY --gtest_list_tests || echo "  $${TEST_BINARY}__list_tests_failure") \
+    | awk '/^[^ ]/ { prefix = $$1 } /^[ ]/ { print prefix $$1 }'`; \
+	echo "  Generating parallel test scripts for $$TEST_BINARY"; \
 	for TEST_NAME in $$TEST_NAMES; do \
 		TEST_SCRIPT=t/run-$$TEST_BINARY-$${TEST_NAME//\//-}; \
-		echo "  GEN     " $$TEST_SCRIPT; \
    printf '%s\n' \
      '#!/bin/sh' \
      "d=\$(TMPD)$$TEST_SCRIPT" \
@@ -878,12 +883,18 @@ J ?= 100%

 # Use this regexp to select the subset of tests whose names match.
 tests-regexp = .
-EXCLUDE_TESTS_REGEX ?= "^$"
+EXCLUDE_TESTS_REGEX ?= "^$$"

 ifeq ($(PRINT_PARALLEL_OUTPUTS), 1)
-	parallel_com = '{}'
+	parallel_redir =
+else ifeq ($(QUIET_PARALLEL_TESTS), 1)
+	parallel_redir = >& t/$(test_log_prefix)log-{/}
 else
-	parallel_com = '{} >& t/log-{/}'
+# Default: print failure output only, as it happens
+# Note: gnu_parallel --eta is now always used, but has been modified to provide
+# only infrequent updates when not connected to a terminal. (CircleCI will
+# kill a job if no output for 10min.)
+	parallel_redir = >& t/$(test_log_prefix)log-{/} || bash -c "cat t/$(test_log_prefix)log-{/}; exit $$?"
 endif

 .PHONY: check_0
@@ -892,7 +903,6 @@ check_0:
 	printf '%s\n' ''						\
 	  'To monitor subtest <duration,pass/fail,name>,'		\
 	  '  run "make watch-log" in a separate window' '';		\
-	test -t 1 && eta=--eta || eta=; \
 	{ \
 		printf './%s\n' $(filter-out $(PARALLEL_TEST),$(TESTS)); \
 		find t -name 'run-*' -print; \
@@ -900,7 +910,7 @@ check_0:
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
 	  | grep -E -v '$(EXCLUDE_TESTS_REGEX)'					\
-	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu  $(parallel_com) ; \
+	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG --eta --gnu '{} $(parallel_redir)' ; \
 	parallel_retcode=$$? ; \
 	awk '{ if ($$7 != 0 || $$8 != 0) { if ($$7 == "Exitval") { h = $$0; } else { if (!f) print h; print; f = 1 } } } END { if(f) exit 1; }' < LOG ; \
 	awk_retcode=$$?; \
@@ -909,12 +919,12 @@ check_0:
 valgrind-exclude-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized|MySQLStyleTransactionTest.TransactionStressTest

 .PHONY: valgrind_check_0
+valgrind_check_0: test_log_prefix := valgrind_
 valgrind_check_0:
 	$(AM_V_GEN)export TEST_TMPDIR=$(TMPD);				\
 	printf '%s\n' ''						\
 	  'To monitor subtest <duration,pass/fail,name>,'		\
 	  '  run "make watch-log" in a separate window' '';		\
-	test -t 1 && eta=--eta || eta=;					\
 	{								\
 	  printf './%s\n' $(filter-out $(PARALLEL_TEST) %skiplist_test options_settable_test, $(TESTS));		\
 	  find t -name 'run-*' -print; \
@@ -922,9 +932,9 @@ valgrind_check_0:
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
 	  | grep -E -v '$(valgrind-exclude-regexp)'					\
-	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu \
-	  '(if [[ "{}" == "./"* ]] ; then $(DRIVER) {}; else {}; fi) ' \
-	  '>& t/valgrind_log-{/}'
+	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG --eta --gnu \
+	  '(if [[ "{}" == "./"* ]] ; then $(DRIVER) {}; else {}; fi) \
+	  $(parallel_redir)' \

 CLEAN_FILES += t LOG $(TMPD)

@@ -978,15 +988,27 @@ check_some: $(ROCKSDBTESTS_SUBSET)
 ldb_tests: ldb
 	$(PYTHON) tools/ldb_test.py

-crash_test: whitebox_crash_test blackbox_crash_test
+crash_test:
+# Do not parallelize
+	$(MAKE) whitebox_crash_test
+	$(MAKE) blackbox_crash_test

-crash_test_with_atomic_flush: whitebox_crash_test_with_atomic_flush blackbox_crash_test_with_atomic_flush
+crash_test_with_atomic_flush:
+# Do not parallelize
+	$(MAKE) whitebox_crash_test_with_atomic_flush
+	$(MAKE) blackbox_crash_test_with_atomic_flush

-crash_test_with_txn: whitebox_crash_test_with_txn blackbox_crash_test_with_txn
+crash_test_with_txn:
+# Do not parallelize
+	$(MAKE) whitebox_crash_test_with_txn
+	$(MAKE) blackbox_crash_test_with_txn

 crash_test_with_best_efforts_recovery: blackbox_crash_test_with_best_efforts_recovery

-crash_test_with_ts: whitebox_crash_test_with_ts blackbox_crash_test_with_ts
+crash_test_with_ts:
+# Do not parallelize
+	$(MAKE) whitebox_crash_test_with_ts
+	$(MAKE) blackbox_crash_test_with_ts

 blackbox_crash_test: db_stress
 	$(PYTHON) -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
@@ -2246,20 +2268,26 @@ rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentr

 rocksdbjavastaticpublishdocker: rocksdbjavastaticreleasedocker rocksdbjavastaticpublishcentral

+ROCKSDB_JAVA_RELEASE_CLASSIFIERS = javadoc sources linux64 linux32 linux64-musl linux32-musl osx win64
+
 rocksdbjavastaticpublishcentral: rocksdbjavageneratepom
-	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-javadoc.jar -Dclassifier=javadoc
-	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-sources.jar -Dclassifier=sources
-	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-linux64.jar -Dclassifier=linux64
-	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-linux32.jar -Dclassifier=linux32
-	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-linux64-musl.jar -Dclassifier=linux64-musl
-	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-linux32-musl.jar -Dclassifier=linux32-musl
-	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-osx.jar -Dclassifier=osx
-	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-win64.jar -Dclassifier=win64
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar
+	$(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/pom.xml -Dfile=java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -Dclassifier=$(classifier);)

 rocksdbjavageneratepom:
 	cd java;cat pom.xml.template | sed 's/\$${ROCKSDB_JAVA_VERSION}/$(ROCKSDB_JAVA_VERSION)/' > pom.xml

+rocksdbjavastaticnexusbundlejar: rocksdbjavageneratepom
+	openssl sha1 -r java/pom.xml | awk '{  print $$1 }' > java/target/pom.xml.sha1
+	openssl sha1 -r java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar | awk '{  print $$1 }' > java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar.sha1
+	$(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), openssl sha1 -r java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar | awk '{  print $$1 }' > java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.sha1;)
+	gpg --yes --output java/target/pom.xml.asc -ab java/pom.xml
+	gpg --yes -ab java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar
+	$(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), gpg --yes -ab java/target/rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar;)
+	jar cvf java/target/nexus-bundle-rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar -C java pom.xml -C java/target pom.xml.sha1 -C java/target pom.xml.asc -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar.sha1 -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar.asc
+	 $(foreach classifier, $(ROCKSDB_JAVA_RELEASE_CLASSIFIERS), jar uf java/target/nexus-bundle-rocksdbjni-$(ROCKSDB_JAVA_VERSION).jar -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.sha1 -C java/target rocksdbjni-$(ROCKSDB_JAVA_VERSION)-$(classifier).jar.asc;)
+
+
 # A version of each $(LIBOBJECTS) compiled with -fPIC

 jl/%.o: %.cc
@@ -159,6 +159,7 @@ cpp_library(
        "db/blob/blob_log_format.cc",
        "db/blob/blob_log_sequential_reader.cc",
        "db/blob/blob_log_writer.cc",
+        "db/blob/prefetch_buffer_collection.cc",
        "db/builder.cc",
        "db/c.cc",
        "db/column_family.cc",
@@ -395,6 +396,7 @@ cpp_library(
        "utilities/env_timed.cc",
        "utilities/fault_injection_env.cc",
        "utilities/fault_injection_fs.cc",
+        "utilities/fault_injection_secondary_cache.cc",
        "utilities/leveldb_options/leveldb_options.cc",
        "utilities/memory/memory_util.cc",
        "utilities/merge_operators.cc",
@@ -486,6 +488,7 @@ cpp_library(
        "db/blob/blob_log_format.cc",
        "db/blob/blob_log_sequential_reader.cc",
        "db/blob/blob_log_writer.cc",
+        "db/blob/prefetch_buffer_collection.cc",
        "db/builder.cc",
        "db/c.cc",
        "db/column_family.cc",
@@ -722,6 +725,7 @@ cpp_library(
        "utilities/env_timed.cc",
        "utilities/fault_injection_env.cc",
        "utilities/fault_injection_fs.cc",
+        "utilities/fault_injection_secondary_cache.cc",
        "utilities/leveldb_options/leveldb_options.cc",
        "utilities/memory/memory_util.cc",
        "utilities/merge_operators.cc",
@@ -675,7 +675,8 @@ else
    # For portability compile for macOS 10.12 (2016) or newer
    COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.12"
    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.12"
-    PLATFORM_SHARED_LDFLAGS="$PLATFORM_SHARED_LDFLAGS -mmacosx-version-min=10.12"
+    # -mmacosx-version-min must come first here.
+    PLATFORM_SHARED_LDFLAGS="-mmacosx-version-min=10.12 $PLATFORM_SHARED_LDFLAGS"
    PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.12"
    JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.12"
    JAVA_STATIC_DEPS_LDFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
@@ -1561,6 +1561,7 @@ sub save_stdin_stdout_stderr {
 	::die_bug("Can't dup STDERR: $!");
    open $Global::original_stdin, "<&", "STDIN" or
 	::die_bug("Can't dup STDIN: $!");
+    $Global::is_terminal = (-t $Global::original_stderr);
 }

 sub enough_file_handles {
@@ -1840,12 +1841,17 @@ sub start_another_job {
 }
 }

+$opt::min_progress_interval = 0;
+
 sub init_progress {
    # Uses:
    #   $opt::bar
    # Returns:
    #   list of computers for progress output
    $|=1;
+    if (not $Global::is_terminal) {
+      $opt::min_progress_interval = 30;
+    }
    if($opt::bar) {
 	return("","");
    }
@@ -1870,6 +1876,9 @@ sub drain_job_queue {
    }
    my $last_header="";
    my $sleep = 0.2;
+    my $last_left = 1000000000;
+    my $last_progress_time = 0;
+    my $ps_reported = 0;
    do {
        while($Global::total_running > 0) {
            debug($Global::total_running, "==", scalar
@@ -1880,14 +1889,38 @@ sub drain_job_queue {
 		    close $job->fh(0,"w");
 		}
 	    }
-            if($opt::progress) {
+            # When not connected to terminal, assume CI (e.g. CircleCI). In
+            # that case we want occasional progress output to prevent abort
+            # due to timeout with no output, but we also need to stop sending
+            # progress output if there has been no actual progress, so that
+            # the job can time out appropriately (CirecleCI: 10m) in case of
+            # a hung test. But without special output, it is extremely
+            # annoying to diagnose which test is hung, so we add that using
+            # `ps` below.
+            if($opt::progress and
+               ($Global::is_terminal or (time() - $last_progress_time) >= 30)) {
                my %progress = progress();
                if($last_header ne $progress{'header'}) {
                    print $Global::original_stderr "\n", $progress{'header'}, "\n";
                    $last_header = $progress{'header'};
                }
-                print $Global::original_stderr "\r",$progress{'status'};
-		flush $Global::original_stderr;
+                if ($Global::is_terminal) {
+                    print $Global::original_stderr "\r",$progress{'status'};
+                }
+                if ($last_left > $Global::left) {
+                    if (not $Global::is_terminal) {
+                        print $Global::original_stderr $progress{'status'},"\n";
+                    }
+                    $last_progress_time = time();
+                    $ps_reported = 0;
+                } elsif (not $ps_reported and (time() - $last_progress_time) >= 60) {
+                    # No progress in at least 60 seconds: run ps
+                    print $Global::original_stderr "\n";
+                    system("ps", "-wf");
+                    $ps_reported = 1;
+                }
+                $last_left = $Global::left;
+                flush $Global::original_stderr;
            }
 	    if($Global::total_running < $Global::max_jobs_running
 	       and not $Global::JobQueue->empty()) {
@@ -1921,7 +1954,7 @@ sub drain_job_queue {
 	     not $Global::start_no_new_jobs and not $Global::JobQueue->empty());
    if($opt::progress) {
 	my %progress = progress();
-	print $Global::original_stderr "\r", $progress{'status'}, "\n";
+	print $Global::original_stderr $opt::progress_sep, $progress{'status'}, "\n";
 	flush $Global::original_stderr;
    }
 }
@@ -1954,10 +1987,11 @@ sub progress {
    my $eta = "";
    my ($status,$header)=("","");
    if($opt::eta) {
-	my($total, $completed, $left, $pctcomplete, $avgtime, $this_eta) =
-	    compute_eta();
-	$eta = sprintf("ETA: %ds Left: %d AVG: %.2fs  ",
-		       $this_eta, $left, $avgtime);
+      my($total, $completed, $left, $pctcomplete, $avgtime, $this_eta) =
+        compute_eta();
+      $eta = sprintf("ETA: %ds Left: %d AVG: %.2fs  ",
+        $this_eta, $left, $avgtime);
+      $Global::left = $left;
    }
    my $termcols = terminal_columns();
    my @workers = sort keys %Global::host;
@@ -20,6 +20,7 @@ std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToCamelString{{
    "OtherBlock",
    "WriteBuffer",
    "CompressionDictionaryBuildingBuffer",
+    "FilterConstruction",
    "Misc",
 }};

@@ -32,6 +33,7 @@ std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToHyphenString{{
    "other-block",
    "write-buffer",
    "compression-dictionary-building-buffer",
+    "filter-construction",
    "misc",
 }};

@@ -7,6 +7,8 @@

 #include <array>
 #include <cstdint>
+#include <memory>
+#include <type_traits>
 #include <unordered_map>

 #include "rocksdb/cache.h"
@@ -34,6 +36,9 @@ enum class CacheEntryRole {
  // BlockBasedTableBuilder reservations to account for
  // compression dictionary building buffer's memory usage
  kCompressionDictionaryBuildingBuffer,
+  // Filter reservations to account for
+  // (new) bloom and ribbon filter construction's memory usage
+  kFilterConstruction,
  // Default bucket, for miscellaneous cache entries. Do not use for
  // entries that could potentially add up to large usage.
  kMisc,
@@ -90,7 +95,9 @@ struct RegisteredDeleter {
  // These have global linkage to help ensure compiler optimizations do not
  // break uniqueness for each <T,R>
  static void Delete(const Slice& /* key */, void* value) {
-    delete static_cast<T*>(value);
+    // Supports T == Something[], unlike delete operator
+    std::default_delete<T>()(
+        static_cast<typename std::remove_extent<T>::type*>(value));
  }
 };

@@ -98,9 +105,9 @@ template <CacheEntryRole R>
 struct RegisteredNoopDeleter {
  RegisteredNoopDeleter() { RegisterCacheDeleterRole(Delete, R); }

-  static void Delete(const Slice& /* key */, void* value) {
-    (void)value;
-    assert(value == nullptr);
+  static void Delete(const Slice& /* key */, void* /* value */) {
+    // Here was `assert(value == nullptr);` but we can also put pointers
+    // to static data in Cache, for testing at least.
  }
 };

@@ -23,7 +23,9 @@
 namespace ROCKSDB_NAMESPACE {
 CacheReservationManager::CacheReservationManager(std::shared_ptr<Cache> cache,
                                                 bool delayed_decrease)
-    : delayed_decrease_(delayed_decrease), cache_allocated_size_(0) {
+    : delayed_decrease_(delayed_decrease),
+      cache_allocated_size_(0),
+      memory_used_(0) {
  assert(cache != nullptr);
  cache_ = cache;
  std::memset(cache_key_, 0, kCacheKeyPrefixSize + kMaxVarint64Length);
@@ -39,6 +41,7 @@ CacheReservationManager::~CacheReservationManager() {
 template <CacheEntryRole R>
 Status CacheReservationManager::UpdateCacheReservation(
    std::size_t new_mem_used) {
+  memory_used_ = new_mem_used;
  std::size_t cur_cache_allocated_size =
      cache_allocated_size_.load(std::memory_order_relaxed);
  if (new_mem_used == cur_cache_allocated_size) {
@@ -76,6 +79,28 @@ template Status CacheReservationManager::UpdateCacheReservation<
 template Status CacheReservationManager::UpdateCacheReservation<
    CacheEntryRole::kMisc>(std::size_t new_mem_used);

+template <CacheEntryRole R>
+Status CacheReservationManager::MakeCacheReservation(
+    std::size_t incremental_memory_used,
+    std::unique_ptr<CacheReservationHandle<R>>* handle) {
+  assert(handle != nullptr);
+  Status s =
+      UpdateCacheReservation<R>(GetTotalMemoryUsed() + incremental_memory_used);
+  (*handle).reset(new CacheReservationHandle<R>(incremental_memory_used,
+                                                shared_from_this()));
+  return s;
+}
+
+template Status
+CacheReservationManager::MakeCacheReservation<CacheEntryRole::kMisc>(
+    std::size_t incremental_memory_used,
+    std::unique_ptr<CacheReservationHandle<CacheEntryRole::kMisc>>* handle);
+template Status CacheReservationManager::MakeCacheReservation<
+    CacheEntryRole::kFilterConstruction>(
+    std::size_t incremental_memory_used,
+    std::unique_ptr<
+        CacheReservationHandle<CacheEntryRole::kFilterConstruction>>* handle);
+
 template <CacheEntryRole R>
 Status CacheReservationManager::IncreaseCacheReservation(
    std::size_t new_mem_used) {
@@ -118,6 +143,10 @@ std::size_t CacheReservationManager::GetTotalReservedCacheSize() {
  return cache_allocated_size_.load(std::memory_order_relaxed);
 }

+std::size_t CacheReservationManager::GetTotalMemoryUsed() {
+  return memory_used_;
+}
+
 Slice CacheReservationManager::GetNextCacheKey() {
  // Calling this function will have the side-effect of changing the
  // underlying cache_key_ that is shared among other keys generated from this
@@ -128,4 +157,36 @@ Slice CacheReservationManager::GetNextCacheKey() {
      EncodeVarint64(cache_key_ + kCacheKeyPrefixSize, next_cache_key_id_++);
  return Slice(cache_key_, static_cast<std::size_t>(end - cache_key_));
 }
-}  // namespace ROCKSDB_NAMESPACE
+
+template <CacheEntryRole R>
+Cache::DeleterFn CacheReservationManager::TEST_GetNoopDeleterForRole() {
+  return GetNoopDeleterForRole<R>();
+}
+
+template Cache::DeleterFn CacheReservationManager::TEST_GetNoopDeleterForRole<
+    CacheEntryRole::kFilterConstruction>();
+
+template <CacheEntryRole R>
+CacheReservationHandle<R>::CacheReservationHandle(
+    std::size_t incremental_memory_used,
+    std::shared_ptr<CacheReservationManager> cache_res_mgr)
+    : incremental_memory_used_(incremental_memory_used) {
+  assert(cache_res_mgr != nullptr);
+  cache_res_mgr_ = cache_res_mgr;
+}
+
+template <CacheEntryRole R>
+CacheReservationHandle<R>::~CacheReservationHandle() {
+  assert(cache_res_mgr_ != nullptr);
+  assert(cache_res_mgr_->GetTotalMemoryUsed() >= incremental_memory_used_);
+
+  Status s = cache_res_mgr_->UpdateCacheReservation<R>(
+      cache_res_mgr_->GetTotalMemoryUsed() - incremental_memory_used_);
+  s.PermitUncheckedError();
+}
+
+// Explicitly instantiate templates for "CacheEntryRole" values we use.
+// This makes it possible to keep the template definitions in the .cc file.
+template class CacheReservationHandle<CacheEntryRole::kMisc>;
+template class CacheReservationHandle<CacheEntryRole::kFilterConstruction>;
+}  // namespace ROCKSDB_NAMESPACE
@@ -24,10 +24,16 @@

 namespace ROCKSDB_NAMESPACE {

+template <CacheEntryRole R>
+class CacheReservationHandle;
+
 // CacheReservationManager is for reserving cache space for the memory used
 // through inserting/releasing dummy entries in the cache.
-// This class is not thread-safe.
-class CacheReservationManager {
+//
+// This class is NOT thread-safe, except that GetTotalReservedCacheSize()
+// can be called without external synchronization.
+class CacheReservationManager
+    : public std::enable_shared_from_this<CacheReservationManager> {
 public:
  // Construct a CacheReservationManager
  // @param cache The cache where dummy entries are inserted and released for
@@ -52,9 +58,13 @@ class CacheReservationManager {

  template <CacheEntryRole R>

+  // One of the two ways of reserving/releasing cache,
+  // see CacheReservationManager::MakeCacheReservation() for the other.
+  // Use ONLY one of them to prevent unexpected behavior.
+  //
  // Insert and release dummy entries in the cache to
-  // match the size of total dummy entries with the smallest multiple of
-  // kSizeDummyEntry that is greater than or equal to new_mem_used
+  // match the size of total dummy entries with the least multiple of
+  // kSizeDummyEntry greater than or equal to new_mem_used
  //
  // Insert dummy entries if new_memory_used > cache_allocated_size_;
  //
@@ -68,15 +78,84 @@ class CacheReservationManager {
  // is set true.
  //
  // @param new_memory_used The number of bytes used by new memory
+  //        The most recent new_memoy_used passed in will be returned
+  //        in GetTotalMemoryUsed() even when the call return non-ok status.
+  //
+  //        Since the class is NOT thread-safe, external synchronization on the
+  //        order of calling UpdateCacheReservation() is needed if you want
+  //        GetTotalMemoryUsed() indeed returns the latest memory used.
+  //
  // @return On inserting dummy entries, it returns Status::OK() if all dummy
-  // entry insertions succeed. Otherwise, it returns the first non-ok status;
-  // On releasing dummy entries, it always returns Status::OK().
-  // On keeping dummy entries the same, it always returns Status::OK().
+  //         entry insertions succeed.
+  //         Otherwise, it returns the first non-ok status;
+  //         On releasing dummy entries, it always returns Status::OK().
+  //         On keeping dummy entries the same, it always returns Status::OK().
  Status UpdateCacheReservation(std::size_t new_memory_used);
+
+  // One of the two ways of reserving/releasing cache,
+  // see CacheReservationManager::UpdateCacheReservation() for the other.
+  // Use ONLY one of them to prevent unexpected behavior.
+  //
+  // Insert dummy entries in the cache for the incremental memory usage
+  // to match the size of total dummy entries with the least multiple of
+  // kSizeDummyEntry greater than or equal to the total memory used.
+  //
+  // A CacheReservationHandle is returned as an output parameter.
+  // The reserved dummy entries are automatically released on the destruction of
+  // this handle, which achieves better RAII per cache reservation.
+  //
+  // WARNING: Deallocate all the handles of the CacheReservationManager object
+  //          before deallocating the object to prevent unexpected behavior.
+  //
+  // @param incremental_memory_used The number of bytes increased in memory
+  //        usage.
+  //
+  //        Calling GetTotalMemoryUsed() afterward will return the total memory
+  //        increased by this number, even when calling MakeCacheReservation()
+  //        returns non-ok status.
+  //
+  //        Since the class is NOT thread-safe, external synchronization in
+  //        calling MakeCacheReservation() is needed if you want
+  //        GetTotalMemoryUsed() indeed returns the latest memory used.
+  //
+  // @param handle An pointer to std::unique_ptr<CacheReservationHandle<R>> that
+  //        manages the lifetime of the handle and its cache reservation.
+  //
+  // @return It returns Status::OK() if all dummy
+  //         entry insertions succeed.
+  //         Otherwise, it returns the first non-ok status;
+  //
+  // REQUIRES: handle != nullptr
+  // REQUIRES: The CacheReservationManager object is NOT managed by
+  //           std::unique_ptr as CacheReservationHandle needs to
+  //           shares ownership to the CacheReservationManager object.
+  template <CacheEntryRole R>
+  Status MakeCacheReservation(
+      std::size_t incremental_memory_used,
+      std::unique_ptr<CacheReservationHandle<R>> *handle);
+
+  // Return the size of the cache (which is a multiple of kSizeDummyEntry)
+  // successfully reserved by calling UpdateCacheReservation().
+  //
+  // When UpdateCacheReservation() returns non-ok status,
+  // calling GetTotalReservedCacheSize() after that might return a slightly
+  // smaller number than the actual reserved cache size due to
+  // the returned number will always be a multiple of kSizeDummyEntry
+  // and cache full might happen in the middle of inserting a dummy entry.
  std::size_t GetTotalReservedCacheSize();

+  // Return the latest total memory used indicated by the most recent call of
+  // UpdateCacheReservation(std::size_t new_memory_used);
+  std::size_t GetTotalMemoryUsed();
+
  static constexpr std::size_t GetDummyEntrySize() { return kSizeDummyEntry; }

+  // For testing only - it is to help ensure the NoopDeleterForRole<R>
+  // accessed from CacheReservationManager and the one accessed from the test
+  // are from the same translation units
+  template <CacheEntryRole R>
+  static Cache::DeleterFn TEST_GetNoopDeleterForRole();
+
 private:
  static constexpr std::size_t kSizeDummyEntry = 256 * 1024;
  // The key will be longer than keys for blocks in SST files so they won't
@@ -92,9 +171,27 @@ class CacheReservationManager {
  std::shared_ptr<Cache> cache_;
  bool delayed_decrease_;
  std::atomic<std::size_t> cache_allocated_size_;
+  std::size_t memory_used_;
  std::vector<Cache::Handle *> dummy_handles_;
  std::uint64_t next_cache_key_id_ = 0;
  // The non-prefix part will be updated according to the ID to use.
  char cache_key_[kCacheKeyPrefixSize + kMaxVarint64Length];
 };
+
+// CacheReservationHandle is for managing the lifetime of a cache reservation
+// This class is NOT thread-safe
+template <CacheEntryRole R>
+class CacheReservationHandle {
+ public:
+  // REQUIRES: cache_res_mgr != nullptr
+  explicit CacheReservationHandle(
+      std::size_t incremental_memory_used,
+      std::shared_ptr<CacheReservationManager> cache_res_mgr);
+
+  ~CacheReservationHandle();
+
+ private:
+  std::size_t incremental_memory_used_;
+  std::shared_ptr<CacheReservationManager> cache_res_mgr_;
+};
 }  // namespace ROCKSDB_NAMESPACE
@@ -22,16 +22,15 @@
 namespace ROCKSDB_NAMESPACE {
 class CacheReservationManagerTest : public ::testing::Test {
 protected:
-  static constexpr std::size_t kOneGigabyte = 1024 * 1024 * 1024;
-  static constexpr int kNumShardBits = 0;  // 2^0 shard
-
  static constexpr std::size_t kSizeDummyEntry =
      CacheReservationManager::GetDummyEntrySize();
+  static constexpr std::size_t kCacheCapacity = 4096 * kSizeDummyEntry;
+  static constexpr int kNumShardBits = 0;  // 2^0 shard
  static const std::size_t kCacheKeyPrefixSize =
      BlockBasedTable::kMaxCacheKeyPrefixSize + kMaxVarint64Length;
  static constexpr std::size_t kMetaDataChargeOverhead = 10000;

-  std::shared_ptr<Cache> cache = NewLRUCache(kOneGigabyte, kNumShardBits);
+  std::shared_ptr<Cache> cache = NewLRUCache(kCacheCapacity, kNumShardBits);
  std::unique_ptr<CacheReservationManager> test_cache_rev_mng;

  CacheReservationManagerTest() {
@@ -82,6 +81,7 @@ TEST_F(CacheReservationManagerTest, KeepCacheReservationTheSame) {
  ASSERT_EQ(s, Status::OK());
  ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
            1 * kSizeDummyEntry);
+  ASSERT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used);
  std::size_t initial_pinned_usage = cache->GetPinnedUsage();
  ASSERT_GE(initial_pinned_usage, 1 * kSizeDummyEntry);
  ASSERT_LT(initial_pinned_usage,
@@ -97,6 +97,9 @@ TEST_F(CacheReservationManagerTest, KeepCacheReservationTheSame) {
            1 * kSizeDummyEntry)
      << "Failed to bookkeep correctly when new_mem_used equals to current "
         "cache reservation";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly when new_mem_used "
+         "equals to current cache reservation";
  EXPECT_EQ(cache->GetPinnedUsage(), initial_pinned_usage)
      << "Failed to keep underlying dummy entries the same when new_mem_used "
         "equals to current cache reservation";
@@ -114,6 +117,8 @@ TEST_F(CacheReservationManagerTest,
  EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
            2 * kSizeDummyEntry)
      << "Failed to bookkeep cache reservation increase correctly";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly";
  EXPECT_GE(cache->GetPinnedUsage(), 2 * kSizeDummyEntry)
      << "Failed to increase underlying dummy entries in cache correctly";
  EXPECT_LT(cache->GetPinnedUsage(),
@@ -133,6 +138,8 @@ TEST_F(CacheReservationManagerTest,
  EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
            3 * kSizeDummyEntry)
      << "Failed to bookkeep cache reservation increase correctly";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly";
  EXPECT_GE(cache->GetPinnedUsage(), 3 * kSizeDummyEntry)
      << "Failed to increase underlying dummy entries in cache correctly";
  EXPECT_LT(cache->GetPinnedUsage(),
@@ -142,21 +149,22 @@ TEST_F(CacheReservationManagerTest,

 TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
     IncreaseCacheReservationOnFullCache) {
-  constexpr std::size_t kOneMegabyte = 1024 * 1024;
-  constexpr std::size_t kOneGigabyte = 1024 * 1024 * 1024;
+  ;
  constexpr std::size_t kSizeDummyEntry =
      CacheReservationManager::GetDummyEntrySize();
+  constexpr std::size_t kSmallCacheCapacity = 4 * kSizeDummyEntry;
+  constexpr std::size_t kBigCacheCapacity = 4096 * kSizeDummyEntry;
  constexpr std::size_t kMetaDataChargeOverhead = 10000;

  LRUCacheOptions lo;
-  lo.capacity = kOneMegabyte;
+  lo.capacity = kSmallCacheCapacity;
  lo.num_shard_bits = 0;  // 2^0 shard
  lo.strict_capacity_limit = true;
  std::shared_ptr<Cache> cache = NewLRUCache(lo);
  std::unique_ptr<CacheReservationManager> test_cache_rev_mng(
      new CacheReservationManager(cache));

-  std::size_t new_mem_used = kOneMegabyte + 1;
+  std::size_t new_mem_used = kSmallCacheCapacity + 1;
  Status s =
      test_cache_rev_mng
          ->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
@@ -168,18 +176,21 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
            1 * kSizeDummyEntry)
      << "Failed to bookkeep correctly before cache resevation failure happens "
         "due to full cache";
-  EXPECT_LE(test_cache_rev_mng->GetTotalReservedCacheSize(), kOneMegabyte)
+  EXPECT_LE(test_cache_rev_mng->GetTotalReservedCacheSize(),
+            kSmallCacheCapacity)
      << "Failed to bookkeep correctly (i.e, bookkeep only successful dummy "
         "entry insertions) when encountering cache resevation failure due to "
         "full cache";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly";
  EXPECT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry)
      << "Failed to insert underlying dummy entries correctly when "
         "encountering cache resevation failure due to full cache";
-  EXPECT_LE(cache->GetPinnedUsage(), kOneMegabyte)
+  EXPECT_LE(cache->GetPinnedUsage(), kSmallCacheCapacity)
      << "Failed to insert underlying dummy entries correctly when "
         "encountering cache resevation failure due to full cache";

-  new_mem_used = kOneMegabyte / 2;  // 2 dummy entries
+  new_mem_used = kSmallCacheCapacity / 2;  // 2 dummy entries
  s = test_cache_rev_mng
          ->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
              new_mem_used);
@@ -190,6 +201,8 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
            2 * kSizeDummyEntry)
      << "Failed to bookkeep cache reservation decrease correctly after "
         "encountering cache reservation due to full cache";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly";
  EXPECT_GE(cache->GetPinnedUsage(), 2 * kSizeDummyEntry)
      << "Failed to release underlying dummy entries correctly on cache "
         "reservation decrease after encountering cache resevation failure due "
@@ -201,7 +214,7 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
         "to full cache";

  // Create cache full again for subsequent tests
-  new_mem_used = kOneMegabyte + 1;
+  new_mem_used = kSmallCacheCapacity + 1;
  s = test_cache_rev_mng
          ->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
              new_mem_used);
@@ -212,21 +225,24 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
            1 * kSizeDummyEntry)
      << "Failed to bookkeep correctly before cache resevation failure happens "
         "due to full cache";
-  EXPECT_LE(test_cache_rev_mng->GetTotalReservedCacheSize(), kOneMegabyte)
+  EXPECT_LE(test_cache_rev_mng->GetTotalReservedCacheSize(),
+            kSmallCacheCapacity)
      << "Failed to bookkeep correctly (i.e, bookkeep only successful dummy "
         "entry insertions) when encountering cache resevation failure due to "
         "full cache";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly";
  EXPECT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry)
      << "Failed to insert underlying dummy entries correctly when "
         "encountering cache resevation failure due to full cache";
-  EXPECT_LE(cache->GetPinnedUsage(), kOneMegabyte)
+  EXPECT_LE(cache->GetPinnedUsage(), kSmallCacheCapacity)
      << "Failed to insert underlying dummy entries correctly when "
         "encountering cache resevation failure due to full cache";

  // Increase cache capacity so the previously failed insertion can fully
  // succeed
-  cache->SetCapacity(kOneGigabyte);
-  new_mem_used = kOneMegabyte + 1;
+  cache->SetCapacity(kBigCacheCapacity);
+  new_mem_used = kSmallCacheCapacity + 1;
  s = test_cache_rev_mng
          ->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
              new_mem_used);
@@ -237,6 +253,8 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
            5 * kSizeDummyEntry)
      << "Failed to bookkeep cache reservation increase correctly after "
         "increasing cache capacity and mitigating cache full error";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly";
  EXPECT_GE(cache->GetPinnedUsage(), 5 * kSizeDummyEntry)
      << "Failed to insert underlying dummy entries correctly after increasing "
         "cache capacity and mitigating cache full error";
@@ -256,6 +274,7 @@ TEST_F(CacheReservationManagerTest,
  ASSERT_EQ(s, Status::OK());
  ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
            2 * kSizeDummyEntry);
+  ASSERT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used);
  ASSERT_GE(cache->GetPinnedUsage(), 2 * kSizeDummyEntry);
  ASSERT_LT(cache->GetPinnedUsage(),
            2 * kSizeDummyEntry + kMetaDataChargeOverhead);
@@ -269,6 +288,8 @@ TEST_F(CacheReservationManagerTest,
  EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
            1 * kSizeDummyEntry)
      << "Failed to bookkeep cache reservation decrease correctly";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly";
  EXPECT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry)
      << "Failed to decrease underlying dummy entries in cache correctly";
  EXPECT_LT(cache->GetPinnedUsage(),
@@ -286,6 +307,7 @@ TEST_F(CacheReservationManagerTest,
  ASSERT_EQ(s, Status::OK());
  ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
            2 * kSizeDummyEntry);
+  ASSERT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used);
  ASSERT_GE(cache->GetPinnedUsage(), 2 * kSizeDummyEntry);
  ASSERT_LT(cache->GetPinnedUsage(),
            2 * kSizeDummyEntry + kMetaDataChargeOverhead);
@@ -299,6 +321,8 @@ TEST_F(CacheReservationManagerTest,
  EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
            1 * kSizeDummyEntry)
      << "Failed to bookkeep cache reservation decrease correctly";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly";
  EXPECT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry)
      << "Failed to decrease underlying dummy entries in cache correctly";
  EXPECT_LT(cache->GetPinnedUsage(),
@@ -308,13 +332,13 @@ TEST_F(CacheReservationManagerTest,

 TEST(CacheReservationManagerWithDelayedDecreaseTest,
     DecreaseCacheReservationWithDelayedDecrease) {
-  constexpr std::size_t kOneGigabyte = 1024 * 1024 * 1024;
  constexpr std::size_t kSizeDummyEntry =
      CacheReservationManager::GetDummyEntrySize();
+  constexpr std::size_t kCacheCapacity = 4096 * kSizeDummyEntry;
  constexpr std::size_t kMetaDataChargeOverhead = 10000;

  LRUCacheOptions lo;
-  lo.capacity = kOneGigabyte;
+  lo.capacity = kCacheCapacity;
  lo.num_shard_bits = 0;
  std::shared_ptr<Cache> cache = NewLRUCache(lo);
  std::unique_ptr<CacheReservationManager> test_cache_rev_mng(
@@ -328,6 +352,7 @@ TEST(CacheReservationManagerWithDelayedDecreaseTest,
  ASSERT_EQ(s, Status::OK());
  ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
            8 * kSizeDummyEntry);
+  ASSERT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used);
  std::size_t initial_pinned_usage = cache->GetPinnedUsage();
  ASSERT_GE(initial_pinned_usage, 8 * kSizeDummyEntry);
  ASSERT_LT(initial_pinned_usage,
@@ -342,6 +367,8 @@ TEST(CacheReservationManagerWithDelayedDecreaseTest,
            8 * kSizeDummyEntry)
      << "Failed to bookkeep correctly when delaying cache reservation "
         "decrease";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly";
  EXPECT_EQ(cache->GetPinnedUsage(), initial_pinned_usage)
      << "Failed to delay decreasing underlying dummy entries in cache";

@@ -354,6 +381,8 @@ TEST(CacheReservationManagerWithDelayedDecreaseTest,
            8 * kSizeDummyEntry)
      << "Failed to bookkeep correctly when delaying cache reservation "
         "decrease";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly";
  EXPECT_EQ(cache->GetPinnedUsage(), initial_pinned_usage)
      << "Failed to delay decreasing underlying dummy entries in cache";

@@ -368,6 +397,8 @@ TEST(CacheReservationManagerWithDelayedDecreaseTest,
            6 * kSizeDummyEntry)
      << "Failed to bookkeep correctly when new_mem_used < "
         "GetTotalReservedCacheSize() * 3 / 4 on delayed decrease mode";
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), new_mem_used)
+      << "Failed to bookkeep the used memory correctly";
  EXPECT_GE(cache->GetPinnedUsage(), 6 * kSizeDummyEntry)
      << "Failed to decrease underlying dummy entries in cache when "
         "new_mem_used < GetTotalReservedCacheSize() * 3 / 4 on delayed "
@@ -381,13 +412,13 @@ TEST(CacheReservationManagerWithDelayedDecreaseTest,

 TEST(CacheReservationManagerDestructorTest,
     ReleaseRemainingDummyEntriesOnDestruction) {
-  constexpr std::size_t kOneGigabyte = 1024 * 1024 * 1024;
  constexpr std::size_t kSizeDummyEntry =
      CacheReservationManager::GetDummyEntrySize();
+  constexpr std::size_t kCacheCapacity = 4096 * kSizeDummyEntry;
  constexpr std::size_t kMetaDataChargeOverhead = 10000;

  LRUCacheOptions lo;
-  lo.capacity = kOneGigabyte;
+  lo.capacity = kCacheCapacity;
  lo.num_shard_bits = 0;
  std::shared_ptr<Cache> cache = NewLRUCache(lo);
  {
@@ -407,6 +438,74 @@ TEST(CacheReservationManagerDestructorTest,
      << "Failed to release remaining underlying dummy entries in cache in "
         "CacheReservationManager's destructor";
 }
+
+TEST(CacheReservationHandleTest, HandleTest) {
+  constexpr std::size_t kOneGigabyte = 1024 * 1024 * 1024;
+  constexpr std::size_t kSizeDummyEntry = 256 * 1024;
+  constexpr std::size_t kMetaDataChargeOverhead = 10000;
+
+  LRUCacheOptions lo;
+  lo.capacity = kOneGigabyte;
+  lo.num_shard_bits = 0;
+  std::shared_ptr<Cache> cache = NewLRUCache(lo);
+
+  std::shared_ptr<CacheReservationManager> test_cache_rev_mng(
+      std::make_shared<CacheReservationManager>(cache));
+
+  std::size_t mem_used = 0;
+  const std::size_t incremental_mem_used_handle_1 = 1 * kSizeDummyEntry;
+  const std::size_t incremental_mem_used_handle_2 = 2 * kSizeDummyEntry;
+  std::unique_ptr<CacheReservationHandle<CacheEntryRole::kMisc>> handle_1,
+      handle_2;
+
+  // To test consecutive CacheReservationManager::MakeCacheReservation works
+  // correctly in terms of returning the handle as well as updating cache
+  // reservation and the latest total memory used
+  Status s = test_cache_rev_mng->MakeCacheReservation<CacheEntryRole::kMisc>(
+      incremental_mem_used_handle_1, &handle_1);
+  mem_used = mem_used + incremental_mem_used_handle_1;
+  ASSERT_EQ(s, Status::OK());
+  EXPECT_TRUE(handle_1 != nullptr);
+  EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), mem_used);
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), mem_used);
+  EXPECT_GE(cache->GetPinnedUsage(), mem_used);
+  EXPECT_LT(cache->GetPinnedUsage(), mem_used + kMetaDataChargeOverhead);
+
+  s = test_cache_rev_mng->MakeCacheReservation<CacheEntryRole::kMisc>(
+      incremental_mem_used_handle_2, &handle_2);
+  mem_used = mem_used + incremental_mem_used_handle_2;
+  ASSERT_EQ(s, Status::OK());
+  EXPECT_TRUE(handle_2 != nullptr);
+  EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), mem_used);
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), mem_used);
+  EXPECT_GE(cache->GetPinnedUsage(), mem_used);
+  EXPECT_LT(cache->GetPinnedUsage(), mem_used + kMetaDataChargeOverhead);
+
+  // To test CacheReservationHandle::~CacheReservationHandle() works correctly
+  // in releasing the cache reserved for the handle
+  handle_1.reset();
+  EXPECT_TRUE(handle_1 == nullptr);
+  mem_used = mem_used - incremental_mem_used_handle_1;
+  EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(), mem_used);
+  EXPECT_EQ(test_cache_rev_mng->GetTotalMemoryUsed(), mem_used);
+  EXPECT_GE(cache->GetPinnedUsage(), mem_used);
+  EXPECT_LT(cache->GetPinnedUsage(), mem_used + kMetaDataChargeOverhead);
+
+  // To test the actual CacheReservationManager object won't be deallocated
+  // as long as there remain handles pointing to it.
+  // We strongly recommend deallocating CacheReservationManager object only
+  // after all its handles are deallocated to keep things easy to reasonate
+  test_cache_rev_mng.reset();
+  EXPECT_GE(cache->GetPinnedUsage(), mem_used);
+  EXPECT_LT(cache->GetPinnedUsage(), mem_used + kMetaDataChargeOverhead);
+
+  handle_2.reset();
+  // The CacheReservationManager object is now deallocated since all the handles
+  // and its original pointer is gone
+  mem_used = mem_used - incremental_mem_used_handle_2;
+  EXPECT_EQ(mem_used, 0);
+  EXPECT_EQ(cache->GetPinnedUsage(), mem_used);
+}
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -9,14 +9,26 @@

 namespace ROCKSDB_NAMESPACE {

-Status BlobFetcher::FetchBlob(const Slice& user_key, const Slice& blob_index,
-                              PinnableSlice* blob_value) {
-  Status s;
+Status BlobFetcher::FetchBlob(const Slice& user_key,
+                              const Slice& blob_index_slice,
+                              FilePrefetchBuffer* prefetch_buffer,
+                              PinnableSlice* blob_value,
+                              uint64_t* bytes_read) const {
  assert(version_);
-  constexpr uint64_t* bytes_read = nullptr;
-  s = version_->GetBlob(read_options_, user_key, blob_index, blob_value,
-                        bytes_read);
-  return s;
+
+  return version_->GetBlob(read_options_, user_key, blob_index_slice,
+                           prefetch_buffer, blob_value, bytes_read);
 }

-}  // namespace ROCKSDB_NAMESPACE
+Status BlobFetcher::FetchBlob(const Slice& user_key,
+                              const BlobIndex& blob_index,
+                              FilePrefetchBuffer* prefetch_buffer,
+                              PinnableSlice* blob_value,
+                              uint64_t* bytes_read) const {
+  assert(version_);
+
+  return version_->GetBlob(read_options_, user_key, blob_index, prefetch_buffer,
+                           blob_value, bytes_read);
+}
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -9,18 +9,29 @@
 #include "rocksdb/status.h"

 namespace ROCKSDB_NAMESPACE {
-class Version;

+class Version;
+class Slice;
+class FilePrefetchBuffer;
+class PinnableSlice;
+class BlobIndex;
+
+// A thin wrapper around the blob retrieval functionality of Version.
 class BlobFetcher {
 public:
-  BlobFetcher(Version* version, const ReadOptions& read_options)
+  BlobFetcher(const Version* version, const ReadOptions& read_options)
      : version_(version), read_options_(read_options) {}

-  Status FetchBlob(const Slice& user_key, const Slice& blob_index,
-                   PinnableSlice* blob_value);
+  Status FetchBlob(const Slice& user_key, const Slice& blob_index_slice,
+                   FilePrefetchBuffer* prefetch_buffer,
+                   PinnableSlice* blob_value, uint64_t* bytes_read) const;
+
+  Status FetchBlob(const Slice& user_key, const BlobIndex& blob_index,
+                   FilePrefetchBuffer* prefetch_buffer,
+                   PinnableSlice* blob_value, uint64_t* bytes_read) const;

 private:
-  Version* version_;
+  const Version* version_;
  ReadOptions read_options_;
 };
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace ROCKSDB_NAMESPACE
@@ -9,6 +9,7 @@
 #include <string>

 #include "db/blob/blob_log_format.h"
+#include "file/file_prefetch_buffer.h"
 #include "file/filename.h"
 #include "monitoring/statistics.h"
 #include "options/cf_options.h"
@@ -282,6 +283,7 @@ Status BlobFileReader::GetBlob(const ReadOptions& read_options,
                               const Slice& user_key, uint64_t offset,
                               uint64_t value_size,
                               CompressionType compression_type,
+                               FilePrefetchBuffer* prefetch_buffer,
                               PinnableSlice* value,
                               uint64_t* bytes_read) const {
  assert(value);
@@ -313,7 +315,21 @@ Status BlobFileReader::GetBlob(const ReadOptions& read_options,
  Buffer buf;
  AlignedBuf aligned_buf;

-  {
+  bool prefetched = false;
+
+  if (prefetch_buffer) {
+    Status s;
+    constexpr bool for_compaction = true;
+
+    prefetched = prefetch_buffer->TryReadFromCache(
+        IOOptions(), file_reader_.get(), record_offset,
+        static_cast<size_t>(record_size), &record_slice, &s, for_compaction);
+    if (!s.ok()) {
+      return s;
+    }
+  }
+
+  if (!prefetched) {
    TEST_SYNC_POINT("BlobFileReader::GetBlob:ReadFromFile");

    const Status s = ReadFromFile(file_reader_.get(), record_offset,
@@ -322,11 +338,11 @@ Status BlobFileReader::GetBlob(const ReadOptions& read_options,
    if (!s.ok()) {
      return s;
    }
-
-    TEST_SYNC_POINT_CALLBACK("BlobFileReader::GetBlob:TamperWithResult",
-                             &record_slice);
  }

+  TEST_SYNC_POINT_CALLBACK("BlobFileReader::GetBlob:TamperWithResult",
+                           &record_slice);
+
  if (read_options.verify_checksums) {
    const Status s = VerifyBlob(record_slice, user_key, value_size);
    if (!s.ok()) {
@@ -21,6 +21,7 @@ struct FileOptions;
 class HistogramImpl;
 struct ReadOptions;
 class Slice;
+class FilePrefetchBuffer;
 class PinnableSlice;
 class Statistics;

@@ -41,7 +42,8 @@ class BlobFileReader {

  Status GetBlob(const ReadOptions& read_options, const Slice& user_key,
                 uint64_t offset, uint64_t value_size,
-                 CompressionType compression_type, PinnableSlice* value,
+                 CompressionType compression_type,
+                 FilePrefetchBuffer* prefetch_buffer, PinnableSlice* value,
                 uint64_t* bytes_read) const;

  // offsets must be sorted in ascending order by caller.
@@ -179,13 +179,15 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {
  ReadOptions read_options;
  read_options.verify_checksums = false;

+  constexpr FilePrefetchBuffer* prefetch_buffer = nullptr;
+
  {
    PinnableSlice value;
    uint64_t bytes_read = 0;

    ASSERT_OK(reader->GetBlob(read_options, keys[0], blob_offsets[0],
-                              blob_sizes[0], kNoCompression, &value,
-                              &bytes_read));
+                              blob_sizes[0], kNoCompression, prefetch_buffer,
+                              &value, &bytes_read));
    ASSERT_EQ(value, blobs[0]);
    ASSERT_EQ(bytes_read, blob_sizes[0]);

@@ -222,8 +224,8 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {
    uint64_t bytes_read = 0;

    ASSERT_OK(reader->GetBlob(read_options, keys[1], blob_offsets[1],
-                              blob_sizes[1], kNoCompression, &value,
-                              &bytes_read));
+                              blob_sizes[1], kNoCompression, prefetch_buffer,
+                              &value, &bytes_read));
    ASSERT_EQ(value, blobs[1]);

    const uint64_t key_size = keys[1].size();
@@ -239,8 +241,8 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {

    ASSERT_TRUE(reader
                    ->GetBlob(read_options, keys[0], blob_offsets[0] - 1,
-                              blob_sizes[0], kNoCompression, &value,
-                              &bytes_read)
+                              blob_sizes[0], kNoCompression, prefetch_buffer,
+                              &value, &bytes_read)
                    .IsCorruption());
    ASSERT_EQ(bytes_read, 0);
  }
@@ -252,8 +254,8 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {

    ASSERT_TRUE(reader
                    ->GetBlob(read_options, keys[2], blob_offsets[2] + 1,
-                              blob_sizes[2], kNoCompression, &value,
-                              &bytes_read)
+                              blob_sizes[2], kNoCompression, prefetch_buffer,
+                              &value, &bytes_read)
                    .IsCorruption());
    ASSERT_EQ(bytes_read, 0);
  }
@@ -265,7 +267,8 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {

    ASSERT_TRUE(reader
                    ->GetBlob(read_options, keys[0], blob_offsets[0],
-                              blob_sizes[0], kZSTD, &value, &bytes_read)
+                              blob_sizes[0], kZSTD, prefetch_buffer, &value,
+                              &bytes_read)
                    .IsCorruption());
    ASSERT_EQ(bytes_read, 0);
  }
@@ -280,8 +283,8 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {
                    ->GetBlob(read_options, shorter_key,
                              blob_offsets[0] -
                                  (keys[0].size() - sizeof(shorter_key) + 1),
-                              blob_sizes[0], kNoCompression, &value,
-                              &bytes_read)
+                              blob_sizes[0], kNoCompression, prefetch_buffer,
+                              &value, &bytes_read)
                    .IsCorruption());
    ASSERT_EQ(bytes_read, 0);

@@ -323,8 +326,8 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {

    ASSERT_TRUE(reader
                    ->GetBlob(read_options, incorrect_key, blob_offsets[0],
-                              blob_sizes[0], kNoCompression, &value,
-                              &bytes_read)
+                              blob_sizes[0], kNoCompression, prefetch_buffer,
+                              &value, &bytes_read)
                    .IsCorruption());
    ASSERT_EQ(bytes_read, 0);

@@ -363,8 +366,8 @@ TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {

    ASSERT_TRUE(reader
                    ->GetBlob(read_options, keys[1], blob_offsets[1],
-                              blob_sizes[1] + 1, kNoCompression, &value,
-                              &bytes_read)
+                              blob_sizes[1] + 1, kNoCompression,
+                              prefetch_buffer, &value, &bytes_read)
                    .IsCorruption());
    ASSERT_EQ(bytes_read, 0);

@@ -642,12 +645,14 @@ TEST_F(BlobFileReaderTest, BlobCRCError) {

  SyncPoint::GetInstance()->EnableProcessing();

+  constexpr FilePrefetchBuffer* prefetch_buffer = nullptr;
  PinnableSlice value;
  uint64_t bytes_read = 0;

  ASSERT_TRUE(reader
                  ->GetBlob(ReadOptions(), key, blob_offset, blob_size,
-                            kNoCompression, &value, &bytes_read)
+                            kNoCompression, prefetch_buffer, &value,
+                            &bytes_read)
                  .IsCorruption());
  ASSERT_EQ(bytes_read, 0);

@@ -695,12 +700,15 @@ TEST_F(BlobFileReaderTest, Compression) {
  ReadOptions read_options;
  read_options.verify_checksums = false;

+  constexpr FilePrefetchBuffer* prefetch_buffer = nullptr;
+
  {
    PinnableSlice value;
    uint64_t bytes_read = 0;

    ASSERT_OK(reader->GetBlob(read_options, key, blob_offset, blob_size,
-                              kSnappyCompression, &value, &bytes_read));
+                              kSnappyCompression, prefetch_buffer, &value,
+                              &bytes_read));
    ASSERT_EQ(value, blob);
    ASSERT_EQ(bytes_read, blob_size);
  }
@@ -712,7 +720,8 @@ TEST_F(BlobFileReaderTest, Compression) {
    uint64_t bytes_read = 0;

    ASSERT_OK(reader->GetBlob(read_options, key, blob_offset, blob_size,
-                              kSnappyCompression, &value, &bytes_read));
+                              kSnappyCompression, prefetch_buffer, &value,
+                              &bytes_read));
    ASSERT_EQ(value, blob);

    constexpr uint64_t key_size = sizeof(key) - 1;
@@ -770,12 +779,14 @@ TEST_F(BlobFileReaderTest, UncompressionError) {

  SyncPoint::GetInstance()->EnableProcessing();

+  constexpr FilePrefetchBuffer* prefetch_buffer = nullptr;
  PinnableSlice value;
  uint64_t bytes_read = 0;

  ASSERT_TRUE(reader
                  ->GetBlob(ReadOptions(), key, blob_offset, blob_size,
-                            kSnappyCompression, &value, &bytes_read)
+                            kSnappyCompression, prefetch_buffer, &value,
+                            &bytes_read)
                  .IsCorruption());
  ASSERT_EQ(bytes_read, 0);

@@ -854,12 +865,14 @@ TEST_P(BlobFileReaderIOErrorTest, IOError) {
  } else {
    ASSERT_OK(s);

+    constexpr FilePrefetchBuffer* prefetch_buffer = nullptr;
    PinnableSlice value;
    uint64_t bytes_read = 0;

    ASSERT_TRUE(reader
                    ->GetBlob(ReadOptions(), key, blob_offset, blob_size,
-                              kNoCompression, &value, &bytes_read)
+                              kNoCompression, prefetch_buffer, &value,
+                              &bytes_read)
                    .IsIOError());
    ASSERT_EQ(bytes_read, 0);
  }
@@ -937,12 +950,14 @@ TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) {
  } else {
    ASSERT_OK(s);

+    constexpr FilePrefetchBuffer* prefetch_buffer = nullptr;
    PinnableSlice value;
    uint64_t bytes_read = 0;

    ASSERT_TRUE(reader
                    ->GetBlob(ReadOptions(), key, blob_offset, blob_size,
-                              kNoCompression, &value, &bytes_read)
+                              kNoCompression, prefetch_buffer, &value,
+                              &bytes_read)
                    .IsCorruption());
    ASSERT_EQ(bytes_read, 0);
  }
@@ -1018,14 +1018,6 @@ TEST_P(DBBlobBasicIOErrorTest, CompactionFilterReadBlob_IOError) {

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -590,15 +590,125 @@ TEST_F(DBBlobCompactionTest, MergeBlobWithBase) {
  Close();
 }

-}  // namespace ROCKSDB_NAMESPACE
+TEST_F(DBBlobCompactionTest, CompactionReadaheadGarbageCollection) {
+  Options options = GetDefaultOptions();
+  options.enable_blob_files = true;
+  options.min_blob_size = 0;
+  options.enable_blob_garbage_collection = true;
+  options.blob_garbage_collection_age_cutoff = 1.0;
+  options.blob_compaction_readahead_size = 1 << 10;
+  options.disable_auto_compactions = true;

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
+  Reopen(options);
+
+  ASSERT_OK(Put("key", "lime"));
+  ASSERT_OK(Put("foo", "bar"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put("key", "pie"));
+  ASSERT_OK(Put("foo", "baz"));
+  ASSERT_OK(Flush());
+
+  size_t num_non_prefetch_reads = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "BlobFileReader::GetBlob:ReadFromFile",
+      [&num_non_prefetch_reads](void* /* arg */) { ++num_non_prefetch_reads; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  constexpr Slice* begin = nullptr;
+  constexpr Slice* end = nullptr;
+
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end));
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_EQ(Get("key"), "pie");
+  ASSERT_EQ(Get("foo"), "baz");
+  ASSERT_EQ(num_non_prefetch_reads, 0);
+
+  Close();
 }
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
+
+TEST_F(DBBlobCompactionTest, CompactionReadaheadFilter) {
+  Options options = GetDefaultOptions();
+
+  std::unique_ptr<CompactionFilter> compaction_filter_guard(
+      new ValueMutationFilter("pie"));
+
+  options.compaction_filter = compaction_filter_guard.get();
+  options.enable_blob_files = true;
+  options.min_blob_size = 0;
+  options.blob_compaction_readahead_size = 1 << 10;
+  options.disable_auto_compactions = true;
+
+  Reopen(options);
+
+  ASSERT_OK(Put("key", "lime"));
+  ASSERT_OK(Put("foo", "bar"));
+  ASSERT_OK(Flush());
+
+  size_t num_non_prefetch_reads = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "BlobFileReader::GetBlob:ReadFromFile",
+      [&num_non_prefetch_reads](void* /* arg */) { ++num_non_prefetch_reads; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  constexpr Slice* begin = nullptr;
+  constexpr Slice* end = nullptr;
+
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end));
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_EQ(Get("key"), "limepie");
+  ASSERT_EQ(Get("foo"), "barpie");
+  ASSERT_EQ(num_non_prefetch_reads, 0);
+
+  Close();
+}
+
+TEST_F(DBBlobCompactionTest, CompactionReadaheadMerge) {
+  Options options = GetDefaultOptions();
+  options.enable_blob_files = true;
+  options.min_blob_size = 0;
+  options.blob_compaction_readahead_size = 1 << 10;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  options.disable_auto_compactions = true;
+
+  Reopen(options);
+
+  ASSERT_OK(Put("key", "lime"));
+  ASSERT_OK(Put("foo", "bar"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Merge("key", "pie"));
+  ASSERT_OK(Merge("foo", "baz"));
+  ASSERT_OK(Flush());
+
+  size_t num_non_prefetch_reads = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "BlobFileReader::GetBlob:ReadFromFile",
+      [&num_non_prefetch_reads](void* /* arg */) { ++num_non_prefetch_reads; });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  constexpr Slice* begin = nullptr;
+  constexpr Slice* end = nullptr;
+
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end));
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_EQ(Get("key"), "lime,pie");
+  ASSERT_EQ(Get("foo"), "bar,baz");
+  ASSERT_EQ(num_non_prefetch_reads, 0);
+
+  Close();
+}
+
+}  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
@@ -74,14 +74,6 @@ TEST_F(DBBlobCorruptionTest, VerifyWholeBlobFileChecksum) {
 #endif  // !ROCKSDB_LITE
 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -564,14 +564,6 @@ TEST_F(DBBlobIndexTest, IntegratedBlobIterate) {

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -0,0 +1,21 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "db/blob/prefetch_buffer_collection.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+FilePrefetchBuffer* PrefetchBufferCollection::GetOrCreatePrefetchBuffer(
+    uint64_t file_number) {
+  auto& prefetch_buffer = prefetch_buffers_[file_number];
+  if (!prefetch_buffer) {
+    prefetch_buffer.reset(
+        new FilePrefetchBuffer(readahead_size_, readahead_size_));
+  }
+
+  return prefetch_buffer.get();
+}
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,38 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <memory>
+#include <unordered_map>
+
+#include "file/file_prefetch_buffer.h"
+#include "rocksdb/rocksdb_namespace.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// A class that owns a collection of FilePrefetchBuffers using the file number
+// as key. Used for implementing compaction readahead for blob files. Designed
+// to be accessed by a single thread only: every (sub)compaction needs its own
+// buffers since they are guaranteed to read different blobs from different
+// positions even when reading the same file.
+class PrefetchBufferCollection {
+ public:
+  explicit PrefetchBufferCollection(uint64_t readahead_size)
+      : readahead_size_(readahead_size) {
+    assert(readahead_size_ > 0);
+  }
+
+  FilePrefetchBuffer* GetOrCreatePrefetchBuffer(uint64_t file_number);
+
+ private:
+  uint64_t readahead_size_;
+  std::unordered_map<uint64_t, std::unique_ptr<FilePrefetchBuffer>>
+      prefetch_buffers_;  // maps file number to prefetch buffer
+};
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -2760,6 +2760,16 @@ double rocksdb_options_get_blob_gc_force_threshold(rocksdb_options_t* opt) {
  return opt->rep.blob_garbage_collection_force_threshold;
 }

+void rocksdb_options_set_blob_compaction_readahead_size(rocksdb_options_t* opt,
+                                                        uint64_t val) {
+  opt->rep.blob_compaction_readahead_size = val;
+}
+
+uint64_t rocksdb_options_get_blob_compaction_readahead_size(
+    rocksdb_options_t* opt) {
+  return opt->rep.blob_compaction_readahead_size;
+}
+
 void rocksdb_options_set_num_levels(rocksdb_options_t* opt, int n) {
  opt->rep.num_levels = n;
 }
@@ -1799,6 +1799,10 @@ int main(int argc, char** argv) {
    rocksdb_options_set_blob_gc_force_threshold(o, 0.75);
    CheckCondition(0.75 == rocksdb_options_get_blob_gc_force_threshold(o));

+    rocksdb_options_set_blob_compaction_readahead_size(o, 262144);
+    CheckCondition(262144 ==
+                   rocksdb_options_get_blob_compaction_readahead_size(o));
+
    // Create a copy that should be equal to the original.
    rocksdb_options_t* copy;
    copy = rocksdb_options_create_copy(o);
@@ -75,11 +75,6 @@ ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
      bool defer_purge =
          db_->immutable_db_options().avoid_unnecessary_blocking_io;
      db_->PurgeObsoleteFiles(job_context, defer_purge);
-      if (defer_purge) {
-        mutex_->Lock();
-        db_->SchedulePurge();
-        mutex_->Unlock();
-      }
    }
    job_context.Clean();
  }
@@ -211,7 +206,8 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
  size_t clamp_max = std::conditional<
      sizeof(size_t) == 4, std::integral_constant<size_t, 0xffffffff>,
      std::integral_constant<uint64_t, 64ull << 30>>::type::value;
-  ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, clamp_max);
+  ClipToRange(&result.write_buffer_size, (static_cast<size_t>(64)) << 10,
+              clamp_max);
  // if user sets arena_block_size, we trust user to use this value. Otherwise,
  // calculate a proper value from writer_buffer_size;
  if (result.arena_block_size <= 0) {
@@ -3433,14 +3433,6 @@ TEST(ColumnFamilyTest, ValidateBlobGCForceThreshold) {

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -9,6 +9,8 @@

 #include "rocksdb/rocksdb_namespace.h"

+namespace ROCKSDB_NAMESPACE {
+
 struct CompactionIterationStats {
  // Compaction statistics

@@ -43,3 +45,5 @@ struct CompactionIterationStats {
  uint64_t num_blobs_relocated = 0;
  uint64_t total_blob_bytes_relocated = 0;
 };
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -8,8 +8,10 @@
 #include <iterator>
 #include <limits>

+#include "db/blob/blob_fetcher.h"
 #include "db/blob/blob_file_builder.h"
 #include "db/blob/blob_index.h"
+#include "db/blob/prefetch_buffer_collection.h"
 #include "db/snapshot_checker.h"
 #include "logging/logging.h"
 #include "port/likely.h"
@@ -88,6 +90,9 @@ CompactionIterator::CompactionIterator(
      merge_out_iter_(merge_helper_),
      blob_garbage_collection_cutoff_file_number_(
          ComputeBlobGarbageCollectionCutoffFileNumber(compaction_.get())),
+      blob_fetcher_(CreateBlobFetcherIfNeeded(compaction_.get())),
+      prefetch_buffers_(
+          CreatePrefetchBufferCollectionIfNeeded(compaction_.get())),
      current_key_committed_(false),
      cmp_with_history_ts_low_(0),
      level_(compaction_ == nullptr ? 0 : compaction_->level()) {
@@ -225,6 +230,13 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
          compaction_filter_skip_until_.rep());
      if (CompactionFilter::Decision::kUndetermined == filter &&
          !compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) {
+        if (compaction_ == nullptr) {
+          status_ =
+              Status::Corruption("Unexpected blob index outside of compaction");
+          valid_ = false;
+          return false;
+        }
+
        // For integrated BlobDB impl, CompactionIterator reads blob value.
        // For Stacked BlobDB impl, the corresponding CompactionFilter's
        // FilterV2 method should read the blob value.
@@ -235,23 +247,19 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
          valid_ = false;
          return false;
        }
-        if (blob_index.HasTTL() || blob_index.IsInlined()) {
-          status_ = Status::Corruption("Unexpected TTL/inlined blob index");
-          valid_ = false;
-          return false;
-        }
-        if (compaction_ == nullptr) {
-          status_ =
-              Status::Corruption("Unexpected blob index outside of compaction");
-          valid_ = false;
-          return false;
-        }
-        const Version* const version = compaction_->input_version();
-        assert(version);
+
+        FilePrefetchBuffer* prefetch_buffer =
+            prefetch_buffers_ ? prefetch_buffers_->GetOrCreatePrefetchBuffer(
+                                    blob_index.file_number())
+                              : nullptr;

        uint64_t bytes_read = 0;
-        s = version->GetBlob(ReadOptions(), ikey_.user_key, blob_index,
-                             &blob_value_, &bytes_read);
+
+        assert(blob_fetcher_);
+
+        s = blob_fetcher_->FetchBlob(ikey_.user_key, blob_index,
+                                     prefetch_buffer, &blob_value_,
+                                     &bytes_read);
        if (!s.ok()) {
          status_ = s;
          valid_ = false;
@@ -407,6 +415,12 @@ void CompactionIterator::NextFromInput() {
      // Copy key for output
      key_ = current_key_.SetInternalKey(key_, &ikey_);

+      int prev_cmp_with_ts_low =
+          !full_history_ts_low_ ? 0
+          : curr_ts_.empty()
+              ? 0
+              : cmp_->CompareTimestamp(curr_ts_, *full_history_ts_low_);
+
      // If timestamp_size_ > 0, then copy from ikey_ to curr_ts_ for the use
      // in next iteration to compare with the timestamp of next key.
      UpdateTimestampAndCompareWithFullHistoryLow();
@@ -416,14 +430,16 @@ void CompactionIterator::NextFromInput() {
      // (2) timestamp is disabled, OR
      // (3) all history will be preserved, OR
      // (4) user key (excluding timestamp) is different from previous key, OR
-      // (5) timestamp is NO older than *full_history_ts_low_
+      // (5) timestamp is NO older than *full_history_ts_low_, OR
+      // (6) timestamp is the largest one older than full_history_ts_low_,
      // then current_user_key_ must be treated as a different user key.
      // This means, if a user key (excluding ts) is the same as the previous
      // user key, and its ts is older than *full_history_ts_low_, then we
      // consider this key for GC, e.g. it may be dropped if certain conditions
      // match.
      if (!has_current_user_key_ || !timestamp_size_ || !full_history_ts_low_ ||
-          !user_key_equal_without_ts || cmp_with_history_ts_low_ >= 0) {
+          !user_key_equal_without_ts || cmp_with_history_ts_low_ >= 0 ||
+          prev_cmp_with_ts_low >= 0) {
        // Initialize for future comparison for rule (A) and etc.
        current_user_key_sequence_ = kMaxSequenceNumber;
        current_user_key_snapshot_ = 0;
@@ -608,11 +624,19 @@ void CompactionIterator::NextFromInput() {

          TEST_SYNC_POINT_CALLBACK(
              "CompactionIterator::NextFromInput:SingleDelete:2", nullptr);
-          if (next_ikey.type == kTypeSingleDeletion) {
+          if (next_ikey.type == kTypeSingleDeletion ||
+              next_ikey.type == kTypeDeletion) {
            // We encountered two SingleDeletes for same key in a row. This
-            // could be due to unexpected user input. Skip the first
-            // SingleDelete and let the next iteration decide how to handle the
-            // second SingleDelete
+            // could be due to unexpected user input. If write-(un)prepared
+            // transaction is used, this could also be due to releasing an old
+            // snapshot between a Put and its matching SingleDelete.
+            // Furthermore, if write-(un)prepared transaction is rolled back
+            // after prepare, we will write a Delete to cancel a prior Put. If
+            // old snapshot is released between a later Put and its matching
+            // SingleDelete, we will end up with a Delete followed by
+            // SingleDelete.
+            // Skip the first SingleDelete and let the next iteration decide
+            // how to handle the second SingleDelete or Delete.

            // First SingleDelete has been skipped since we already called
            // input_.Next().
@@ -815,15 +839,15 @@ void CompactionIterator::NextFromInput() {
      }

      pinned_iters_mgr_.StartPinning();
-      Version* version = compaction_ ? compaction_->input_version() : nullptr;

      // We know the merge type entry is not hidden, otherwise we would
      // have hit (A)
      // We encapsulate the merge related state machine in a different
      // object to minimize change to the existing flow.
-      Status s = merge_helper_->MergeUntil(&input_, range_del_agg_,
-                                           prev_snapshot, bottommost_level_,
-                                           allow_data_in_errors_, version);
+      Status s = merge_helper_->MergeUntil(
+          &input_, range_del_agg_, prev_snapshot, bottommost_level_,
+          allow_data_in_errors_, blob_fetcher_.get(), prefetch_buffers_.get(),
+          &iter_stats_);
      merge_out_iter_.SeekToFirst();

      if (!s.ok() && !s.IsMergeInProgress()) {
@@ -943,26 +967,23 @@ void CompactionIterator::GarbageCollectBlobIfNeeded() {
      }
    }

-    if (blob_index.IsInlined() || blob_index.HasTTL()) {
-      status_ = Status::Corruption("Unexpected TTL/inlined blob index");
-      valid_ = false;
-
-      return;
-    }
-
    if (blob_index.file_number() >=
        blob_garbage_collection_cutoff_file_number_) {
      return;
    }

-    const Version* const version = compaction_->input_version();
-    assert(version);
+    FilePrefetchBuffer* prefetch_buffer =
+        prefetch_buffers_ ? prefetch_buffers_->GetOrCreatePrefetchBuffer(
+                                blob_index.file_number())
+                          : nullptr;

    uint64_t bytes_read = 0;

    {
-      const Status s = version->GetBlob(ReadOptions(), user_key(), blob_index,
-                                        &blob_value_, &bytes_read);
+      assert(blob_fetcher_);
+
+      const Status s = blob_fetcher_->FetchBlob(
+          user_key(), blob_index, prefetch_buffer, &blob_value_, &bytes_read);

      if (!s.ok()) {
        status_ = s;
@@ -1135,7 +1156,7 @@ uint64_t CompactionIterator::ComputeBlobGarbageCollectionCutoffFileNumber(
    return 0;
  }

-  Version* const version = compaction->input_version();
+  const Version* const version = compaction->input_version();
  assert(version);

  const VersionStorageInfo* const storage_info = version->storage_info();
@@ -1151,4 +1172,42 @@ uint64_t CompactionIterator::ComputeBlobGarbageCollectionCutoffFileNumber(
                                : std::numeric_limits<uint64_t>::max();
 }

+std::unique_ptr<BlobFetcher> CompactionIterator::CreateBlobFetcherIfNeeded(
+    const CompactionProxy* compaction) {
+  if (!compaction) {
+    return nullptr;
+  }
+
+  const Version* const version = compaction->input_version();
+  if (!version) {
+    return nullptr;
+  }
+
+  return std::unique_ptr<BlobFetcher>(new BlobFetcher(version, ReadOptions()));
+}
+
+std::unique_ptr<PrefetchBufferCollection>
+CompactionIterator::CreatePrefetchBufferCollectionIfNeeded(
+    const CompactionProxy* compaction) {
+  if (!compaction) {
+    return nullptr;
+  }
+
+  if (!compaction->input_version()) {
+    return nullptr;
+  }
+
+  if (compaction->allow_mmap_reads()) {
+    return nullptr;
+  }
+
+  const uint64_t readahead_size = compaction->blob_compaction_readahead_size();
+  if (!readahead_size) {
+    return nullptr;
+  }
+
+  return std::unique_ptr<PrefetchBufferCollection>(
+      new PrefetchBufferCollection(readahead_size));
+}
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -23,6 +23,8 @@
 namespace ROCKSDB_NAMESPACE {

 class BlobFileBuilder;
+class BlobFetcher;
+class PrefetchBufferCollection;

 // A wrapper of internal iterator whose purpose is to count how
 // many entries there are in the iterator.
@@ -92,11 +94,15 @@ class CompactionIterator {

    virtual bool preserve_deletes() const = 0;

+    virtual bool allow_mmap_reads() const = 0;
+
    virtual bool enable_blob_garbage_collection() const = 0;

    virtual double blob_garbage_collection_age_cutoff() const = 0;

-    virtual Version* input_version() const = 0;
+    virtual uint64_t blob_compaction_readahead_size() const = 0;
+
+    virtual const Version* input_version() const = 0;

    virtual bool DoesInputReferenceBlobFiles() const = 0;

@@ -137,6 +143,10 @@ class CompactionIterator {
      return compaction_->immutable_options()->preserve_deletes;
    }

+    bool allow_mmap_reads() const override {
+      return compaction_->immutable_options()->allow_mmap_reads;
+    }
+
    bool enable_blob_garbage_collection() const override {
      return compaction_->mutable_cf_options()->enable_blob_garbage_collection;
    }
@@ -146,7 +156,11 @@ class CompactionIterator {
          ->blob_garbage_collection_age_cutoff;
    }

-    Version* input_version() const override {
+    uint64_t blob_compaction_readahead_size() const override {
+      return compaction_->mutable_cf_options()->blob_compaction_readahead_size;
+    }
+
+    const Version* input_version() const override {
      return compaction_->input_version();
    }

@@ -291,6 +305,10 @@ class CompactionIterator {

  static uint64_t ComputeBlobGarbageCollectionCutoffFileNumber(
      const CompactionProxy* compaction);
+  static std::unique_ptr<BlobFetcher> CreateBlobFetcherIfNeeded(
+      const CompactionProxy* compaction);
+  static std::unique_ptr<PrefetchBufferCollection>
+  CreatePrefetchBufferCollectionIfNeeded(const CompactionProxy* compaction);

  SequenceIterWrapper input_;
  const Comparator* cmp_;
@@ -379,6 +397,9 @@ class CompactionIterator {

  uint64_t blob_garbage_collection_cutoff_file_number_;

+  std::unique_ptr<BlobFetcher> blob_fetcher_;
+  std::unique_ptr<PrefetchBufferCollection> prefetch_buffers_;
+
  std::string blob_index_;
  PinnableSlice blob_value_;
  std::string compaction_filter_value_;
@@ -168,11 +168,15 @@ class FakeCompaction : public CompactionIterator::CompactionProxy {

  bool preserve_deletes() const override { return false; }

+  bool allow_mmap_reads() const override { return false; }
+
  bool enable_blob_garbage_collection() const override { return false; }

  double blob_garbage_collection_age_cutoff() const override { return 0.0; }

-  Version* input_version() const override { return nullptr; }
+  uint64_t blob_compaction_readahead_size() const override { return 0; }
+
+  const Version* input_version() const override { return nullptr; }

  bool DoesInputReferenceBlobFiles() const override { return false; }

@@ -1171,9 +1175,10 @@ TEST_P(CompactionIteratorTsGcTest, NewHidesOldSameSnapshot) {
    std::string full_history_ts_low;
    // Keys whose timestamps larger than or equal to 102 will be preserved.
    PutFixed64(&full_history_ts_low, 102);
-    const std::vector<std::string> expected_keys = {input_keys[0],
-                                                    input_keys[1]};
-    const std::vector<std::string> expected_values = {"", "a2"};
+    const std::vector<std::string> expected_keys = {
+        input_keys[0], input_keys[1], input_keys[2]};
+    const std::vector<std::string> expected_values = {"", input_values[1],
+                                                      input_values[2]};
    RunTest(input_keys, input_values, expected_keys, expected_values,
            /*last_committed_seq=*/kMaxSequenceNumber,
            /*merge_operator=*/nullptr, /*compaction_filter=*/nullptr,
@@ -21,6 +21,7 @@
 #include "db/version_set.h"
 #include "file/writable_file_writer.h"
 #include "rocksdb/cache.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/file_system.h"
 #include "rocksdb/options.h"
@@ -73,9 +74,7 @@ class CompactionJobTestBase : public testing::Test {
 protected:
  CompactionJobTestBase(std::string dbname, const Comparator* ucmp,
                        std::function<std::string(uint64_t)> encode_u64_ts)
-      : env_(Env::Default()),
-        fs_(env_->GetFileSystem()),
-        dbname_(std::move(dbname)),
+      : dbname_(std::move(dbname)),
        ucmp_(ucmp),
        db_options_(),
        mutable_cf_options_(cf_options_),
@@ -91,7 +90,13 @@ class CompactionJobTestBase : public testing::Test {
        preserve_deletes_seqnum_(0),
        mock_table_factory_(new mock::MockTableFactory()),
        error_handler_(nullptr, db_options_, &mutex_),
-        encode_u64_ts_(std::move(encode_u64_ts)) {}
+        encode_u64_ts_(std::move(encode_u64_ts)) {
+    Env* base_env = Env::Default();
+    EXPECT_OK(
+        test::CreateEnvFromSystem(ConfigOptions(), &base_env, &env_guard_));
+    env_ = base_env;
+    fs_ = env_->GetFileSystem();
+  }

  void SetUp() override {
    EXPECT_OK(env_->CreateDirIfMissing(dbname_));
@@ -200,7 +205,8 @@ class CompactionJobTestBase : public testing::Test {
    edit.AddFile(level, file_number, 0, 10, smallest_key, largest_key,
                 smallest_seqno, largest_seqno, false, oldest_blob_file_number,
                 kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-                 kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+                 kUnknownFileChecksum, kUnknownFileChecksumFuncName,
+                 kDisableUserTimestamp, kDisableUserTimestamp);

    mutex_.Lock();
    EXPECT_OK(
@@ -386,6 +392,7 @@ class CompactionJobTestBase : public testing::Test {
    }
  }

+  std::shared_ptr<Env> env_guard_;
  Env* env_;
  std::shared_ptr<FileSystem> fs_;
  std::string dbname_;
@@ -1392,6 +1399,7 @@ TEST_F(CompactionJobTimestampTest, SomeKeysExpired) {

  auto expected_results =
      mock::MakeMockFile({{KeyStr("a", 5, ValueType::kTypeValue, 50), "a5"},
+                          {KeyStr("a", 0, ValueType::kTypeValue, 0), "a3"},
                          {KeyStr("b", 6, ValueType::kTypeValue, 49), "b6"}});
  const auto& files = cfd_->current()->storage_info()->LevelFiles(0);

@@ -1403,6 +1411,7 @@ TEST_F(CompactionJobTimestampTest, SomeKeysExpired) {

 int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
+  RegisterCustomObjects(argc, argv);
  return RUN_ALL_TESTS();
 }

@@ -114,7 +114,8 @@ class CompactionPickerTest : public testing::Test {
        InternalKey(largest, largest_seq, kTypeValue), smallest_seq,
        largest_seq, marked_for_compact, kInvalidBlobFileNumber,
        kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-        kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+        kUnknownFileChecksum, kUnknownFileChecksumFuncName,
+        kDisableUserTimestamp, kDisableUserTimestamp);
    f->compensated_file_size =
        (compensated_file_size != 0) ? compensated_file_size : file_size;
    f->temperature = temperature;
@@ -806,14 +806,6 @@ INSTANTIATE_TEST_CASE_P(

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -544,7 +544,7 @@ TEST_F(CorruptionTest, RangeDeletionCorrupted) {
      fs->GetFileSize(filename, file_opts.io_options, &file_size, nullptr));

  BlockHandle range_del_handle;
-  ASSERT_OK(FindMetaBlock(
+  ASSERT_OK(FindMetaBlockInFile(
      file_reader.get(), file_size, kBlockBasedTableMagicNumber,
      ImmutableOptions(options_), kRangeDelBlock, &range_del_handle));

@@ -890,14 +890,6 @@ TEST_F(CorruptionTest, VerifyWholeTableChecksum) {

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -3860,14 +3860,6 @@ INSTANTIATE_TEST_CASE_P(DBBasicTestDeadline, DBBasicTestDeadline,
                                          std::make_tuple(true, true)));
 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -336,6 +336,143 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
  CheckCacheCounters(options, 1, 0, 1, 0);
  CheckCompressedCacheCounters(options, 0, 1, 0, 0);
 }
+
+namespace {
+class PersistentCacheFromCache : public PersistentCache {
+ public:
+  PersistentCacheFromCache(std::shared_ptr<Cache> cache, bool read_only)
+      : cache_(cache), read_only_(read_only) {}
+
+  Status Insert(const Slice& key, const char* data,
+                const size_t size) override {
+    if (read_only_) {
+      return Status::NotSupported();
+    }
+    std::unique_ptr<char[]> copy{new char[size]};
+    std::copy_n(data, size, copy.get());
+    Status s = cache_->Insert(
+        key, copy.get(), size,
+        GetCacheEntryDeleterForRole<char[], CacheEntryRole::kMisc>());
+    if (s.ok()) {
+      copy.release();
+    }
+    return s;
+  }
+
+  Status Lookup(const Slice& key, std::unique_ptr<char[]>* data,
+                size_t* size) override {
+    auto handle = cache_->Lookup(key);
+    if (handle) {
+      char* ptr = static_cast<char*>(cache_->Value(handle));
+      *size = cache_->GetCharge(handle);
+      data->reset(new char[*size]);
+      std::copy_n(ptr, *size, data->get());
+      cache_->Release(handle);
+      return Status::OK();
+    } else {
+      return Status::NotFound();
+    }
+  }
+
+  bool IsCompressed() override { return false; }
+
+  StatsType Stats() override { return StatsType(); }
+
+  std::string GetPrintableOptions() const override { return ""; }
+
+  uint64_t NewId() override { return cache_->NewId(); }
+
+ private:
+  std::shared_ptr<Cache> cache_;
+  bool read_only_;
+};
+
+class ReadOnlyCacheWrapper : public CacheWrapper {
+  using CacheWrapper::CacheWrapper;
+
+  using Cache::Insert;
+  Status Insert(const Slice& /*key*/, void* /*value*/, size_t /*charge*/,
+                void (*)(const Slice& key, void* value) /*deleter*/,
+                Handle** /*handle*/, Priority /*priority*/) override {
+    return Status::NotSupported();
+  }
+};
+
+}  // namespace
+
+TEST_F(DBBlockCacheTest, TestWithSameCompressed) {
+  auto table_options = GetTableOptions();
+  auto options = GetOptions(table_options);
+  InitTable(options);
+
+  std::shared_ptr<Cache> rw_cache{NewLRUCache(1000000)};
+  std::shared_ptr<PersistentCacheFromCache> rw_pcache{
+      new PersistentCacheFromCache(rw_cache, /*read_only*/ false)};
+  // Exercise some obscure behavior with read-only wrappers
+  std::shared_ptr<Cache> ro_cache{new ReadOnlyCacheWrapper(rw_cache)};
+  std::shared_ptr<PersistentCacheFromCache> ro_pcache{
+      new PersistentCacheFromCache(rw_cache, /*read_only*/ true)};
+
+  // Simple same pointer
+  table_options.block_cache = rw_cache;
+  table_options.block_cache_compressed = rw_cache;
+  table_options.persistent_cache.reset();
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  ASSERT_EQ(TryReopen(options).ToString(),
+            "Invalid argument: block_cache same as block_cache_compressed not "
+            "currently supported, and would be bad for performance anyway");
+
+  // Other cases
+  table_options.block_cache = ro_cache;
+  table_options.block_cache_compressed = rw_cache;
+  table_options.persistent_cache.reset();
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  ASSERT_EQ(TryReopen(options).ToString(),
+            "Invalid argument: block_cache and block_cache_compressed share "
+            "the same key space, which is not supported");
+
+  table_options.block_cache = rw_cache;
+  table_options.block_cache_compressed = ro_cache;
+  table_options.persistent_cache.reset();
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  ASSERT_EQ(TryReopen(options).ToString(),
+            "Invalid argument: block_cache_compressed and block_cache share "
+            "the same key space, which is not supported");
+
+  table_options.block_cache = ro_cache;
+  table_options.block_cache_compressed.reset();
+  table_options.persistent_cache = rw_pcache;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  ASSERT_EQ(TryReopen(options).ToString(),
+            "Invalid argument: block_cache and persistent_cache share the same "
+            "key space, which is not supported");
+
+  table_options.block_cache = rw_cache;
+  table_options.block_cache_compressed.reset();
+  table_options.persistent_cache = ro_pcache;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  ASSERT_EQ(TryReopen(options).ToString(),
+            "Invalid argument: persistent_cache and block_cache share the same "
+            "key space, which is not supported");
+
+  table_options.block_cache.reset();
+  table_options.no_block_cache = true;
+  table_options.block_cache_compressed = ro_cache;
+  table_options.persistent_cache = rw_pcache;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  ASSERT_EQ(TryReopen(options).ToString(),
+            "Invalid argument: block_cache_compressed and persistent_cache "
+            "share the same key space, which is not supported");
+
+  table_options.block_cache.reset();
+  table_options.no_block_cache = true;
+  table_options.block_cache_compressed = rw_cache;
+  table_options.persistent_cache = ro_pcache;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  ASSERT_EQ(TryReopen(options).ToString(),
+            "Invalid argument: persistent_cache and block_cache_compressed "
+            "share the same key space, which is not supported");
+}
 #endif  // SNAPPY

 #ifndef ROCKSDB_LITE
@@ -513,13 +650,32 @@ TEST_F(DBBlockCacheTest, WarmCacheWithDataBlocksDuringFlush) {
 }

 // This test cache data, index and filter blocks during flush.
-TEST_F(DBBlockCacheTest, WarmCacheWithBlocksDuringFlush) {
+class DBBlockCacheTest1 : public DBTestBase,
+                          public ::testing::WithParamInterface<bool> {
+ public:
+  const size_t kNumBlocks = 10;
+  const size_t kValueSize = 100;
+  DBBlockCacheTest1() : DBTestBase("db_block_cache_test1", true) {}
+};
+
+INSTANTIATE_TEST_CASE_P(DBBlockCacheTest1, DBBlockCacheTest1,
+                        ::testing::Bool());
+
+TEST_P(DBBlockCacheTest1, WarmCacheWithBlocksDuringFlush) {
  Options options = CurrentOptions();
  options.create_if_missing = true;
  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();

  BlockBasedTableOptions table_options;
  table_options.block_cache = NewLRUCache(1 << 25, 0, false);
+
+  bool use_partition = GetParam();
+  if (use_partition) {
+    table_options.partition_filters = true;
+    table_options.index_type =
+        BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+  }
+
  table_options.cache_index_and_filter_blocks = true;
  table_options.prepopulate_block_cache =
      BlockBasedTableOptions::PrepopulateBlockCache::kFlushOnly;
@@ -532,9 +688,15 @@ TEST_F(DBBlockCacheTest, WarmCacheWithBlocksDuringFlush) {
    ASSERT_OK(Put(ToString(i), value));
    ASSERT_OK(Flush());
    ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_DATA_ADD));
-    ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD));
-    ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD));
-
+    if (use_partition) {
+      ASSERT_EQ(2 * i,
+                options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD));
+      ASSERT_EQ(2 * i,
+                options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD));
+    } else {
+      ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD));
+      ASSERT_EQ(i, options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD));
+    }
    ASSERT_EQ(value, Get(ToString(i)));

    ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_DATA_MISS));
@@ -542,11 +704,16 @@ TEST_F(DBBlockCacheTest, WarmCacheWithBlocksDuringFlush) {

    ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_INDEX_MISS));
    ASSERT_EQ(i * 3, options.statistics->getTickerCount(BLOCK_CACHE_INDEX_HIT));
-
+    if (use_partition) {
+      ASSERT_EQ(i * 3,
+                options.statistics->getTickerCount(BLOCK_CACHE_FILTER_HIT));
+    } else {
+      ASSERT_EQ(i * 2,
+                options.statistics->getTickerCount(BLOCK_CACHE_FILTER_HIT));
+    }
    ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_FILTER_MISS));
-    ASSERT_EQ(i * 2,
-              options.statistics->getTickerCount(BLOCK_CACHE_FILTER_HIT));
  }
+
  // Verify compaction not counted
  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
                              /*end=*/nullptr));
@@ -555,10 +722,17 @@ TEST_F(DBBlockCacheTest, WarmCacheWithBlocksDuringFlush) {
  // Index and filter blocks are automatically warmed when the new table file
  // is automatically opened at the end of compaction. This is not easily
  // disabled so results in the new index and filter blocks being warmed.
-  EXPECT_EQ(1 + kNumBlocks,
-            options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD));
-  EXPECT_EQ(1 + kNumBlocks,
-            options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD));
+  if (use_partition) {
+    EXPECT_EQ(2 * (1 + kNumBlocks),
+              options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD));
+    EXPECT_EQ(2 * (1 + kNumBlocks),
+              options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD));
+  } else {
+    EXPECT_EQ(1 + kNumBlocks,
+              options.statistics->getTickerCount(BLOCK_CACHE_INDEX_ADD));
+    EXPECT_EQ(1 + kNumBlocks,
+              options.statistics->getTickerCount(BLOCK_CACHE_FILTER_ADD));
+  }
 }

 TEST_F(DBBlockCacheTest, DynamicallyWarmCacheDuringFlush) {
@@ -10,9 +10,12 @@
 #include <iomanip>
 #include <sstream>

+#include "cache/cache_entry_roles.h"
+#include "cache/cache_reservation_manager.h"
 #include "db/db_test_util.h"
 #include "options/options_helper.h"
 #include "port/stack_trace.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/perf_context.h"
 #include "table/block_based/filter_policy_internal.h"
 #include "test_util/testutil.h"
@@ -667,6 +670,439 @@ TEST_F(DBBloomFilterTest, BloomFilterReverseCompatibility) {
  }
 }

+/*
+ * A cache wrapper that tracks peaks and increments of filter
+ * construction cache reservation.
+ *        p0
+ *       / \   p1
+ *      /   \  /\
+ *     /     \/  \
+ *  a /       b   \
+ * peaks = {p0, p1}
+ * increments = {p1-a, p2-b}
+ */
+class FilterConstructResPeakTrackingCache : public CacheWrapper {
+ public:
+  explicit FilterConstructResPeakTrackingCache(std::shared_ptr<Cache> target)
+      : CacheWrapper(std::move(target)),
+        cur_cache_res_(0),
+        cache_res_peak_(0),
+        cache_res_increment_(0),
+        last_peak_tracked_(false),
+        cache_res_increments_sum_(0) {}
+
+  using Cache::Insert;
+  Status Insert(const Slice& key, void* value, size_t charge,
+                void (*deleter)(const Slice& key, void* value),
+                Handle** handle = nullptr,
+                Priority priority = Priority::LOW) override {
+    Status s = target_->Insert(key, value, charge, deleter, handle, priority);
+    if (deleter == kNoopDeleterForFilterConstruction) {
+      if (last_peak_tracked_) {
+        cache_res_peak_ = 0;
+        cache_res_increment_ = 0;
+        last_peak_tracked_ = false;
+      }
+      cur_cache_res_ += charge;
+      cache_res_peak_ = std::max(cache_res_peak_, cur_cache_res_);
+      cache_res_increment_ += charge;
+    }
+    return s;
+  }
+
+  using Cache::Release;
+  bool Release(Handle* handle, bool force_erase = false) override {
+    auto deleter = GetDeleter(handle);
+    if (deleter == kNoopDeleterForFilterConstruction) {
+      if (!last_peak_tracked_) {
+        cache_res_peaks_.push_back(cache_res_peak_);
+        cache_res_increments_sum_ += cache_res_increment_;
+        last_peak_tracked_ = true;
+      }
+      cur_cache_res_ -= GetCharge(handle);
+    }
+    bool is_successful = target_->Release(handle, force_erase);
+    return is_successful;
+  }
+
+  std::deque<std::size_t> GetReservedCachePeaks() { return cache_res_peaks_; }
+
+  std::size_t GetReservedCacheIncrementSum() {
+    return cache_res_increments_sum_;
+  }
+
+ private:
+  static const Cache::DeleterFn kNoopDeleterForFilterConstruction;
+
+  std::size_t cur_cache_res_;
+  std::size_t cache_res_peak_;
+  std::size_t cache_res_increment_;
+  bool last_peak_tracked_;
+  std::deque<std::size_t> cache_res_peaks_;
+  std::size_t cache_res_increments_sum_;
+};
+
+const Cache::DeleterFn
+    FilterConstructResPeakTrackingCache::kNoopDeleterForFilterConstruction =
+        CacheReservationManager::TEST_GetNoopDeleterForRole<
+            CacheEntryRole::kFilterConstruction>();
+
+// To align with the type of hash entry being reserved in implementation.
+using FilterConstructionReserveMemoryHash = uint64_t;
+
+class DBFilterConstructionReserveMemoryTestWithParam
+    : public DBTestBase,
+      public testing::WithParamInterface<
+          std::tuple<bool, BloomFilterPolicy::Mode, bool>> {
+ public:
+  DBFilterConstructionReserveMemoryTestWithParam()
+      : DBTestBase("db_bloom_filter_tests",
+                   /*env_do_fsync=*/true),
+        num_key_(0),
+        reserve_table_builder_memory_(std::get<0>(GetParam())),
+        policy_(std::get<1>(GetParam())),
+        partition_filters_(std::get<2>(GetParam())) {
+    if (!reserve_table_builder_memory_ ||
+        policy_ == BloomFilterPolicy::Mode::kDeprecatedBlock ||
+        policy_ == BloomFilterPolicy::Mode::kLegacyBloom) {
+      // For these cases, we only interested in whether filter construction
+      // cache resevation happens instead of its accuracy. Therefore we don't
+      // need many keys.
+      num_key_ = 5;
+    } else if (partition_filters_) {
+      // For PartitionFilter case, since we set
+      // table_options.metadata_block_size big enough such that each partition
+      // trigger at least 1 dummy entry reservation each for hash entries and
+      // final filter, we need a large number of keys to ensure we have at least
+      // two partitions.
+      num_key_ = 18 * CacheReservationManager::GetDummyEntrySize() /
+                 sizeof(FilterConstructionReserveMemoryHash);
+    } else if (policy_ == BloomFilterPolicy::Mode::kFastLocalBloom) {
+      // For Bloom Filter + FullFilter case, since we design the num_key_ to
+      // make hash entry cache reservation be a multiple of dummy entries, the
+      // correct behavior of charging final filter on top of it will trigger at
+      // least another dummy entry insertion. Therefore we can assert that
+      // behavior and we don't need a large number of keys to verify we
+      // indeed charge the final filter for cache reservation, even though final
+      // filter is a lot smaller than hash entries.
+      num_key_ = 1 * CacheReservationManager::GetDummyEntrySize() /
+                 sizeof(FilterConstructionReserveMemoryHash);
+    } else {
+      // For Ribbon Filter + FullFilter case, we need a large enough number of
+      // keys so that charging final filter after releasing the hash entries
+      // reservation will trigger at least another dummy entry (or equivalently
+      // to saying, causing another peak in cache reservation) as banding
+      // reservation might not be a multiple of dummy entry.
+      num_key_ = 12 * CacheReservationManager::GetDummyEntrySize() /
+                 sizeof(FilterConstructionReserveMemoryHash);
+    }
+  }
+
+  BlockBasedTableOptions GetBlockBasedTableOptions() {
+    BlockBasedTableOptions table_options;
+
+    // We set cache capacity big enough to prevent cache full for convenience in
+    // calculation.
+    constexpr std::size_t kCacheCapacity = 100 * 1024 * 1024;
+
+    table_options.reserve_table_builder_memory = reserve_table_builder_memory_;
+    table_options.filter_policy.reset(new BloomFilterPolicy(10, policy_));
+    table_options.partition_filters = partition_filters_;
+    if (table_options.partition_filters) {
+      table_options.index_type =
+          BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+      // We set table_options.metadata_block_size big enough so that each
+      // partition trigger at least 1 dummy entry insertion each for hash
+      // entries and final filter.
+      table_options.metadata_block_size = 409000;
+    }
+
+    LRUCacheOptions lo;
+    lo.capacity = kCacheCapacity;
+    lo.num_shard_bits = 0;  // 2^0 shard
+    lo.strict_capacity_limit = true;
+    cache_ = std::make_shared<FilterConstructResPeakTrackingCache>(
+        (NewLRUCache(lo)));
+    table_options.block_cache = cache_;
+
+    return table_options;
+  }
+
+  std::size_t GetNumKey() { return num_key_; }
+
+  bool ReserveTableBuilderMemory() { return reserve_table_builder_memory_; }
+
+  BloomFilterPolicy::Mode GetFilterPolicy() { return policy_; }
+
+  bool PartitionFilters() { return partition_filters_; }
+
+  std::shared_ptr<FilterConstructResPeakTrackingCache>
+  GetFilterConstructResPeakTrackingCache() {
+    return cache_;
+  }
+
+ private:
+  std::size_t num_key_;
+  bool reserve_table_builder_memory_;
+  BloomFilterPolicy::Mode policy_;
+  bool partition_filters_;
+  std::shared_ptr<FilterConstructResPeakTrackingCache> cache_;
+};
+
+INSTANTIATE_TEST_CASE_P(
+    BlockBasedTableOptions, DBFilterConstructionReserveMemoryTestWithParam,
+    ::testing::Values(
+        std::make_tuple(false, BloomFilterPolicy::Mode::kFastLocalBloom, false),
+        std::make_tuple(true, BloomFilterPolicy::Mode::kFastLocalBloom, false),
+        std::make_tuple(true, BloomFilterPolicy::Mode::kFastLocalBloom, true),
+        std::make_tuple(true, BloomFilterPolicy::Mode::kStandard128Ribbon,
+                        false),
+        std::make_tuple(true, BloomFilterPolicy::Mode::kStandard128Ribbon,
+                        true),
+        std::make_tuple(true, BloomFilterPolicy::Mode::kDeprecatedBlock, false),
+        std::make_tuple(true, BloomFilterPolicy::Mode::kLegacyBloom, false)));
+
+// TODO: Speed up this test.
+// The current test inserts many keys (on the scale of dummy entry size)
+// in order to make small memory user (e.g, final filter, partitioned hash
+// entries/filter/banding) , which is proportional to the number of
+// keys, big enough so that its cache reservation triggers dummy entry insertion
+// and becomes observable in the test.
+//
+// However, inserting that many keys slows down this test and leaves future
+// developers an opportunity to speed it up.
+//
+// Possible approaches & challenges:
+// 1. Use sync point during cache reservation of filter construction
+//
+// Benefit: It does not rely on triggering dummy entry insertion
+// but the sync point to verify small memory user is charged correctly.
+//
+// Challenge: this approach is intrusive.
+//
+// 2. Make dummy entry size configurable and set it small in the test
+//
+// Benefit: It increases the precision of cache reservation and therefore
+// small memory usage can still trigger insertion of dummy entry.
+//
+// Challenge: change CacheReservationManager related APIs and a hack
+// might be needed to control the size of dummmy entry of
+// CacheReservationManager used in filter construction for testing
+// since CacheReservationManager is not exposed at the high level.
+//
+TEST_P(DBFilterConstructionReserveMemoryTestWithParam, ReserveMemory) {
+  Options options = CurrentOptions();
+  // We set write_buffer_size big enough so that in the case where there is
+  // filter construction cache reservation, flush won't be triggered before we
+  // manually trigger it for clean testing
+  options.write_buffer_size = 640 << 20;
+  options.table_factory.reset(
+      NewBlockBasedTableFactory(GetBlockBasedTableOptions()));
+  std::shared_ptr<FilterConstructResPeakTrackingCache> cache =
+      GetFilterConstructResPeakTrackingCache();
+  options.create_if_missing = true;
+  // Disable auto compaction to prevent its unexpected side effect
+  // to the number of keys per partition designed by us in the test
+  options.disable_auto_compactions = true;
+  DestroyAndReopen(options);
+  int num_key = static_cast<int>(GetNumKey());
+  for (int i = 0; i < num_key; i++) {
+    ASSERT_OK(Put(Key(i), Key(i)));
+  }
+
+  ASSERT_EQ(cache->GetReservedCacheIncrementSum(), 0)
+      << "Flush was triggered too early in the test case with filter "
+         "construction cache reservation - please make sure no flush triggered "
+         "during the key insertions above";
+
+  ASSERT_OK(Flush());
+
+  bool reserve_table_builder_memory = ReserveTableBuilderMemory();
+  BloomFilterPolicy::Mode policy = GetFilterPolicy();
+  bool partition_filters = PartitionFilters();
+
+  std::deque<std::size_t> filter_construction_cache_res_peaks =
+      cache->GetReservedCachePeaks();
+  std::size_t filter_construction_cache_res_increments_sum =
+      cache->GetReservedCacheIncrementSum();
+
+  if (!reserve_table_builder_memory) {
+    EXPECT_EQ(filter_construction_cache_res_peaks.size(), 0);
+    return;
+  }
+
+  if (policy == BloomFilterPolicy::Mode::kDeprecatedBlock ||
+      policy == BloomFilterPolicy::Mode::kLegacyBloom) {
+    EXPECT_EQ(filter_construction_cache_res_peaks.size(), 0)
+        << "There shouldn't be filter construction cache reservation as this "
+           "feature does not support BloomFilterPolicy::Mode::kDeprecatedBlock "
+           "nor BloomFilterPolicy::Mode::kLegacyBloom";
+    return;
+  }
+
+  const std::size_t kDummyEntrySize =
+      CacheReservationManager::GetDummyEntrySize();
+
+  const std::size_t predicted_hash_entries_cache_res =
+      num_key * sizeof(FilterConstructionReserveMemoryHash);
+  ASSERT_EQ(predicted_hash_entries_cache_res % kDummyEntrySize, 0)
+      << "It's by this test's design that predicted_hash_entries_cache_res is "
+         "a multipe of dummy entry";
+
+  const std::size_t predicted_hash_entries_cache_res_dummy_entry_num =
+      predicted_hash_entries_cache_res / kDummyEntrySize;
+  const std::size_t predicted_final_filter_cache_res =
+      static_cast<std::size_t>(std::ceil(
+          1.0 * predicted_hash_entries_cache_res_dummy_entry_num / 6 *
+          (policy == BloomFilterPolicy::Mode::kStandard128Ribbon ? 0.7 : 1))) *
+      kDummyEntrySize;
+  const std::size_t predicted_banding_cache_res =
+      static_cast<std::size_t>(
+          std::ceil(predicted_hash_entries_cache_res_dummy_entry_num * 2.5)) *
+      kDummyEntrySize;
+
+  if (policy == BloomFilterPolicy::Mode::kFastLocalBloom) {
+    /* BloomFilterPolicy::Mode::kFastLocalBloom + FullFilter
+     *        p0
+     *       /  \
+     *    b /    \
+     *     /      \
+     *    /        \
+     *  0/          \
+     *  hash entries = b - 0, final filter = p0 - b
+     *  p0 = hash entries + final filter
+     *
+     *  The test is designed in a way such that the reservation for b is a
+     *  multiple of dummy entries so that reservation for (p0 - b)
+     *  will trigger at least another dummy entry insertion.
+     *
+     * BloomFilterPolicy::Mode::kFastLocalBloom + PartitionedFilter
+     *                   p1
+     *                  /  \
+     *        p0     b'/    \
+     *       /  \     /      \
+     *    b /    \   /        \
+     *     /      \ /          \
+     *    /        a            \
+     *  0/                       \
+     *  partitioned hash entries1 = b - 0, partitioned hash entries1 = b' - a
+     *  parittioned final filter1 = p0 - b, parittioned final filter2 = p1 - b'
+     *
+     *  (increment p0 - 0) + (increment p1 - a)
+     *  = partitioned hash entries1 + partitioned hash entries2
+     *  + parittioned final filter1 + parittioned final filter2
+     *  = hash entries + final filter
+     *
+     */
+    if (!partition_filters) {
+      EXPECT_EQ(filter_construction_cache_res_peaks.size(), 1)
+          << "Filter construction cache reservation should have only 1 peak in "
+             "case: BloomFilterPolicy::Mode::kFastLocalBloom + FullFilter";
+      std::size_t filter_construction_cache_res_peak =
+          filter_construction_cache_res_peaks[0];
+      EXPECT_GT(filter_construction_cache_res_peak,
+                predicted_hash_entries_cache_res)
+          << "The testing number of hash entries is designed to make hash "
+             "entries cache reservation be multiples of dummy entries"
+             " so the correct behavior of charging final filter on top of it"
+             " should've triggered at least another dummy entry insertion";
+
+      std::size_t predicted_filter_construction_cache_res_peak =
+          predicted_hash_entries_cache_res + predicted_final_filter_cache_res;
+      EXPECT_GE(filter_construction_cache_res_peak,
+                predicted_filter_construction_cache_res_peak * 0.9);
+      EXPECT_LE(filter_construction_cache_res_peak,
+                predicted_filter_construction_cache_res_peak * 1.1);
+      return;
+    } else {
+      EXPECT_GE(filter_construction_cache_res_peaks.size(), 2)
+          << "Filter construction cache reservation should have multiple peaks "
+             "in case: BloomFilterPolicy::Mode::kFastLocalBloom + "
+             "PartitionedFilter";
+      std::size_t predicted_filter_construction_cache_res_increments_sum =
+          predicted_hash_entries_cache_res + predicted_final_filter_cache_res;
+      EXPECT_GE(filter_construction_cache_res_increments_sum,
+                predicted_filter_construction_cache_res_increments_sum * 0.9);
+      EXPECT_LE(filter_construction_cache_res_increments_sum,
+                predicted_filter_construction_cache_res_increments_sum * 1.1);
+      return;
+    }
+  }
+
+  if (policy == BloomFilterPolicy::Mode::kStandard128Ribbon) {
+    /* BloomFilterPolicy::Mode::kStandard128Ribbon + FullFilter
+     *        p0
+     *       /  \  p1
+     *      /    \/\
+     *   b /     b' \
+     *    /          \
+     *  0/            \
+     *  hash entries = b - 0, banding = p0 - b, final filter = p1 - b'
+     *  p0 = hash entries + banding
+     *
+     *  The test is designed in a way such that the reservation for (p1 - b')
+     *  will trigger at least another dummy entry insertion
+     *  (or equivelantly to saying, creating another peak).
+     *
+     *  BloomFilterPolicy::Mode::kStandard128Ribbon + PartitionedFilter
+     *                     p3
+     *        p0           /\  p4
+     *       /  \ p1      /  \ /\
+     *      /    \/\  b''/    a' \
+     *   b /     b' \   /         \
+     *    /          \ /           \
+     *  0/            a             \
+     *  partitioned hash entries1 = b - 0, partitioned hash entries2 = b'' - a
+     *  partitioned banding1 = p0 - b, partitioned banding2 = p3 - b''
+     *  parittioned final filter1 = p1 - b',parittioned final filter2 = p4 - a'
+     *
+     *  (increment p0 - 0) + (increment p1 - b')
+     *  + (increment p3 - a) + (increment p4 - a')
+     *  = partitioned hash entries1 + partitioned hash entries2
+     *  + parittioned banding1 + parittioned banding2
+     *  + parittioned final filter1 + parittioned final filter2
+     *  = hash entries + banding + final filter
+     */
+    if (!partition_filters) {
+      ASSERT_GE(std::floor(1.0 * predicted_final_filter_cache_res /
+                           CacheReservationManager::GetDummyEntrySize()),
+                1)
+          << "Final filter cache reservation too small for this test - please "
+             "increase the number of keys";
+      EXPECT_EQ(filter_construction_cache_res_peaks.size(), 2)
+          << "Filter construction cache reservation should have 2 peaks in "
+             "case: BloomFilterPolicy::Mode::kStandard128Ribbon + FullFilter. "
+             "The second peak is resulted from charging the final filter after "
+             "decreasing the hash entry reservation since the testing final "
+             "filter reservation is designed to be at least 1 dummy entry size";
+
+      std::size_t filter_construction_cache_res_peak =
+          filter_construction_cache_res_peaks[0];
+      std::size_t predicted_filter_construction_cache_res_peak =
+          predicted_hash_entries_cache_res + predicted_banding_cache_res;
+      EXPECT_GE(filter_construction_cache_res_peak,
+                predicted_filter_construction_cache_res_peak * 0.9);
+      EXPECT_LE(filter_construction_cache_res_peak,
+                predicted_filter_construction_cache_res_peak * 1.1);
+      return;
+    } else {
+      EXPECT_GE(filter_construction_cache_res_peaks.size(), 3)
+          << "Filter construction cache reservation should have more than 3 "
+             "peaks "
+             "in case: BloomFilterPolicy::Mode::kStandard128Ribbon + "
+             "PartitionedFilter";
+      std::size_t predicted_filter_construction_cache_res_increments_sum =
+          predicted_hash_entries_cache_res + predicted_banding_cache_res +
+          predicted_final_filter_cache_res;
+      EXPECT_GE(filter_construction_cache_res_increments_sum,
+                predicted_filter_construction_cache_res_increments_sum * 0.9);
+      EXPECT_LE(filter_construction_cache_res_increments_sum,
+                predicted_filter_construction_cache_res_increments_sum * 1.1);
+      return;
+    }
+  }
+}
+
 namespace {
 // A wrapped bloom over block-based FilterPolicy
 class TestingWrappedBlockBasedFilterPolicy : public FilterPolicy {
@@ -7000,6 +7000,65 @@ TEST_F(DBCompactionTest, ChangeLevelConflictsWithManual) {
  refit_level_thread.join();
 }

+TEST_F(DBCompactionTest, BottomPriCompactionCountsTowardConcurrencyLimit) {
+  // Flushes several files to trigger compaction while lock is released during
+  // a bottom-pri compaction. Verifies it does not get scheduled to thread pool
+  // because per-DB limit for compaction parallelism is one (default).
+  const int kNumL0Files = 4;
+  const int kNumLevels = 3;
+
+  env_->SetBackgroundThreads(1, Env::Priority::BOTTOM);
+
+  Options options = CurrentOptions();
+  options.level0_file_num_compaction_trigger = kNumL0Files;
+  options.num_levels = kNumLevels;
+  DestroyAndReopen(options);
+
+  // Setup last level to be non-empty since it's a bit unclear whether
+  // compaction to an empty level would be considered "bottommost".
+  ASSERT_OK(Put(Key(0), "val"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(kNumLevels - 1);
+
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::BGWorkBottomCompaction",
+        "DBCompactionTest::BottomPriCompactionCountsTowardConcurrencyLimit:"
+        "PreTriggerCompaction"},
+       {"DBCompactionTest::BottomPriCompactionCountsTowardConcurrencyLimit:"
+        "PostTriggerCompaction",
+        "BackgroundCallCompaction:0"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  port::Thread compact_range_thread([&] {
+    CompactRangeOptions cro;
+    cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+    cro.exclusive_manual_compaction = false;
+    ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
+  });
+
+  // Sleep in the low-pri thread so any newly scheduled compaction will be
+  // queued. Otherwise it might finish before we check its existence.
+  test::SleepingBackgroundTask sleeping_task_low;
+  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
+                 Env::Priority::LOW);
+
+  TEST_SYNC_POINT(
+      "DBCompactionTest::BottomPriCompactionCountsTowardConcurrencyLimit:"
+      "PreTriggerCompaction");
+  for (int i = 0; i < kNumL0Files; ++i) {
+    ASSERT_OK(Put(Key(0), "val"));
+    ASSERT_OK(Flush());
+  }
+  ASSERT_EQ(0u, env_->GetThreadPoolQueueLen(Env::Priority::LOW));
+  TEST_SYNC_POINT(
+      "DBCompactionTest::BottomPriCompactionCountsTowardConcurrencyLimit:"
+      "PostTriggerCompaction");
+
+  sleeping_task_low.WakeUp();
+  sleeping_task_low.WaitUntilDone();
+  compact_range_thread.join();
+}
+
 #endif  // !defined(ROCKSDB_LITE)

 }  // namespace ROCKSDB_NAMESPACE
@@ -127,31 +127,30 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
 }

 Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
+  // If caller disabled deletions, this function should return files that are
+  // guaranteed not to be deleted until deletions are re-enabled. We need to
+  // wait for pending purges to finish since WalManager doesn't know which
+  // files are going to be purged. Additional purges won't be scheduled as
+  // long as deletions are disabled (so the below loop must terminate).
+  // Also note that we disable deletions anyway to avoid the case where a
+  // file is deleted in the middle of the scan, causing IO error.
+  Status deletions_disabled = DisableFileDeletions();
  {
-    // If caller disabled deletions, this function should return files that are
-    // guaranteed not to be deleted until deletions are re-enabled. We need to
-    // wait for pending purges to finish since WalManager doesn't know which
-    // files are going to be purged. Additional purges won't be scheduled as
-    // long as deletions are disabled (so the below loop must terminate).
    InstrumentedMutexLock l(&mutex_);
-    while (disable_delete_obsolete_files_ > 0 &&
-           (pending_purge_obsolete_files_ > 0 || bg_purge_scheduled_ > 0)) {
+    while (pending_purge_obsolete_files_ > 0 || bg_purge_scheduled_ > 0) {
      bg_cv_.Wait();
    }
  }

-  // Disable deletion in order to avoid the case where a file is deleted in
-  // the middle of the process so IO error is returned.
-  Status s = DisableFileDeletions();
-  bool file_deletion_supported = !s.IsNotSupported();
-  if (s.ok() || !file_deletion_supported) {
-    s = wal_manager_.GetSortedWalFiles(files);
-    if (file_deletion_supported) {
-      Status s2 = EnableFileDeletions(false);
-      if (!s2.ok() && s.ok()) {
-        s = s2;
-      }
-    }
+  Status s = wal_manager_.GetSortedWalFiles(files);
+
+  // DisableFileDeletions / EnableFileDeletions not supported in read-only DB
+  if (deletions_disabled.ok()) {
+    Status s2 = EnableFileDeletions(/*force*/ false);
+    assert(s2.ok());
+    s2.PermitUncheckedError();
+  } else {
+    assert(deletions_disabled.IsNotSupported());
  }

  return s;
@@ -1199,7 +1199,7 @@ TEST_F(DBFlushTest, MemPurgeAndCompactionFilter) {
  ASSERT_EQ(Get(KEY5), p_v5);
 }

-TEST_F(DBFlushTest, MemPurgeWALSupport) {
+TEST_F(DBFlushTest, DISABLED_MemPurgeWALSupport) {
  Options options = CurrentOptions();

  options.statistics = CreateDBStatistics();
@@ -1858,6 +1858,50 @@ TEST_F(DBFlushTest, FlushWithChecksumHandoffManifest2) {
  Destroy(options);
 }

+TEST_F(DBFlushTest, PickRightMemtables) {
+  Options options = CurrentOptions();
+  DestroyAndReopen(options);
+  options.create_if_missing = true;
+
+  const std::string test_cf_name = "test_cf";
+  options.max_write_buffer_number = 128;
+  CreateColumnFamilies({test_cf_name}, options);
+
+  Close();
+
+  ReopenWithColumnFamilies({kDefaultColumnFamilyName, test_cf_name}, options);
+
+  ASSERT_OK(db_->Put(WriteOptions(), "key", "value"));
+
+  ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "key", "value"));
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::SyncClosedLogs:BeforeReLock", [&](void* /*arg*/) {
+        ASSERT_OK(db_->Put(WriteOptions(), handles_[1], "what", "v"));
+        auto* cfhi =
+            static_cast_with_check<ColumnFamilyHandleImpl>(handles_[1]);
+        assert(cfhi);
+        ASSERT_OK(dbfull()->TEST_SwitchMemtable(cfhi->cfd()));
+      });
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::FlushMemTableToOutputFile:AfterPickMemtables", [&](void* arg) {
+        auto* job = reinterpret_cast<FlushJob*>(arg);
+        assert(job);
+        const auto& mems = job->GetMemTables();
+        assert(mems.size() == 1);
+        assert(mems[0]);
+        ASSERT_EQ(1, mems[0]->GetID());
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(db_->Flush(FlushOptions(), handles_[1]));
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 class DBFlushTestBlobError : public DBFlushTest,
                             public testing::WithParamInterface<std::string> {
 public:
@@ -1546,6 +1546,8 @@ void DBImpl::BackgroundCallPurge() {
    mutex_.Lock();
  }

+  assert(bg_purge_scheduled_ > 0);
+
  // Can't use iterator to go over purge_files_ because inside the loop we're
  // unlocking the mutex that protects purge_files_.
  while (!purge_files_.empty()) {
@@ -1613,17 +1615,7 @@ static void CleanupIteratorState(void* arg1, void* /*arg2*/) {
      delete state->super_version;
    }
    if (job_context.HaveSomethingToDelete()) {
-      if (state->background_purge) {
-        // PurgeObsoleteFiles here does not delete files. Instead, it adds the
-        // files to be deleted to a job queue, and deletes it in a separate
-        // background thread.
-        state->db->PurgeObsoleteFiles(job_context, true /* schedule only */);
-        state->mu->Lock();
-        state->db->SchedulePurge();
-        state->mu->Unlock();
-      } else {
-        state->db->PurgeObsoleteFiles(job_context);
-      }
+      state->db->PurgeObsoleteFiles(job_context, state->background_purge);
    }
    job_context.Clean();
  }
@@ -2903,6 +2895,13 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
  }
  // if iterator wants internal keys, we can only proceed if
  // we can guarantee the deletes haven't been processed yet
+  if (read_options.iter_start_seqnum > 0 &&
+      !iter_start_seqnum_deprecation_warned_.exchange(true)) {
+    ROCKS_LOG_WARN(
+        immutable_db_options_.info_log,
+        "iter_start_seqnum is deprecated, will be removed in a future release. "
+        "Please try using user-defined timestamp instead.");
+  }
  if (immutable_db_options_.preserve_deletes &&
      read_options.iter_start_seqnum > 0 &&
      read_options.iter_start_seqnum < preserve_deletes_seqnum_.load()) {
@@ -4376,25 +4375,38 @@ SequenceNumber DBImpl::GetEarliestMemTableSequenceNumber(SuperVersion* sv,
  return earliest_seq;
 }

-Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
-                                       bool cache_only,
-                                       SequenceNumber lower_bound_seq,
-                                       SequenceNumber* seq,
-                                       bool* found_record_for_key,
-                                       bool* is_blob_index) {
+Status DBImpl::GetLatestSequenceForKey(
+    SuperVersion* sv, const Slice& key, bool cache_only,
+    SequenceNumber lower_bound_seq, SequenceNumber* seq, std::string* timestamp,
+    bool* found_record_for_key, bool* is_blob_index) {
  Status s;
  MergeContext merge_context;
  SequenceNumber max_covering_tombstone_seq = 0;

  ReadOptions read_options;
  SequenceNumber current_seq = versions_->LastSequence();
-  LookupKey lkey(key, current_seq);
+
+  ColumnFamilyData* cfd = sv->cfd;
+  assert(cfd);
+  const Comparator* const ucmp = cfd->user_comparator();
+  assert(ucmp);
+  size_t ts_sz = ucmp->timestamp_size();
+  std::string ts_buf;
+  if (ts_sz > 0) {
+    assert(timestamp);
+    ts_buf.assign(ts_sz, '\xff');
+  } else {
+    assert(!timestamp);
+  }
+  Slice ts(ts_buf);
+
+  LookupKey lkey(key, current_seq, ts_sz == 0 ? nullptr : &ts);

  *seq = kMaxSequenceNumber;
  *found_record_for_key = false;

  // Check if there is a record for this key in the latest memtable
-  sv->mem->Get(lkey, nullptr, nullptr, &s, &merge_context,
+  sv->mem->Get(lkey, /*value=*/nullptr, timestamp, &s, &merge_context,
               &max_covering_tombstone_seq, seq, read_options,
               nullptr /*read_callback*/, is_blob_index);

@@ -4406,6 +4418,10 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,

    return s;
  }
+  assert(!ts_sz ||
+         (*seq != kMaxSequenceNumber &&
+          *timestamp != std::string(ts_sz, '\xff')) ||
+         (*seq == kMaxSequenceNumber && timestamp->empty()));

  if (*seq != kMaxSequenceNumber) {
    // Found a sequence number, no need to check immutable memtables
@@ -4421,7 +4437,7 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
  }

  // Check if there is a record for this key in the immutable memtables
-  sv->imm->Get(lkey, nullptr, nullptr, &s, &merge_context,
+  sv->imm->Get(lkey, /*value=*/nullptr, timestamp, &s, &merge_context,
               &max_covering_tombstone_seq, seq, read_options,
               nullptr /*read_callback*/, is_blob_index);

@@ -4434,6 +4450,11 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
    return s;
  }

+  assert(!ts_sz ||
+         (*seq != kMaxSequenceNumber &&
+          *timestamp != std::string(ts_sz, '\xff')) ||
+         (*seq == kMaxSequenceNumber && timestamp->empty()));
+
  if (*seq != kMaxSequenceNumber) {
    // Found a sequence number, no need to check memtable history
    *found_record_for_key = true;
@@ -4448,9 +4469,9 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
  }

  // Check if there is a record for this key in the immutable memtables
-  sv->imm->GetFromHistory(lkey, nullptr, nullptr, &s, &merge_context,
-                          &max_covering_tombstone_seq, seq, read_options,
-                          is_blob_index);
+  sv->imm->GetFromHistory(lkey, /*value=*/nullptr, timestamp, &s,
+                          &merge_context, &max_covering_tombstone_seq, seq,
+                          read_options, is_blob_index);

  if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
    // unexpected error reading memtable.
@@ -4462,8 +4483,13 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
    return s;
  }

+  assert(!ts_sz ||
+         (*seq != kMaxSequenceNumber &&
+          *timestamp != std::string(ts_sz, '\xff')) ||
+         (*seq == kMaxSequenceNumber && timestamp->empty()));
  if (*seq != kMaxSequenceNumber) {
    // Found a sequence number, no need to check SST files
+    assert(0 == ts_sz || *timestamp != std::string(ts_sz, '\xff'));
    *found_record_for_key = true;
    return Status::OK();
  }
@@ -4476,10 +4502,10 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
  // SST files if cache_only=true?
  if (!cache_only) {
    // Check tables
-    sv->current->Get(read_options, lkey, nullptr, nullptr, &s, &merge_context,
-                     &max_covering_tombstone_seq, nullptr /* value_found */,
-                     found_record_for_key, seq, nullptr /*read_callback*/,
-                     is_blob_index);
+    sv->current->Get(read_options, lkey, /*value=*/nullptr, timestamp, &s,
+                     &merge_context, &max_covering_tombstone_seq,
+                     nullptr /* value_found */, found_record_for_key, seq,
+                     nullptr /*read_callback*/, is_blob_index);

    if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
      // unexpected error reading SST files
@@ -582,9 +582,15 @@ class DBImpl : public DB {
  // in the memtables, including memtable history.  If cache_only is false,
  // SST files will also be checked.
  //
+  // `key` should NOT have user-defined timestamp appended to user key even if
+  // timestamp is enabled.
+  //
  // If a key is found, *found_record_for_key will be set to true and
  // *seq will be set to the stored sequence number for the latest
-  // operation on this key or kMaxSequenceNumber if unknown.
+  // operation on this key or kMaxSequenceNumber if unknown. If user-defined
+  // timestamp is enabled for this column family and timestamp is not nullptr,
+  // then *timestamp will be set to the stored timestamp for the latest
+  // operation on this key.
  // If no key is found, *found_record_for_key will be set to false.
  //
  // Note: If cache_only=false, it is possible for *seq to be set to 0 if
@@ -608,9 +614,9 @@ class DBImpl : public DB {
  Status GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
                                 bool cache_only,
                                 SequenceNumber lower_bound_seq,
-                                 SequenceNumber* seq,
+                                 SequenceNumber* seq, std::string* timestamp,
                                 bool* found_record_for_key,
-                                 bool* is_blob_index = nullptr);
+                                 bool* is_blob_index);

  Status TraceIteratorSeek(const uint32_t& cf_id, const Slice& key,
                           const Slice& lower_bound, const Slice upper_bound);
@@ -2315,6 +2321,10 @@ class DBImpl : public DB {

  // Pointer to WriteBufferManager stalling interface.
  std::unique_ptr<StallInterface> wbm_stall_;
+
+  // Indicate if deprecation warning message is logged before. Will be removed
+  // soon with the deprecated feature.
+  std::atomic_bool iter_start_seqnum_deprecation_warned_{false};
 };

 extern Options SanitizeOptions(const std::string& db, const Options& src,
@@ -103,6 +103,8 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context) {
  if (!logs_to_sync.empty()) {
    mutex_.Unlock();

+    assert(job_context);
+
    for (log::Writer* log : logs_to_sync) {
      ROCKS_LOG_INFO(immutable_db_options_.info_log,
                     "[JOB %d] Syncing log #%" PRIu64, job_context->job_id,
@@ -125,6 +127,8 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context) {
          DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced));
    }

+    TEST_SYNC_POINT_CALLBACK("DBImpl::SyncClosedLogs:BeforeReLock",
+                             /*arg=*/nullptr);
    mutex_.Lock();

    // "number <= current_log_number - 1" is equivalent to
@@ -153,14 +157,34 @@ Status DBImpl::FlushMemTableToOutputFile(
    Env::Priority thread_pri) {
  mutex_.AssertHeld();
  assert(cfd);
+  assert(cfd->imm());
  assert(cfd->imm()->NumNotFlushed() != 0);
  assert(cfd->imm()->IsFlushPending());
+  assert(versions_);
+  assert(versions_->GetColumnFamilySet());
+  // If there are more than one column families, we need to make sure that
+  // all the log files except the most recent one are synced. Otherwise if
+  // the host crashes after flushing and before WAL is persistent, the
+  // flushed SST may contain data from write batches whose updates to
+  // other (unflushed) column families are missing.
+  const bool needs_to_sync_closed_wals =
+      logfile_number_ > 0 &&
+      versions_->GetColumnFamilySet()->NumberOfColumnFamilies() > 1;
+  // If needs_to_sync_closed_wals is true, we need to record the current
+  // maximum memtable ID of this column family so that a later PickMemtables()
+  // call will not pick memtables whose IDs are higher. This is due to the fact
+  // that SyncClosedLogs() may release the db mutex, and memtable switch can
+  // happen for this column family in the meantime. The newly created memtables
+  // have their data backed by unsynced WALs, thus they cannot be included in
+  // this flush job.
+  uint64_t max_memtable_id = needs_to_sync_closed_wals
+                                 ? cfd->imm()->GetLatestMemTableID()
+                                 : port::kMaxUint64;
  FlushJob flush_job(
-      dbname_, cfd, immutable_db_options_, mutable_cf_options,
-      port::kMaxUint64 /* memtable_id */, file_options_for_compaction_,
-      versions_.get(), &mutex_, &shutting_down_, snapshot_seqs,
-      earliest_write_conflict_snapshot, snapshot_checker, job_context,
-      log_buffer, directories_.GetDbDir(), GetDataDir(cfd, 0U),
+      dbname_, cfd, immutable_db_options_, mutable_cf_options, max_memtable_id,
+      file_options_for_compaction_, versions_.get(), &mutex_, &shutting_down_,
+      snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker,
+      job_context, log_buffer, directories_.GetDbDir(), GetDataDir(cfd, 0U),
      GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
      &event_logger_, mutable_cf_options.report_bg_io_stats,
      true /* sync_output_directory */, true /* write_manifest */, thread_pri,
@@ -176,13 +200,7 @@ Status DBImpl::FlushMemTableToOutputFile(
  Status s;
  bool need_cancel = false;
  IOStatus log_io_s = IOStatus::OK();
-  if (logfile_number_ > 0 &&
-      versions_->GetColumnFamilySet()->NumberOfColumnFamilies() > 1) {
-    // If there are more than one column families, we need to make sure that
-    // all the log files except the most recent one are synced. Otherwise if
-    // the host crashes after flushing and before WAL is persistent, the
-    // flushed SST may contain data from write batches whose updates to
-    // other column families are missing.
+  if (needs_to_sync_closed_wals) {
    // SyncClosedLogs() may unlock and re-lock the db_mutex.
    log_io_s = SyncClosedLogs(job_context);
    if (!log_io_s.ok() && !log_io_s.IsShutdownInProgress() &&
@@ -201,7 +219,8 @@ Status DBImpl::FlushMemTableToOutputFile(
    flush_job.PickMemTable();
    need_cancel = true;
  }
-  TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:AfterPickMemtables");
+  TEST_SYNC_POINT_CALLBACK(
+      "DBImpl::FlushMemTableToOutputFile:AfterPickMemtables", &flush_job);
  bool switched_to_mempurge = false;
  // Within flush_job.Run, rocksdb may call event listener to notify
  // file creation and deletion.
@@ -1589,12 +1608,12 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
    edit.SetColumnFamily(cfd->GetID());
    for (const auto& f : vstorage->LevelFiles(level)) {
      edit.DeleteFile(level, f->fd.GetNumber());
-      edit.AddFile(to_level, f->fd.GetNumber(), f->fd.GetPathId(),
-                   f->fd.GetFileSize(), f->smallest, f->largest,
-                   f->fd.smallest_seqno, f->fd.largest_seqno,
-                   f->marked_for_compaction, f->oldest_blob_file_number,
-                   f->oldest_ancester_time, f->file_creation_time,
-                   f->file_checksum, f->file_checksum_func_name);
+      edit.AddFile(
+          to_level, f->fd.GetNumber(), f->fd.GetPathId(), f->fd.GetFileSize(),
+          f->smallest, f->largest, f->fd.smallest_seqno, f->fd.largest_seqno,
+          f->marked_for_compaction, f->oldest_blob_file_number,
+          f->oldest_ancester_time, f->file_creation_time, f->file_checksum,
+          f->file_checksum_func_name, f->min_timestamp, f->max_timestamp);
    }
    ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                    "[%s] Apply version edit:\n%s", cfd->GetName().c_str(),
@@ -1843,7 +1862,6 @@ Status DBImpl::RunManualCompaction(
      }
      ca = new CompactionArg;
      ca->db = this;
-      ca->compaction_pri_ = Env::Priority::LOW;
      ca->prepicked_compaction = new PrepickedCompaction;
      ca->prepicked_compaction->manual_compaction_state = &manual;
      ca->prepicked_compaction->compaction = compaction;
@@ -1853,14 +1871,19 @@ Status DBImpl::RunManualCompaction(
        assert(false);
      }
      manual.incomplete = false;
-      bg_compaction_scheduled_++;
-      Env::Priority thread_pool_pri = Env::Priority::LOW;
      if (compaction->bottommost_level() &&
          env_->GetBackgroundThreads(Env::Priority::BOTTOM) > 0) {
-        thread_pool_pri = Env::Priority::BOTTOM;
+        bg_bottom_compaction_scheduled_++;
+        ca->compaction_pri_ = Env::Priority::BOTTOM;
+        env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca,
+                       Env::Priority::BOTTOM, this,
+                       &DBImpl::UnscheduleCompactionCallback);
+      } else {
+        bg_compaction_scheduled_++;
+        ca->compaction_pri_ = Env::Priority::LOW;
+        env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
+                       &DBImpl::UnscheduleCompactionCallback);
      }
-      env_->Schedule(&DBImpl::BGWorkCompaction, ca, thread_pool_pri, this,
-                     &DBImpl::UnscheduleCompactionCallback);
      scheduled = true;
    }
  }
@@ -2375,7 +2398,8 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
    return;
  }

-  while (bg_compaction_scheduled_ < bg_job_limits.max_compactions &&
+  while (bg_compaction_scheduled_ + bg_bottom_compaction_scheduled_ <
+             bg_job_limits.max_compactions &&
         unscheduled_compactions_ > 0) {
    CompactionArg* ca = new CompactionArg;
    ca->db = this;
@@ -2564,8 +2588,7 @@ void DBImpl::BGWorkBottomCompaction(void* arg) {
  IOSTATS_SET_THREAD_POOL_ID(Env::Priority::BOTTOM);
  TEST_SYNC_POINT("DBImpl::BGWorkBottomCompaction");
  auto* prepicked_compaction = ca.prepicked_compaction;
-  assert(prepicked_compaction && prepicked_compaction->compaction &&
-         !prepicked_compaction->manual_compaction_state);
+  assert(prepicked_compaction && prepicked_compaction->compaction);
  ca.db->BackgroundCallCompaction(prepicked_compaction, Env::Priority::BOTTOM);
  delete prepicked_compaction;
 }
@@ -3170,13 +3193,13 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      for (size_t i = 0; i < c->num_input_files(l); i++) {
        FileMetaData* f = c->input(l, i);
        c->edit()->DeleteFile(c->level(l), f->fd.GetNumber());
-        c->edit()->AddFile(c->output_level(), f->fd.GetNumber(),
-                           f->fd.GetPathId(), f->fd.GetFileSize(), f->smallest,
-                           f->largest, f->fd.smallest_seqno,
-                           f->fd.largest_seqno, f->marked_for_compaction,
-                           f->oldest_blob_file_number, f->oldest_ancester_time,
-                           f->file_creation_time, f->file_checksum,
-                           f->file_checksum_func_name);
+        c->edit()->AddFile(
+            c->output_level(), f->fd.GetNumber(), f->fd.GetPathId(),
+            f->fd.GetFileSize(), f->smallest, f->largest, f->fd.smallest_seqno,
+            f->fd.largest_seqno, f->marked_for_compaction,
+            f->oldest_blob_file_number, f->oldest_ancester_time,
+            f->file_creation_time, f->file_checksum, f->file_checksum_func_name,
+            f->min_timestamp, f->max_timestamp);

        ROCKS_LOG_BUFFER(
            log_buffer,
@@ -131,7 +131,8 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
                   f->fd.smallest_seqno, f->fd.largest_seqno,
                   f->marked_for_compaction, f->oldest_blob_file_number,
                   f->oldest_ancester_time, f->file_creation_time,
-                   f->file_checksum, f->file_checksum_func_name);
+                   f->file_checksum, f->file_checksum_func_name,
+                   f->min_timestamp, f->max_timestamp);
    }

    status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
@@ -56,6 +56,8 @@ Status DBImpl::DisableFileDeletions() {
  return s;
 }

+// FIXME: can be inconsistent with DisableFileDeletions in cases like
+// DBImplReadOnly
 Status DBImpl::DisableFileDeletionsWithLock() {
  mutex_.AssertHeld();
  ++disable_delete_obsolete_files_;
@@ -642,6 +644,11 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
  InstrumentedMutexLock l(&mutex_);
  --pending_purge_obsolete_files_;
  assert(pending_purge_obsolete_files_ >= 0);
+  if (schedule_only) {
+    // Must change from pending_purge_obsolete_files_ to bg_purge_scheduled_
+    // while holding mutex (for GetSortedWalFiles() etc.)
+    SchedulePurge();
+  }
  if (pending_purge_obsolete_files_ == 0) {
    bg_cv_.SignalAll();
  }
@@ -657,11 +664,6 @@ void DBImpl::DeleteObsoleteFiles() {
  if (job_context.HaveSomethingToDelete()) {
    bool defer_purge = immutable_db_options_.avoid_unnecessary_blocking_io;
    PurgeObsoleteFiles(job_context, defer_purge);
-    if (defer_purge) {
-      mutex_.Lock();
-      SchedulePurge();
-      mutex_.Unlock();
-    }
  }
  job_context.Clean();
  mutex_.Lock();
@@ -200,6 +200,13 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src,
                   "file size check will be skipped during open.");
  }

+  if (result.preserve_deletes) {
+    ROCKS_LOG_WARN(
+        result.info_log,
+        "preserve_deletes is deprecated, will be removed in a future release. "
+        "Please try using user-defined timestamp instead.");
+  }
+
  return result;
 }

@@ -1463,7 +1470,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
                  meta.fd.smallest_seqno, meta.fd.largest_seqno,
                  meta.marked_for_compaction, meta.oldest_blob_file_number,
                  meta.oldest_ancester_time, meta.file_creation_time,
-                  meta.file_checksum, meta.file_checksum_func_name);
+                  meta.file_checksum, meta.file_checksum_func_name,
+                  meta.min_timestamp, meta.max_timestamp);

    for (const auto& blob : blob_file_additions) {
      edit->AddBlobFile(blob);
@@ -588,5 +588,6 @@ TEST_F(DBIOFailureTest, CompactionSstSyncError) {
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
+  RegisterCustomObjects(argc, argv);
  return RUN_ALL_TESTS();
 }
@@ -192,10 +192,11 @@ bool DBIter::SetBlobValueIfNeeded(const Slice& user_key,
  read_options.read_tier = read_tier_;
  read_options.verify_checksums = verify_checksums_;

+  constexpr FilePrefetchBuffer* prefetch_buffer = nullptr;
  constexpr uint64_t* bytes_read = nullptr;

  const Status s = version_->GetBlob(read_options, user_key, blob_index,
-                                     &blob_value_, bytes_read);
+                                     prefetch_buffer, &blob_value_, bytes_read);

  if (!s.ok()) {
    status_ = s;
@@ -266,6 +266,50 @@ TEST_F(DBOptionsTest, SetMutableTableOptions) {
  ASSERT_EQ(c_bbto->block_restart_interval, 13);
 }

+TEST_F(DBOptionsTest, SetWithCustomMemTableFactory) {
+  class DummySkipListFactory : public SkipListFactory {
+   public:
+    static const char* kClassName() { return "DummySkipListFactory"; }
+    const char* Name() const override { return kClassName(); }
+    explicit DummySkipListFactory() : SkipListFactory(2) {}
+  };
+  {
+    // Verify the DummySkipList cannot be created
+    ConfigOptions config_options;
+    config_options.ignore_unsupported_options = false;
+    std::unique_ptr<MemTableRepFactory> factory;
+    ASSERT_NOK(MemTableRepFactory::CreateFromString(
+        config_options, DummySkipListFactory::kClassName(), &factory));
+  }
+  Options options;
+  options.create_if_missing = true;
+  // Try with fail_if_options_file_error=false/true to update the options
+  for (bool on_error : {false, true}) {
+    options.fail_if_options_file_error = on_error;
+    options.env = env_;
+    options.disable_auto_compactions = false;
+
+    options.memtable_factory.reset(new DummySkipListFactory());
+    Reopen(options);
+
+    ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+    ASSERT_OK(
+        dbfull()->SetOptions(cfh, {{"disable_auto_compactions", "true"}}));
+    ColumnFamilyDescriptor cfd;
+    ASSERT_OK(cfh->GetDescriptor(&cfd));
+    ASSERT_STREQ(cfd.options.memtable_factory->Name(),
+                 DummySkipListFactory::kClassName());
+    ColumnFamilyHandle* test = nullptr;
+    ASSERT_OK(dbfull()->CreateColumnFamily(options, "test", &test));
+    ASSERT_OK(test->GetDescriptor(&cfd));
+    ASSERT_STREQ(cfd.options.memtable_factory->Name(),
+                 DummySkipListFactory::kClassName());
+
+    ASSERT_OK(dbfull()->DropColumnFamily(test));
+    delete test;
+  }
+}
+
 TEST_F(DBOptionsTest, SetBytesPerSync) {
  const size_t kValueSize = 1024 * 1024;  // 1MB
  Options options;
@@ -851,12 +851,14 @@ TEST_F(DBSecondaryTest, SwitchManifest) {
  Options options;
  options.env = env_;
  options.level0_file_num_compaction_trigger = 4;
-  Reopen(options);
+  const std::string cf1_name("test_cf");
+  CreateAndReopenWithCF({cf1_name}, options);

  Options options1;
  options1.env = env_;
  options1.max_open_files = -1;
-  OpenSecondary(options1);
+  OpenSecondaryWithColumnFamilies({kDefaultColumnFamilyName, cf1_name},
+                                  options1);

  const int kNumFiles = options.level0_file_num_compaction_trigger - 1;
  // Keep it smaller than 10 so that key0, key1, ..., key9 are sorted as 0, 1,
@@ -890,11 +892,11 @@ TEST_F(DBSecondaryTest, SwitchManifest) {
  // restart primary, performs full compaction, close again, restart again so
  // that next time secondary tries to catch up with primary, the secondary
  // will skip the MANIFEST in middle.
-  Reopen(options);
+  ReopenWithColumnFamilies({kDefaultColumnFamilyName, cf1_name}, options);
  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
  ASSERT_OK(dbfull()->TEST_WaitForCompact());

-  Reopen(options);
+  ReopenWithColumnFamilies({kDefaultColumnFamilyName, cf1_name}, options);
  ASSERT_OK(dbfull()->SetOptions({{"disable_auto_compactions", "false"}}));

  ASSERT_OK(db_secondary_->TryCatchUpWithPrimary());
@@ -905,12 +907,14 @@ TEST_F(DBSecondaryTest, SwitchManifestTwice) {
  Options options;
  options.env = env_;
  options.disable_auto_compactions = true;
-  Reopen(options);
+  const std::string cf1_name("test_cf");
+  CreateAndReopenWithCF({cf1_name}, options);

  Options options1;
  options1.env = env_;
  options1.max_open_files = -1;
-  OpenSecondary(options1);
+  OpenSecondaryWithColumnFamilies({kDefaultColumnFamilyName, cf1_name},
+                                  options1);

  ASSERT_OK(Put("0", "value0"));
  ASSERT_OK(Flush());
@@ -921,9 +925,9 @@ TEST_F(DBSecondaryTest, SwitchManifestTwice) {
  ASSERT_OK(db_secondary_->Get(ropts, "0", &value));
  ASSERT_EQ("value0", value);

-  Reopen(options);
+  ReopenWithColumnFamilies({kDefaultColumnFamilyName, cf1_name}, options);
  ASSERT_OK(dbfull()->SetOptions({{"disable_auto_compactions", "false"}}));
-  Reopen(options);
+  ReopenWithColumnFamilies({kDefaultColumnFamilyName, cf1_name}, options);
  ASSERT_OK(Put("0", "value1"));
  ASSERT_OK(db_secondary_->TryCatchUpWithPrimary());

@@ -1681,14 +1681,6 @@ TEST_F(DBSSTTest, DBWithSFMForBlobFilesAtomicFlush) {

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -14,8 +14,10 @@
 #include "port/port.h"
 #include "port/stack_trace.h"
 #include "rocksdb/db.h"
+#include "rocksdb/types.h"
 #include "rocksdb/utilities/table_properties_collectors.h"
 #include "table/format.h"
+#include "table/meta_blocks.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
 #include "util/random.h"
@@ -63,21 +65,49 @@ class DBTablePropertiesTest : public DBTestBase,
 TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {
  Options options = CurrentOptions();
  options.level0_file_num_compaction_trigger = 8;
+  // Part of strategy to prevent pinning table files
+  options.max_open_files = 42;
  Reopen(options);
+
  // Create 4 tables
  for (int table = 0; table < 4; ++table) {
+    // Use old meta name for table properties for one file
+    if (table == 3) {
+      SyncPoint::GetInstance()->SetCallBack(
+          "BlockBasedTableBuilder::WritePropertiesBlock:Meta", [&](void* meta) {
+            *reinterpret_cast<const std::string**>(meta) =
+                &kPropertiesBlockOldName;
+          });
+      SyncPoint::GetInstance()->EnableProcessing();
+    }
+    // Build file
    for (int i = 0; i < 10 + table; ++i) {
      ASSERT_OK(db_->Put(WriteOptions(), ToString(table * 100 + i), "val"));
    }
    ASSERT_OK(db_->Flush(FlushOptions()));
  }
+  SyncPoint::GetInstance()->DisableProcessing();
+  std::string original_session_id;
+  ASSERT_OK(db_->GetDbSessionId(original_session_id));
+
+  // Part of strategy to prevent pinning table files
+  SyncPoint::GetInstance()->SetCallBack(
+      "VersionEditHandler::LoadTables:skip_load_table_files",
+      [&](void* skip_load) { *reinterpret_cast<bool*>(skip_load) = true; });
+  SyncPoint::GetInstance()->EnableProcessing();

  // 1. Read table properties directly from file
  Reopen(options);
+  // Clear out auto-opened files
+  dbfull()->TEST_table_cache()->EraseUnRefEntries();
+  ASSERT_EQ(dbfull()->TEST_table_cache()->GetUsage(), 0U);
  VerifyTableProperties(db_, 10 + 11 + 12 + 13);

  // 2. Put two tables to table cache and
  Reopen(options);
+  // Clear out auto-opened files
+  dbfull()->TEST_table_cache()->EraseUnRefEntries();
+  ASSERT_EQ(dbfull()->TEST_table_cache()->GetUsage(), 0U);
  // fetch key from 1st and 2nd table, which will internally place that table to
  // the table cache.
  for (int i = 0; i < 2; ++i) {
@@ -88,12 +118,82 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfAllTablesTest) {

  // 3. Put all tables to table cache
  Reopen(options);
-  // fetch key from 1st and 2nd table, which will internally place that table to
-  // the table cache.
+  // fetch key from all tables, which will place them in table cache.
  for (int i = 0; i < 4; ++i) {
    Get(ToString(i * 100 + 0));
  }
  VerifyTableProperties(db_, 10 + 11 + 12 + 13);
+
+  // 4. Try to read CORRUPT properties (a) directly from file, and (b)
+  // through reader on Get
+
+  // It's not practical to prevent table file read on Open, so we
+  // corrupt after open and after purging table cache.
+  for (bool direct : {true, false}) {
+    Reopen(options);
+    // Clear out auto-opened files
+    dbfull()->TEST_table_cache()->EraseUnRefEntries();
+    ASSERT_EQ(dbfull()->TEST_table_cache()->GetUsage(), 0U);
+
+    TablePropertiesCollection props;
+    ASSERT_OK(db_->GetPropertiesOfAllTables(&props));
+    std::string sst_file = props.begin()->first;
+
+    // Corrupt the file's TableProperties using session id
+    std::string contents;
+    ASSERT_OK(
+        ReadFileToString(env_->GetFileSystem().get(), sst_file, &contents));
+    size_t pos = contents.find(original_session_id);
+    ASSERT_NE(pos, std::string::npos);
+    ASSERT_OK(test::CorruptFile(env_, sst_file, static_cast<int>(pos), 1,
+                                /*verify checksum fails*/ false));
+
+    // Try to read CORRUPT properties
+    if (direct) {
+      ASSERT_TRUE(db_->GetPropertiesOfAllTables(&props).IsCorruption());
+    } else {
+      bool found_corruption = false;
+      for (int i = 0; i < 4; ++i) {
+        std::string result = Get(ToString(i * 100 + 0));
+        if (result.find_first_of("Corruption: block checksum mismatch") !=
+            std::string::npos) {
+          found_corruption = true;
+        }
+      }
+      ASSERT_TRUE(found_corruption);
+    }
+
+    // UN-corrupt file for next iteration
+    ASSERT_OK(test::CorruptFile(env_, sst_file, static_cast<int>(pos), 1,
+                                /*verify checksum fails*/ false));
+  }
+
+  SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBTablePropertiesTest, InvalidIgnored) {
+  // RocksDB versions 2.5 - 2.7 generate some properties that Block considers
+  // invalid in some way. This approximates that.
+
+  // Inject properties block data that Block considers invalid
+  SyncPoint::GetInstance()->SetCallBack(
+      "BlockBasedTableBuilder::WritePropertiesBlock:BlockData",
+      [&](void* block_data) {
+        *reinterpret_cast<Slice*>(block_data) = Slice("X");
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // Build file
+  for (int i = 0; i < 10; ++i) {
+    ASSERT_OK(db_->Put(WriteOptions(), ToString(i), "val"));
+  }
+  ASSERT_OK(db_->Flush(FlushOptions()));
+
+  SyncPoint::GetInstance()->DisableProcessing();
+
+  // Not crashing is good enough
+  TablePropertiesCollection props;
+  ASSERT_OK(db_->GetPropertiesOfAllTables(&props));
 }

 TEST_F(DBTablePropertiesTest, CreateOnDeletionCollectorFactory) {
@@ -6956,14 +6956,6 @@ TEST_F(DBTest, MemoryUsageWithMaxWriteBufferSizeToMaintain) {

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -3973,7 +3973,9 @@ TEST_F(DBTest2, RateLimitedCompactionReads) {
        ASSERT_OK(Put(Key(j), DummyString(kBytesPerKey)));
      }
      ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
-      ASSERT_EQ(i + 1, NumTableFilesAtLevel(0));
+      if (i + 1 < kNumL0Files) {
+        ASSERT_EQ(i + 1, NumTableFilesAtLevel(0));
+      }
    }
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(0, NumTableFilesAtLevel(0));
@@ -6501,6 +6503,7 @@ TEST_F(DBTest2, BottommostTemperature) {
  Options options = CurrentOptions();
  options.bottommost_temperature = Temperature::kWarm;
  options.level0_file_num_compaction_trigger = 2;
+  options.statistics = CreateDBStatistics();
  Reopen(options);

  auto size = GetSstSizeHelper(Temperature::kUnknown);
@@ -6532,6 +6535,9 @@ TEST_F(DBTest2, BottommostTemperature) {
  ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0);
  ASSERT_EQ(iostats->file_io_stats_by_temperature.warm_file_read_count, 0);
  ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0);
+  ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0);
+  ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0);

  ASSERT_EQ("bar", Get("foo"));

@@ -6541,6 +6547,12 @@ TEST_F(DBTest2, BottommostTemperature) {
  ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_bytes_read, 0);
  ASSERT_GT(iostats->file_io_stats_by_temperature.warm_file_bytes_read, 0);
  ASSERT_EQ(iostats->file_io_stats_by_temperature.cold_file_bytes_read, 0);
+  ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0);
+  ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_COUNT), 0);
+  ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_COUNT), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_COUNT), 0);

  // non-bottommost file still has unknown temperature
  ASSERT_OK(Put("foo", "bar"));
@@ -6553,6 +6565,12 @@ TEST_F(DBTest2, BottommostTemperature) {
  ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_bytes_read, 0);
  ASSERT_GT(iostats->file_io_stats_by_temperature.warm_file_bytes_read, 0);
  ASSERT_EQ(iostats->file_io_stats_by_temperature.cold_file_bytes_read, 0);
+  ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0);
+  ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_COUNT), 0);
+  ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_COUNT), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_COUNT), 0);

  db_->GetColumnFamilyMetaData(&metadata);
  ASSERT_EQ(2, metadata.file_count);
@@ -6593,7 +6611,7 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
  options.compaction_style = kCompactionStyleUniversal;
  options.level0_file_num_compaction_trigger = kTriggerNum;
  options.num_levels = kNumLevels;
-
+  options.statistics = CreateDBStatistics();
  DestroyAndReopen(options);

  auto size = GetSstSizeHelper(Temperature::kUnknown);
@@ -6624,6 +6642,12 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
  ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0);
  ASSERT_EQ(iostats->file_io_stats_by_temperature.warm_file_read_count, 0);
  ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0);
+  ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_COUNT), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(WARM_FILE_READ_COUNT), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_COUNT), 0);
  ASSERT_EQ("bar", Get("foo"));

  ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_read_count, 0);
@@ -6632,6 +6656,12 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
  ASSERT_EQ(iostats->file_io_stats_by_temperature.hot_file_bytes_read, 0);
  ASSERT_EQ(iostats->file_io_stats_by_temperature.warm_file_bytes_read, 0);
  ASSERT_EQ(iostats->file_io_stats_by_temperature.cold_file_bytes_read, 0);
+  ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_COUNT), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(WARM_FILE_READ_COUNT), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_COUNT), 0);

  ASSERT_OK(Put("foo", "bar"));
  ASSERT_OK(Put("bar", "bar"));
@@ -6667,6 +6697,12 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
  ASSERT_EQ(size, 0);
  size = GetSstSizeHelper(Temperature::kWarm);
  ASSERT_GT(size, 0);
+  ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_BYTES), 0);
+  ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_BYTES), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(HOT_FILE_READ_COUNT), 0);
+  ASSERT_GT(options.statistics->getTickerCount(WARM_FILE_READ_COUNT), 0);
+  ASSERT_EQ(options.statistics->getTickerCount(COLD_FILE_READ_COUNT), 0);

  // non-bottommost file still has unknown temperature
  ASSERT_OK(Put("foo", "bar"));
@@ -6763,15 +6799,70 @@ TEST_F(DBTest2, RenameDirectory) {
  Destroy(options);
  dbname_ = old_dbname;
 }
-}  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
+#ifndef ROCKSDB_LITE
+TEST_F(DBTest2, GetLatestSeqAndTsForKey) {
+  Destroy(last_options_);
+
+  Options options = CurrentOptions();
+  options.max_write_buffer_size_to_maintain = 64 << 10;
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.comparator = test::ComparatorWithU64Ts();
+  options.statistics = CreateDBStatistics();
+
+  Reopen(options);
+
+  constexpr uint64_t kTsU64Value = 12;
+
+  for (uint64_t key = 0; key < 100; ++key) {
+    std::string ts_str;
+    PutFixed64(&ts_str, kTsU64Value);
+    Slice ts = ts_str;
+    WriteOptions write_opts;
+    write_opts.timestamp = &ts;
+
+    std::string key_str;
+    PutFixed64(&key_str, key);
+    std::reverse(key_str.begin(), key_str.end());
+    ASSERT_OK(Put(key_str, "value", write_opts));
+  }
+
+  ASSERT_OK(Flush());
+
+  constexpr bool cache_only = true;
+  constexpr SequenceNumber lower_bound_seq = 0;
+  auto* cfhi = static_cast_with_check<ColumnFamilyHandleImpl>(
+      dbfull()->DefaultColumnFamily());
+  assert(cfhi);
+  assert(cfhi->cfd());
+  SuperVersion* sv = cfhi->cfd()->GetSuperVersion();
+  for (uint64_t key = 0; key < 100; ++key) {
+    std::string key_str;
+    PutFixed64(&key_str, key);
+    std::reverse(key_str.begin(), key_str.end());
+    std::string ts;
+    SequenceNumber seq = kMaxSequenceNumber;
+    bool found_record_for_key = false;
+    bool is_blob_index = false;
+
+    const Status s = dbfull()->GetLatestSequenceForKey(
+        sv, key_str, cache_only, lower_bound_seq, &seq, &ts,
+        &found_record_for_key, &is_blob_index);
+    ASSERT_OK(s);
+    std::string expected_ts;
+    PutFixed64(&expected_ts, kTsU64Value);
+    ASSERT_EQ(expected_ts, ts);
+    ASSERT_TRUE(found_record_for_key);
+    ASSERT_FALSE(is_blob_index);
+  }
+
+  // Verify that no read to SST files.
+  ASSERT_EQ(0, options.statistics->getTickerCount(GET_HIT_L0));
 }
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
+#endif  // ROCKSDB_LITE
+
+}  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
@@ -1680,6 +1680,7 @@ TEST_P(DBTestUniversalCompaction, ConcurrentBottomPriLowPriCompactions) {
  Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM);
  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
+  options.max_background_compactions = 2;
  options.num_levels = num_levels_;
  options.write_buffer_size = 100 << 10;     // 100KB
  options.target_file_size_base = 32 << 10;  // 32KB
@@ -1688,6 +1689,10 @@ TEST_P(DBTestUniversalCompaction, ConcurrentBottomPriLowPriCompactions) {
  options.compaction_options_universal.max_size_amplification_percent = 110;
  DestroyAndReopen(options);

+  // Need to get a token to enable compaction parallelism up to
+  // `max_background_compactions` jobs.
+  auto pressure_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
      {// wait for the full compaction to be picked before adding files intended
       // for the second one.
@@ -287,6 +287,51 @@ TEST_F(DBBasicTestWithTimestamp, CompactRangeWithSpecifiedRange) {
  Close();
 }

+TEST_F(DBBasicTestWithTimestamp, GcPreserveLatestVersionBelowFullHistoryLow) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.create_if_missing = true;
+  const size_t kTimestampSize = Timestamp(0, 0).size();
+  TestComparator test_cmp(kTimestampSize);
+  options.comparator = &test_cmp;
+  DestroyAndReopen(options);
+
+  std::string ts_str = Timestamp(1, 0);
+  WriteOptions wopts;
+  Slice ts = ts_str;
+  wopts.timestamp = &ts;
+  ASSERT_OK(db_->Put(wopts, "k1", "v1"));
+  ASSERT_OK(db_->Put(wopts, "k2", "v2"));
+  ASSERT_OK(db_->Put(wopts, "k3", "v3"));
+
+  ts_str = Timestamp(2, 0);
+  ts = ts_str;
+  wopts.timestamp = &ts;
+  ASSERT_OK(db_->Delete(wopts, "k3"));
+
+  ts_str = Timestamp(4, 0);
+  ts = ts_str;
+  wopts.timestamp = &ts;
+  ASSERT_OK(db_->Put(wopts, "k1", "v5"));
+
+  ts_str = Timestamp(3, 0);
+  ts = ts_str;
+  CompactRangeOptions cro;
+  cro.full_history_ts_low = &ts;
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+
+  ASSERT_OK(Flush());
+
+  ReadOptions ropts;
+  ropts.timestamp = &ts;
+  std::string value;
+  Status s = db_->Get(ropts, "k1", &value);
+  ASSERT_OK(s);
+  ASSERT_EQ("v1", value);
+
+  Close();
+}
+
 TEST_F(DBBasicTestWithTimestamp, UpdateFullHistoryTsLow) {
  Options options = CurrentOptions();
  options.env = env_;
@@ -324,6 +369,7 @@ TEST_F(DBBasicTestWithTimestamp, UpdateFullHistoryTsLow) {
  }
  ASSERT_OK(Flush());

+  // TODO return a non-ok for read ts < current_ts_low and test it.
  for (int i = 0; i < 10; i++) {
    ReadOptions read_opts;
    std::string ts_str = Timestamp(i, 0);
@@ -331,7 +377,7 @@ TEST_F(DBBasicTestWithTimestamp, UpdateFullHistoryTsLow) {
    read_opts.timestamp = &ts;
    std::string value;
    Status status = db_->Get(read_opts, kKey, &value);
-    if (i < current_ts_low) {
+    if (i < current_ts_low - 1) {
      ASSERT_TRUE(status.IsNotFound());
    } else {
      ASSERT_OK(status);
@@ -358,19 +404,16 @@ TEST_F(DBBasicTestWithTimestamp, UpdateFullHistoryTsLow) {
  result_ts_low = cfd->GetFullHistoryTsLow();
  ASSERT_TRUE(test_cmp.CompareTimestamp(ts_low, result_ts_low) == 0);

-  for (int i = 0; i < 20; i++) {
+  // TODO return a non-ok for read ts < current_ts_low and test it.
+  for (int i = current_ts_low; i < 20; i++) {
    ReadOptions read_opts;
    std::string ts_str = Timestamp(i, 0);
    Slice ts = ts_str;
    read_opts.timestamp = &ts;
    std::string value;
    Status status = db_->Get(read_opts, kKey, &value);
-    if (i < current_ts_low) {
-      ASSERT_TRUE(status.IsNotFound());
-    } else {
-      ASSERT_OK(status);
-      ASSERT_TRUE(value.compare(Key(i)) == 0);
-    }
+    ASSERT_OK(status);
+    ASSERT_TRUE(value.compare(Key(i)) == 0);
  }

  // Test invalid compaction with range
@@ -552,6 +595,41 @@ TEST_F(DBBasicTestWithTimestamp, SimpleIterate) {
  Close();
 }

+#ifndef ROCKSDB_LITE
+TEST_F(DBBasicTestWithTimestamp, GetTimestampTableProperties) {
+  Options options = CurrentOptions();
+  const size_t kTimestampSize = Timestamp(0, 0).size();
+  TestComparator test_cmp(kTimestampSize);
+  options.comparator = &test_cmp;
+  DestroyAndReopen(options);
+  // Create 2 tables
+  for (int table = 0; table < 2; ++table) {
+    for (int i = 0; i < 10; i++) {
+      WriteOptions write_opts;
+      std::string ts_str = Timestamp(i, 0);
+      Slice ts = ts_str;
+      write_opts.timestamp = &ts;
+      ASSERT_OK(db_->Put(write_opts, "key", Key(i)));
+    }
+    ASSERT_OK(Flush());
+  }
+
+  TablePropertiesCollection props;
+  ASSERT_OK(db_->GetPropertiesOfAllTables(&props));
+  ASSERT_EQ(2U, props.size());
+  for (const auto& item : props) {
+    auto& user_collected = item.second->user_collected_properties;
+    ASSERT_TRUE(user_collected.find("rocksdb.timestamp_min") !=
+                user_collected.end());
+    ASSERT_TRUE(user_collected.find("rocksdb.timestamp_max") !=
+                user_collected.end());
+    ASSERT_EQ(user_collected.at("rocksdb.timestamp_min"), Timestamp(0, 0));
+    ASSERT_EQ(user_collected.at("rocksdb.timestamp_max"), Timestamp(9, 0));
+  }
+  Close();
+}
+#endif  // !ROCKSDB_LITE
+
 class DBBasicTestWithTimestampTableOptions
    : public DBBasicTestWithTimestampBase,
      public testing::WithParamInterface<BlockBasedTableOptions::IndexType> {
@@ -2632,6 +2710,8 @@ TEST_P(DBBasicTestWithTimestampCompressionSettings, PutAndGetWithCompaction) {
          // incremental positions such that lowerlevel[1].smallest.userkey ==
          // higherlevel[0].largest.userkey
          ASSERT_OK(Flush(cf));
+          ASSERT_OK(dbfull()->TEST_WaitForCompact());  // wait for flush (which
+                                                       // is also a compaction)

          // compact files (2 at each level) to a lower level such that all
          // keys with the same timestamp is at one level, with newer versions
@@ -3082,14 +3162,6 @@ INSTANTIATE_TEST_CASE_P(

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -785,14 +785,6 @@ INSTANTIATE_TEST_CASE_P(DBWriteBufferManagerTest, DBWriteBufferManagerTest,

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -461,5 +461,6 @@ INSTANTIATE_TEST_CASE_P(DBWriteTestInstance, DBWriteTest,
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
+  RegisterCustomObjects(argc, argv);
  return RUN_ALL_TESTS();
 }
@@ -28,6 +28,7 @@ namespace ROCKSDB_NAMESPACE {
 // ValueType, not the lowest).
 const ValueType kValueTypeForSeek = kTypeDeletionWithTimestamp;
 const ValueType kValueTypeForSeekForPrev = kTypeDeletion;
+const std::string kDisableUserTimestamp("");

 EntryType GetEntryType(ValueType value_type) {
  switch (value_type) {
@@ -93,6 +93,9 @@ static const SequenceNumber kDisableGlobalSequenceNumber = port::kMaxUint64;

 constexpr uint64_t kNumInternalBytes = 8;

+// Defined in dbformat.cc
+extern const std::string kDisableUserTimestamp;
+
 // The data structure that represents an internal key in the way that user_key,
 // sequence number and type are stored in separated forms.
 struct ParsedInternalKey {
@@ -8,7 +8,9 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

 #include "db/dbformat.h"
+
 #include "test_util/testharness.h"
+#include "test_util/testutil.h"

 namespace ROCKSDB_NAMESPACE {

@@ -202,5 +204,6 @@ TEST_F(FormatTest, RangeTombstoneSerializeEndKey) {

 int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
+  RegisterCustomObjects(argc, argv);
  return RUN_ALL_TESTS();
 }
@@ -596,14 +596,6 @@ TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) {

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -406,10 +406,7 @@ const Status& ErrorHandler::SetBGError(const IOStatus& bg_io_err,
    // it can directly overwrite any existing bg_error_.
    bool auto_recovery = false;
    Status bg_err(new_bg_io_err, Status::Severity::kUnrecoverableError);
-    bg_error_ = bg_err;
-    if (recovery_in_prog_ && recovery_error_.ok()) {
-      recovery_error_ = bg_err;
-    }
+    CheckAndSetRecoveryAndBGError(bg_err);
    if (bg_error_stats_ != nullptr) {
      RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_ERROR_COUNT);
      RecordTick(bg_error_stats_.get(), ERROR_HANDLER_BG_IO_ERROR_COUNT);
@@ -468,24 +465,14 @@ const Status& ErrorHandler::SetBGError(const IOStatus& bg_io_err,
      // to many small memtable being generated during auto resume, the flush
      // reason is set to kErrorRecoveryRetryFlush.
      Status bg_err(new_bg_io_err, Status::Severity::kSoftError);
-      if (recovery_in_prog_ && recovery_error_.ok()) {
-        recovery_error_ = bg_err;
-      }
-      if (bg_err.severity() > bg_error_.severity()) {
-        bg_error_ = bg_err;
-      }
+      CheckAndSetRecoveryAndBGError(bg_err);
      soft_error_no_bg_work_ = true;
      context.flush_reason = FlushReason::kErrorRecoveryRetryFlush;
      recover_context_ = context;
      return StartRecoverFromRetryableBGIOError(bg_io_err);
    } else {
      Status bg_err(new_bg_io_err, Status::Severity::kHardError);
-      if (recovery_in_prog_ && recovery_error_.ok()) {
-        recovery_error_ = bg_err;
-      }
-      if (bg_err.severity() > bg_error_.severity()) {
-        bg_error_ = bg_err;
-      }
+      CheckAndSetRecoveryAndBGError(bg_err);
      recover_context_ = context;
      return StartRecoverFromRetryableBGIOError(bg_io_err);
    }
@@ -663,7 +650,6 @@ const Status& ErrorHandler::StartRecoverFromRetryableBGIOError(
  if (recovery_io_error_.ok() && recovery_error_.ok()) {
    return recovery_error_;
  } else {
-    TEST_SYNC_POINT("StartRecoverRetryableBGIOError:RecoverFail");
    return bg_error_;
  }
 #else
@@ -677,7 +663,6 @@ const Status& ErrorHandler::StartRecoverFromRetryableBGIOError(
 void ErrorHandler::RecoverFromRetryableBGIOError() {
 #ifndef ROCKSDB_LITE
  TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeStart");
-  TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeStart1");
  InstrumentedMutexLock l(db_mutex_);
  if (end_recovery_) {
    return;
@@ -697,8 +682,6 @@ void ErrorHandler::RecoverFromRetryableBGIOError() {
    recovery_error_ = Status::OK();
    retry_count++;
    Status s = db_->ResumeImpl(context);
-    TEST_SYNC_POINT("RecoverFromRetryableBGIOError:AfterResume0");
-    TEST_SYNC_POINT("RecoverFromRetryableBGIOError:AfterResume1");
    if (bg_error_stats_ != nullptr) {
      RecordTick(bg_error_stats_.get(),
                 ERROR_HANDLER_AUTORESUME_RETRY_TOTAL_COUNT);
@@ -707,7 +690,6 @@ void ErrorHandler::RecoverFromRetryableBGIOError() {
        bg_error_.severity() >= Status::Severity::kFatalError) {
      // If DB shutdown in progress or the error severity is higher than
      // Hard Error, stop auto resume and returns.
-      TEST_SYNC_POINT("RecoverFromRetryableBGIOError:RecoverFail0");
      recovery_in_prog_ = false;
      if (bg_error_stats_ != nullptr) {
        RecordInHistogram(bg_error_stats_.get(),
@@ -725,7 +707,6 @@ void ErrorHandler::RecoverFromRetryableBGIOError() {
      TEST_SYNC_POINT("RecoverFromRetryableBGIOError:BeforeWait1");
      int64_t wait_until = db_options_.clock->NowMicros() + wait_interval;
      cv_.TimedWait(wait_until);
-      TEST_SYNC_POINT("RecoverFromRetryableBGIOError:AfterWait0");
    } else {
      // There are three possibility: 1) recover_io_error is set during resume
      // and the error is not retryable, 2) recover is successful, 3) other
@@ -751,7 +732,6 @@ void ErrorHandler::RecoverFromRetryableBGIOError() {
        }
        return;
      } else {
-        TEST_SYNC_POINT("RecoverFromRetryableBGIOError:RecoverFail1");
        // In this case: 1) recovery_io_error is more serious or not retryable
        // 2) other Non IO recovery_error happens. The auto recovery stops.
        recovery_in_prog_ = false;
@@ -776,6 +756,16 @@ void ErrorHandler::RecoverFromRetryableBGIOError() {
 #endif
 }

+void ErrorHandler::CheckAndSetRecoveryAndBGError(const Status& bg_err) {
+  if (recovery_in_prog_ && recovery_error_.ok()) {
+    recovery_error_ = bg_err;
+  }
+  if (bg_err.severity() > bg_error_.severity()) {
+    bg_error_ = bg_err;
+  }
+  return;
+}
+
 void ErrorHandler::EndAutoRecovery() {
  db_mutex_->AssertHeld();
  if (!end_recovery_) {
@@ -116,6 +116,10 @@ class ErrorHandler {
    void RecoverFromNoSpace();
    const Status& StartRecoverFromRetryableBGIOError(const IOStatus& io_error);
    void RecoverFromRetryableBGIOError();
+    // First, if it is in recovery and the recovery_error is ok. Set the
+    // recovery_error_ to bg_err. Second, if the severity is higher than the
+    // current bg_error_, overwrite it.
+    void CheckAndSetRecoveryAndBGError(const Status& bg_err);
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -487,6 +487,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWALAtomicWriteRetryableError) {
  Destroy(options);
 }

+// The flush error is injected before we finish the table build
 TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) {
  std::shared_ptr<ErrorHandlerFSListener> listener(
      new ErrorHandlerFSListener());
@@ -542,6 +543,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableError1) {
  Destroy(options);
 }

+// The retryable IO error is injected before we sync table
 TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError2) {
  std::shared_ptr<ErrorHandlerFSListener> listener(
      new ErrorHandlerFSListener());
@@ -585,6 +587,7 @@ TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError2) {
  Destroy(options);
 }

+// The retryable IO error is injected before we close the table file
 TEST_F(DBErrorHandlingFSTest, FLushWriteNoWALRetryableError3) {
  std::shared_ptr<ErrorHandlerFSListener> listener(
      new ErrorHandlerFSListener());
@@ -1874,6 +1877,8 @@ TEST_F(DBErrorHandlingFSTest, FLushWritNoWALRetryableErrorAutoRecover2) {
  Destroy(options);
 }

+// Auto resume fromt the flush retryable IO error. Activate the FS before the
+// first resume. Resume is successful
 TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover1) {
  // Activate the FS before the first resume
  std::shared_ptr<ErrorHandlerFSListener> listener(
@@ -1914,6 +1919,8 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover1) {
  Destroy(options);
 }

+// Auto resume fromt the flush retryable IO error and set the retry limit count.
+// Never activate the FS and auto resume should fail at the end
 TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) {
  // Fail all the resume and let user to resume
  std::shared_ptr<ErrorHandlerFSListener> listener(
@@ -1963,6 +1970,8 @@ TEST_F(DBErrorHandlingFSTest, FLushWritRetryableErrorAutoRecover2) {
  Destroy(options);
 }

+// Auto resume fromt the flush retryable IO error and set the retry limit count.
+// Fail the first resume and let the second resume be successful.
 TEST_F(DBErrorHandlingFSTest, ManifestWriteRetryableErrorAutoRecover) {
  // Fail the first resume and let the second resume be successful
  std::shared_ptr<ErrorHandlerFSListener> listener(
@@ -1806,14 +1806,6 @@ INSTANTIATE_TEST_CASE_P(ExternalSSTFileBasicTest, ExternalSSTFileBasicTest,

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -441,7 +441,8 @@ Status ExternalSstFileIngestionJob::Run() {
        f.fd.GetNumber(), f.fd.GetPathId(), f.fd.GetFileSize(),
        f.smallest_internal_key, f.largest_internal_key, f.assigned_seqno,
        f.assigned_seqno, false, kInvalidBlobFileNumber, oldest_ancester_time,
-        current_time, f.file_checksum, f.file_checksum_func_name);
+        current_time, f.file_checksum, f.file_checksum_func_name,
+        kDisableUserTimestamp, kDisableUserTimestamp);
    f_metadata.temperature = f.file_temperature;
    edit_.AddFile(f.picked_level, f_metadata);
  }
@@ -17,6 +17,7 @@
 #include "env/mock_env.h"
 #include "file/filename.h"
 #include "rocksdb/cache.h"
+#include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/table.h"
@@ -81,7 +82,11 @@ class FaultInjectionTest
        sync_use_compact_(true),
        base_env_(nullptr),
        env_(nullptr),
-        db_(nullptr) {}
+        db_(nullptr) {
+    EXPECT_OK(
+        test::CreateEnvFromSystem(ConfigOptions(), &system_env_, &env_guard_));
+    EXPECT_NE(system_env_, nullptr);
+  }

  ~FaultInjectionTest() override {
    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
@@ -94,7 +99,7 @@ class FaultInjectionTest
      return false;
    } else {
      if (option_config_ == kMultiLevels) {
-        base_env_.reset(MockEnv::Create(Env::Default()));
+        base_env_.reset(MockEnv::Create(system_env_));
      }
      return true;
    }
@@ -146,8 +151,7 @@ class FaultInjectionTest
    assert(tiny_cache_ == nullptr);
    assert(env_ == nullptr);

-    env_ =
-        new FaultInjectionTestEnv(base_env_ ? base_env_.get() : Env::Default());
+    env_ = new FaultInjectionTestEnv(base_env_ ? base_env_.get() : system_env_);

    options_ = CurrentOptions();
    options_.env = env_;
@@ -345,6 +349,10 @@ class FaultInjectionTest
    ASSERT_OK(static_cast<DBImpl*>(db_->GetRootDB())->TEST_WaitForCompact());
    ASSERT_OK(db_->Put(WriteOptions(), "", ""));
  }
+
+ private:
+  Env* system_env_;
+  std::shared_ptr<Env> env_guard_;
 };

 class FaultInjectionTestSplitted : public FaultInjectionTest {};
@@ -552,5 +560,6 @@ INSTANTIATE_TEST_CASE_P(

 int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
+  RegisterCustomObjects(argc, argv);
  return RUN_ALL_TESTS();
 }
@@ -962,7 +962,8 @@ Status FlushJob::WriteLevel0Table() {
                   meta_.fd.smallest_seqno, meta_.fd.largest_seqno,
                   meta_.marked_for_compaction, meta_.oldest_blob_file_number,
                   meta_.oldest_ancester_time, meta_.file_creation_time,
-                   meta_.file_checksum, meta_.file_checksum_func_name);
+                   meta_.file_checksum, meta_.file_checksum_func_name,
+                   meta_.min_timestamp, meta_.max_timestamp);

    edit_->SetBlobFileAdditions(std::move(blob_file_additions));
  }
@@ -155,7 +155,8 @@ Status ImportColumnFamilyJob::Run() {
                  f.largest_internal_key, file_metadata.smallest_seqno,
                  file_metadata.largest_seqno, false, kInvalidBlobFileNumber,
                  oldest_ancester_time, current_time, kUnknownFileChecksum,
-                  kUnknownFileChecksumFuncName);
+                  kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                  kDisableUserTimestamp);

    // If incoming sequence number is higher, update local sequence number.
    if (file_metadata.largest_seqno > versions_->LastSequence()) {
@@ -56,7 +56,7 @@ using ProtectionInfoKVOS64 = ProtectionInfoKVOS<uint64_t>;
 template <typename T>
 class ProtectionInfo {
 public:
-  ProtectionInfo<T>() = default;
+  ProtectionInfo() = default;

  Status GetStatus() const;
  ProtectionInfoKVO<T> ProtectKVO(const Slice& key, const Slice& value,
@@ -83,7 +83,7 @@ class ProtectionInfo {
  static const uint64_t kSeedS = 0x77A00858DDD37F21;
  static const uint64_t kSeedC = 0x4A2AB5CBD26F542C;

-  ProtectionInfo<T>(T val) : val_(val) {
+  ProtectionInfo(T val) : val_(val) {
    static_assert(sizeof(ProtectionInfo<T>) == sizeof(T), "");
  }

@@ -96,7 +96,7 @@ class ProtectionInfo {
 template <typename T>
 class ProtectionInfoKVO {
 public:
-  ProtectionInfoKVO<T>() = default;
+  ProtectionInfoKVO() = default;

  ProtectionInfo<T> StripKVO(const Slice& key, const Slice& value,
                             ValueType op_type) const;
@@ -117,7 +117,7 @@ class ProtectionInfoKVO {
  friend class ProtectionInfoKVOS<T>;
  friend class ProtectionInfoKVOC<T>;

-  explicit ProtectionInfoKVO<T>(T val) : info_(val) {
+  explicit ProtectionInfoKVO(T val) : info_(val) {
    static_assert(sizeof(ProtectionInfoKVO<T>) == sizeof(T), "");
  }

@@ -130,7 +130,7 @@ class ProtectionInfoKVO {
 template <typename T>
 class ProtectionInfoKVOC {
 public:
-  ProtectionInfoKVOC<T>() = default;
+  ProtectionInfoKVOC() = default;

  ProtectionInfoKVO<T> StripC(ColumnFamilyId column_family_id) const;

@@ -155,7 +155,7 @@ class ProtectionInfoKVOC {
 private:
  friend class ProtectionInfoKVO<T>;

-  explicit ProtectionInfoKVOC<T>(T val) : kvo_(val) {
+  explicit ProtectionInfoKVOC(T val) : kvo_(val) {
    static_assert(sizeof(ProtectionInfoKVOC<T>) == sizeof(T), "");
  }

@@ -168,7 +168,7 @@ class ProtectionInfoKVOC {
 template <typename T>
 class ProtectionInfoKVOS {
 public:
-  ProtectionInfoKVOS<T>() = default;
+  ProtectionInfoKVOS() = default;

  ProtectionInfoKVO<T> StripS(SequenceNumber sequence_number) const;

@@ -193,7 +193,7 @@ class ProtectionInfoKVOS {
 private:
  friend class ProtectionInfoKVO<T>;

-  explicit ProtectionInfoKVOS<T>(T val) : kvo_(val) {
+  explicit ProtectionInfoKVOS(T val) : kvo_(val) {
    static_assert(sizeof(ProtectionInfoKVOS<T>) == sizeof(T), "");
  }

@@ -8,6 +8,9 @@
 #include <string>

 #include "db/blob/blob_fetcher.h"
+#include "db/blob/blob_index.h"
+#include "db/blob/prefetch_buffer_collection.h"
+#include "db/compaction/compaction_iteration_stats.h"
 #include "db/dbformat.h"
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/statistics.h"
@@ -121,7 +124,9 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
                               const SequenceNumber stop_before,
                               const bool at_bottom,
                               const bool allow_data_in_errors,
-                               Version* version) {
+                               const BlobFetcher* blob_fetcher,
+                               PrefetchBufferCollection* prefetch_buffers,
+                               CompactionIterationStats* c_iter_stats) {
  // Get a copy of the internal key, before it's invalidated by iter->Next()
  // Also maintain the list of merge operands seen.
  assert(HasOperator());
@@ -212,13 +217,35 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
           !range_del_agg->ShouldDelete(
               ikey, RangeDelPositioningMode::kForwardTraversal))) {
        if (ikey.type == kTypeBlobIndex) {
-          assert(version);
-          BlobFetcher blob_fetcher(version, ReadOptions());
-          s = blob_fetcher.FetchBlob(ikey.user_key, val, &blob_value);
+          BlobIndex blob_index;
+
+          s = blob_index.DecodeFrom(val);
          if (!s.ok()) {
            return s;
          }
+
+          FilePrefetchBuffer* prefetch_buffer =
+              prefetch_buffers ? prefetch_buffers->GetOrCreatePrefetchBuffer(
+                                     blob_index.file_number())
+                               : nullptr;
+
+          uint64_t bytes_read = 0;
+
+          assert(blob_fetcher);
+
+          s = blob_fetcher->FetchBlob(ikey.user_key, blob_index,
+                                      prefetch_buffer, &blob_value,
+                                      &bytes_read);
+          if (!s.ok()) {
+            return s;
+          }
+
          val_ptr = &blob_value;
+
+          if (c_iter_stats) {
+            ++c_iter_stats->num_blobs_read;
+            c_iter_stats->total_blob_bytes_read += bytes_read;
+          }
        } else {
          val_ptr = &val;
        }
@@ -26,7 +26,9 @@ class Logger;
 class MergeOperator;
 class Statistics;
 class SystemClock;
-class Version;
+class BlobFetcher;
+class PrefetchBufferCollection;
+struct CompactionIterationStats;

 class MergeHelper {
 public:
@@ -70,6 +72,10 @@ class MergeHelper {
  //                   we could reach the start of the history of this user key.
  // allow_data_in_errors: (IN) if true, data details will be displayed in
  //                   error/log messages.
+  // blob_fetcher: (IN) blob fetcher object for the compaction's input version.
+  // prefetch_buffers: (IN/OUT) a collection of blob file prefetch buffers
+  //                            used for compaction readahead.
+  // c_iter_stats: (OUT) compaction iteration statistics.
  //
  // Returns one of the following statuses:
  // - OK: Entries were successfully merged.
@@ -82,11 +88,12 @@ class MergeHelper {
  //
  // REQUIRED: The first key in the input is not corrupted.
  Status MergeUntil(InternalIterator* iter,
-                    CompactionRangeDelAggregator* range_del_agg = nullptr,
-                    const SequenceNumber stop_before = 0,
-                    const bool at_bottom = false,
-                    const bool allow_data_in_errors = false,
-                    Version* version = nullptr);
+                    CompactionRangeDelAggregator* range_del_agg,
+                    const SequenceNumber stop_before, const bool at_bottom,
+                    const bool allow_data_in_errors,
+                    const BlobFetcher* blob_fetcher,
+                    PrefetchBufferCollection* prefetch_buffers,
+                    CompactionIterationStats* c_iter_stats);

  // Filters a merge operand using the compaction filter specified
  // in the constructor. Returns the decision that the filter made.
@@ -32,8 +32,10 @@ class MergeHelperTest : public testing::Test {
    merge_helper_.reset(new MergeHelper(env_, icmp_.user_comparator(),
                                        merge_op_.get(), filter_.get(), nullptr,
                                        false, latest_snapshot));
-    return merge_helper_->MergeUntil(iter_.get(), nullptr /* range_del_agg */,
-                                     stop_before, at_bottom);
+    return merge_helper_->MergeUntil(
+        iter_.get(), nullptr /* range_del_agg */, stop_before, at_bottom,
+        false /* allow_data_in_errors */, nullptr /* blob_fetcher */,
+        nullptr /* prefetch_buffers */, nullptr /* c_iter_stats */);
  }

  void AddKeyVal(const std::string& user_key, const SequenceNumber& seq,
@@ -308,14 +308,6 @@ TEST_F(ObsoleteFilesTest, BlobFiles) {

 }  // namespace ROCKSDB_NAMESPACE

-#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-extern "C" {
-void RegisterCustomObjects(int argc, char** argv);
-}
-#else
-void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
-#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
-
 int main(int argc, char** argv) {
  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
  ::testing::InitGoogleTest(&argc, argv);
@@ -266,24 +266,22 @@ class TestPlainTableReader : public PlainTableReader {
      const EnvOptions& env_options, const InternalKeyComparator& icomparator,
      EncodingType encoding_type, uint64_t file_size, int bloom_bits_per_key,
      double hash_table_ratio, size_t index_sparseness,
-      const TableProperties* table_properties,
+      std::unique_ptr<TableProperties>&& props,
      std::unique_ptr<RandomAccessFileReader>&& file,
      const ImmutableOptions& ioptions, const SliceTransform* prefix_extractor,
      bool* expect_bloom_not_match, bool store_index_in_file,
      uint32_t column_family_id, const std::string& column_family_name)
      : PlainTableReader(ioptions, std::move(file), env_options, icomparator,
-                         encoding_type, file_size, table_properties,
+                         encoding_type, file_size, props.get(),
                         prefix_extractor),
        expect_bloom_not_match_(expect_bloom_not_match) {
    Status s = MmapDataIfNeeded();
    EXPECT_TRUE(s.ok());

-    s = PopulateIndex(const_cast<TableProperties*>(table_properties),
-                      bloom_bits_per_key, hash_table_ratio, index_sparseness,
-                      2 * 1024 * 1024);
+    s = PopulateIndex(props.get(), bloom_bits_per_key, hash_table_ratio,
+                      index_sparseness, 2 * 1024 * 1024);
    EXPECT_TRUE(s.ok());

-    TableProperties* props = const_cast<TableProperties*>(table_properties);
    EXPECT_EQ(column_family_id, static_cast<uint32_t>(props->column_family_id));
    EXPECT_EQ(column_family_name, props->column_family_name);
    if (store_index_in_file) {
@@ -297,7 +295,7 @@ class TestPlainTableReader : public PlainTableReader {
        EXPECT_TRUE(num_blocks_ptr != props->user_collected_properties.end());
      }
    }
-    table_properties_.reset(props);
+    table_properties_ = std::move(props);
  }

  ~TestPlainTableReader() override {}
@@ -337,26 +335,24 @@ class TestPlainTableFactory : public PlainTableFactory {
      std::unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
      std::unique_ptr<TableReader>* table,
      bool /*prefetch_index_and_filter_in_cache*/) const override {
-    TableProperties* props = nullptr;
-    auto s =
-        ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber,
-                            table_reader_options.ioptions, &props,
-                            true /* compression_type_missing */);
+    std::unique_ptr<TableProperties> props;
+    auto s = ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber,
+                                 table_reader_options.ioptions, &props);
    EXPECT_TRUE(s.ok());

    if (store_index_in_file_) {
      BlockHandle bloom_block_handle;
-      s = FindMetaBlock(file.get(), file_size, kPlainTableMagicNumber,
-                        table_reader_options.ioptions,
-                        BloomBlockBuilder::kBloomBlock, &bloom_block_handle,
-                        /* compression_type_missing */ true);
+      s = FindMetaBlockInFile(file.get(), file_size, kPlainTableMagicNumber,
+                              table_reader_options.ioptions,
+                              BloomBlockBuilder::kBloomBlock,
+                              &bloom_block_handle);
      EXPECT_TRUE(s.ok());

      BlockHandle index_block_handle;
-      s = FindMetaBlock(file.get(), file_size, kPlainTableMagicNumber,
-                        table_reader_options.ioptions,
-                        PlainTableIndexBuilder::kPlainTableIndexBlock,
-                        &index_block_handle, /* compression_type_missing */ true);
+      s = FindMetaBlockInFile(file.get(), file_size, kPlainTableMagicNumber,
+                              table_reader_options.ioptions,
+                              PlainTableIndexBuilder::kPlainTableIndexBlock,
+                              &index_block_handle);
      EXPECT_TRUE(s.ok());
    }

@@ -370,8 +366,8 @@ class TestPlainTableFactory : public PlainTableFactory {
    std::unique_ptr<PlainTableReader> new_reader(new TestPlainTableReader(
        table_reader_options.env_options,
        table_reader_options.internal_comparator, encoding_type, file_size,
-        bloom_bits_per_key_, hash_table_ratio_, index_sparseness_, props,
-        std::move(file), table_reader_options.ioptions,
+        bloom_bits_per_key_, hash_table_ratio_, index_sparseness_,
+        std::move(props), std::move(file), table_reader_options.ioptions,
        table_reader_options.prefix_extractor, expect_bloom_not_match_,
        store_index_in_file_, column_family_id_, column_family_name_));

@@ -635,7 +635,8 @@ class Repairer {
            table->meta.fd.largest_seqno, table->meta.marked_for_compaction,
            table->meta.oldest_blob_file_number,
            table->meta.oldest_ancester_time, table->meta.file_creation_time,
-            table->meta.file_checksum, table->meta.file_checksum_func_name);
+            table->meta.file_checksum, table->meta.file_checksum_func_name,
+            table->meta.min_timestamp, table->meta.max_timestamp);
      }
      assert(next_file_number_ > 0);
      vset_.MarkFileNumberUsed(next_file_number_ - 1);
@@ -655,6 +655,16 @@ size_t TableCache::GetMemoryUsageByTableReader(
  return ret;
 }

+bool TableCache::HasEntry(Cache* cache, uint64_t file_number) {
+  Cache::Handle* handle = cache->Lookup(GetSliceForFileNumber(&file_number));
+  if (handle) {
+    cache->Release(handle);
+    return true;
+  } else {
+    return false;
+  }
+}
+
 void TableCache::Evict(Cache* cache, uint64_t file_number) {
  cache->Erase(GetSliceForFileNumber(&file_number));
 }
@@ -125,6 +125,9 @@ class TableCache {
  // Evict any entry for the specified file number
  static void Evict(Cache* cache, uint64_t file_number);

+  // Query whether specified file number is currently in cache
+  static bool HasEntry(Cache* cache, uint64_t file_number);
+
  // Clean table handle and erase it from the table cache
  // Used in DB close, or the file is not live anymore.
  void EraseHandle(const FileDescriptor& fd, Cache::Handle* handle);
@@ -6,12 +6,14 @@
 // This file defines a collection of statistics collectors.
 #pragma once

-#include "rocksdb/table_properties.h"
-
 #include <memory>
 #include <string>
 #include <vector>

+#include "db/dbformat.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/table_properties.h"
+
 namespace ROCKSDB_NAMESPACE {

 // Base class for internal table properties collector.
@@ -108,4 +110,66 @@ class UserKeyTablePropertiesCollectorFactory
  std::shared_ptr<TablePropertiesCollectorFactory> user_collector_factory_;
 };

+// When rocksdb creates a newtable, it will encode all "user keys" into
+// "internal keys". This class collects min/max timestamp from the encoded
+// internal key when Add() is invoked.
+//
+// @param cmp  the user comparator to compare the timestamps in internal key.
+class TimestampTablePropertiesCollector : public IntTblPropCollector {
+ public:
+  explicit TimestampTablePropertiesCollector(const Comparator* cmp)
+      : cmp_(cmp),
+        timestamp_min_(kDisableUserTimestamp),
+        timestamp_max_(kDisableUserTimestamp) {}
+
+  Status InternalAdd(const Slice& key, const Slice& /* value */,
+                     uint64_t /* file_size */) override {
+    auto user_key = ExtractUserKey(key);
+    assert(cmp_ && cmp_->timestamp_size() > 0);
+    if (user_key.size() < cmp_->timestamp_size()) {
+      return Status::Corruption(
+          "User key size mismatch when comparing to timestamp size.");
+    }
+    auto timestamp_in_key =
+        ExtractTimestampFromUserKey(user_key, cmp_->timestamp_size());
+    if (timestamp_max_ == kDisableUserTimestamp ||
+        cmp_->CompareTimestamp(timestamp_in_key, timestamp_max_) > 0) {
+      timestamp_max_.assign(timestamp_in_key.data(), timestamp_in_key.size());
+    }
+    if (timestamp_min_ == kDisableUserTimestamp ||
+        cmp_->CompareTimestamp(timestamp_min_, timestamp_in_key) > 0) {
+      timestamp_min_.assign(timestamp_in_key.data(), timestamp_in_key.size());
+    }
+    return Status::OK();
+  }
+
+  void BlockAdd(uint64_t /* block_raw_bytes */,
+                uint64_t /* block_compressed_bytes_fast */,
+                uint64_t /* block_compressed_bytes_slow */) override {
+    return;
+  }
+
+  Status Finish(UserCollectedProperties* properties) override {
+    assert(timestamp_min_.size() == timestamp_max_.size() &&
+           timestamp_max_.size() == cmp_->timestamp_size());
+    properties->insert({"rocksdb.timestamp_min", timestamp_min_});
+    properties->insert({"rocksdb.timestamp_max", timestamp_max_});
+    return Status::OK();
+  }
+
+  const char* Name() const override {
+    return "TimestampTablePropertiesCollector";
+  }
+
+  UserCollectedProperties GetReadableProperties() const override {
+    return {{"rocksdb.timestamp_min", Slice(timestamp_min_).ToString(true)},
+            {"rocksdb.timestamp_max", Slice(timestamp_max_).ToString(true)}};
+  }
+
+ protected:
+  const Comparator* const cmp_;
+  std::string timestamp_min_;
+  std::string timestamp_max_;
+};
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -291,11 +291,9 @@ void TestCustomizedTablePropertiesCollector(
  std::unique_ptr<RandomAccessFileReader> fake_file_reader(
      new RandomAccessFileReader(std::move(source), "test"));

-  TableProperties* props;
+  std::unique_ptr<TableProperties> props;
  Status s = ReadTableProperties(fake_file_reader.get(), fwf->contents().size(),
-                                 magic_number, ioptions, &props,
-                                 true /* compression_type_missing */);
-  std::unique_ptr<TableProperties> props_guard(props);
+                                 magic_number, ioptions, &props);
  ASSERT_OK(s);

  auto user_collected = props->user_collected_properties;
@@ -432,13 +430,11 @@ void TestInternalKeyPropertiesCollector(
    std::unique_ptr<RandomAccessFileReader> reader(
        new RandomAccessFileReader(std::move(source), "test"));

-    TableProperties* props;
-    Status s =
-        ReadTableProperties(reader.get(), fwf->contents().size(), magic_number,
-                            ioptions, &props, true /* compression_type_missing */);
+    std::unique_ptr<TableProperties> props;
+    Status s = ReadTableProperties(reader.get(), fwf->contents().size(),
+                                   magic_number, ioptions, &props);
    ASSERT_OK(s);

-    std::unique_ptr<TableProperties> props_guard(props);
    auto user_collected = props->user_collected_properties;
    uint64_t deleted = GetDeletedKeys(user_collected);
    ASSERT_EQ(5u, deleted);  // deletes + single-deletes
@@ -176,6 +176,11 @@ class VersionBuilder::Rep {
      return shared_meta_;
    }

+    uint64_t GetBlobFileNumber() const {
+      assert(shared_meta_);
+      return shared_meta_->GetBlobFileNumber();
+    }
+
    bool HasDelta() const { return !delta_.IsEmpty(); }

    const std::unordered_set<uint64_t>& GetLinkedSsts() const {
@@ -309,7 +314,7 @@ class VersionBuilder::Rep {
    (*expected_linked_ssts)[blob_file_number].emplace(table_file_number);
  }

-  template <class Checker>
+  template <typename Checker>
  Status CheckConsistencyDetailsForLevel(
      const VersionStorageInfo* vstorage, int level, Checker checker,
      const std::string& sync_point,
@@ -669,28 +674,6 @@ class VersionBuilder::Rep {
    return meta->oldest_blob_file_number;
  }

-  uint64_t GetMinOldestBlobFileNumber() const {
-    uint64_t min_oldest_blob_file_num = std::numeric_limits<uint64_t>::max();
-    for (int level = 0; level < num_levels_; ++level) {
-      const auto& base_files = base_vstorage_->LevelFiles(level);
-      for (const auto* fmeta : base_files) {
-        assert(fmeta);
-        min_oldest_blob_file_num =
-            std::min(min_oldest_blob_file_num, fmeta->oldest_blob_file_number);
-      }
-      const auto& added_files = levels_[level].added_files;
-      for (const auto& elem : added_files) {
-        assert(elem.second);
-        min_oldest_blob_file_num = std::min(
-            min_oldest_blob_file_num, elem.second->oldest_blob_file_number);
-      }
-    }
-    if (min_oldest_blob_file_num == std::numeric_limits<uint64_t>::max()) {
-      min_oldest_blob_file_num = kInvalidBlobFileNumber;
-    }
-    return min_oldest_blob_file_num;
-  }
-
  Status ApplyFileDeletion(int level, uint64_t file_number) {
    assert(level != VersionStorageInfo::FileLocation::Invalid().GetLevel());

@@ -864,48 +847,26 @@ class VersionBuilder::Rep {
    return Status::OK();
  }

-  static std::shared_ptr<BlobFileMetaData> CreateBlobFileMetaData(
-      const MutableBlobFileMetaData& mutable_meta) {
-    return BlobFileMetaData::Create(
-        mutable_meta.GetSharedMeta(), mutable_meta.GetLinkedSsts(),
-        mutable_meta.GetGarbageBlobCount(), mutable_meta.GetGarbageBlobBytes());
-  }
-
-  // Add the blob file specified by meta to *vstorage if it is determined to
-  // contain valid data (blobs). We make this decision based on the amount
-  // of garbage in the file, and whether the file or any lower-numbered blob
-  // files have any linked SSTs. The latter condition is tracked using the
-  // flag *found_first_non_empty.
-  void AddBlobFileIfNeeded(VersionStorageInfo* vstorage,
-                           const std::shared_ptr<BlobFileMetaData>& meta,
-                           bool* found_first_non_empty) const {
-    assert(vstorage);
-    assert(meta);
-    assert(found_first_non_empty);
-
-    if (!meta->GetLinkedSsts().empty()) {
-      (*found_first_non_empty) = true;
-    } else if (!(*found_first_non_empty) ||
-               meta->GetGarbageBlobCount() >= meta->GetTotalBlobCount()) {
-      return;
-    }
-
-    vstorage->AddBlobFile(meta);
-  }
-
-  // Merge the blob file metadata from the base version with the changes (edits)
-  // applied, and save the result into *vstorage.
-  void SaveBlobFilesTo(VersionStorageInfo* vstorage) const {
+  // Helper function template for merging the blob file metadata from the base
+  // version with the mutable metadata representing the state after applying the
+  // edits. The function objects process_base and process_mutable are
+  // respectively called to handle a base version object when there is no
+  // matching mutable object, and a mutable object when there is no matching
+  // base version object. process_both is called to perform the merge when a
+  // given blob file appears both in the base version and the mutable list. The
+  // helper stops processing objects if a function object returns false. Blob
+  // files with a file number below first_blob_file are not processed.
+  template <typename ProcessBase, typename ProcessMutable, typename ProcessBoth>
+  void MergeBlobFileMetas(uint64_t first_blob_file, ProcessBase process_base,
+                          ProcessMutable process_mutable,
+                          ProcessBoth process_both) const {
    assert(base_vstorage_);
-    assert(vstorage);
-
-    bool found_first_non_empty = false;

    const auto& base_blob_files = base_vstorage_->GetBlobFiles();
-    auto base_it = base_blob_files.begin();
+    auto base_it = base_blob_files.lower_bound(first_blob_file);
    const auto base_it_end = base_blob_files.end();

-    auto mutable_it = mutable_blob_file_metas_.begin();
+    auto mutable_it = mutable_blob_file_metas_.lower_bound(first_blob_file);
    const auto mutable_it_end = mutable_blob_file_metas_.end();

    while (base_it != base_it_end && mutable_it != mutable_it_end) {
@@ -915,15 +876,17 @@ class VersionBuilder::Rep {
      if (base_blob_file_number < mutable_blob_file_number) {
        const auto& base_meta = base_it->second;

-        AddBlobFileIfNeeded(vstorage, base_meta, &found_first_non_empty);
+        if (!process_base(base_meta)) {
+          return;
+        }

        ++base_it;
      } else if (mutable_blob_file_number < base_blob_file_number) {
        const auto& mutable_meta = mutable_it->second;

-        auto meta = CreateBlobFileMetaData(mutable_meta);
-
-        AddBlobFileIfNeeded(vstorage, meta, &found_first_non_empty);
+        if (!process_mutable(mutable_meta)) {
+          return;
+        }

        ++mutable_it;
      } else {
@@ -932,21 +895,8 @@ class VersionBuilder::Rep {
        const auto& base_meta = base_it->second;
        const auto& mutable_meta = mutable_it->second;

-        assert(base_meta);
-        assert(base_meta->GetSharedMeta() == mutable_meta.GetSharedMeta());
-
-        if (!mutable_meta.HasDelta()) {
-          assert(base_meta->GetGarbageBlobCount() ==
-                 mutable_meta.GetGarbageBlobCount());
-          assert(base_meta->GetGarbageBlobBytes() ==
-                 mutable_meta.GetGarbageBlobBytes());
-          assert(base_meta->GetLinkedSsts() == mutable_meta.GetLinkedSsts());
-
-          AddBlobFileIfNeeded(vstorage, base_meta, &found_first_non_empty);
-        } else {
-          auto meta = CreateBlobFileMetaData(mutable_meta);
-
-          AddBlobFileIfNeeded(vstorage, meta, &found_first_non_empty);
+        if (!process_both(base_meta, mutable_meta)) {
+          return;
        }

        ++base_it;
@@ -957,7 +907,9 @@ class VersionBuilder::Rep {
    while (base_it != base_it_end) {
      const auto& base_meta = base_it->second;

-      AddBlobFileIfNeeded(vstorage, base_meta, &found_first_non_empty);
+      if (!process_base(base_meta)) {
+        return;
+      }

      ++base_it;
    }
@@ -965,14 +917,142 @@ class VersionBuilder::Rep {
    while (mutable_it != mutable_it_end) {
      const auto& mutable_meta = mutable_it->second;

-      auto meta = CreateBlobFileMetaData(mutable_meta);
-
-      AddBlobFileIfNeeded(vstorage, meta, &found_first_non_empty);
+      if (!process_mutable(mutable_meta)) {
+        return;
+      }

      ++mutable_it;
    }
  }

+  // Helper function template for finding the first blob file that has linked
+  // SSTs.
+  template <typename Meta>
+  static bool CheckLinkedSsts(const Meta& meta,
+                              uint64_t* min_oldest_blob_file_num) {
+    assert(min_oldest_blob_file_num);
+
+    if (!meta.GetLinkedSsts().empty()) {
+      assert(*min_oldest_blob_file_num == kInvalidBlobFileNumber);
+
+      *min_oldest_blob_file_num = meta.GetBlobFileNumber();
+
+      return false;
+    }
+
+    return true;
+  }
+
+  // Find the oldest blob file that has linked SSTs.
+  uint64_t GetMinOldestBlobFileNumber() const {
+    uint64_t min_oldest_blob_file_num = kInvalidBlobFileNumber;
+
+    auto process_base =
+        [&min_oldest_blob_file_num](
+            const std::shared_ptr<BlobFileMetaData>& base_meta) {
+          assert(base_meta);
+
+          return CheckLinkedSsts(*base_meta, &min_oldest_blob_file_num);
+        };
+
+    auto process_mutable = [&min_oldest_blob_file_num](
+                               const MutableBlobFileMetaData& mutable_meta) {
+      return CheckLinkedSsts(mutable_meta, &min_oldest_blob_file_num);
+    };
+
+    auto process_both = [&min_oldest_blob_file_num](
+                            const std::shared_ptr<BlobFileMetaData>& base_meta,
+                            const MutableBlobFileMetaData& mutable_meta) {
+#ifndef NDEBUG
+      assert(base_meta);
+      assert(base_meta->GetSharedMeta() == mutable_meta.GetSharedMeta());
+#else
+      (void)base_meta;
+#endif
+
+      // Look at mutable_meta since it supersedes *base_meta
+      return CheckLinkedSsts(mutable_meta, &min_oldest_blob_file_num);
+    };
+
+    MergeBlobFileMetas(kInvalidBlobFileNumber, process_base, process_mutable,
+                       process_both);
+
+    return min_oldest_blob_file_num;
+  }
+
+  static std::shared_ptr<BlobFileMetaData> CreateBlobFileMetaData(
+      const MutableBlobFileMetaData& mutable_meta) {
+    return BlobFileMetaData::Create(
+        mutable_meta.GetSharedMeta(), mutable_meta.GetLinkedSsts(),
+        mutable_meta.GetGarbageBlobCount(), mutable_meta.GetGarbageBlobBytes());
+  }
+
+  // Add the blob file specified by meta to *vstorage if it is determined to
+  // contain valid data (blobs).
+  template <typename Meta>
+  static void AddBlobFileIfNeeded(VersionStorageInfo* vstorage, Meta&& meta) {
+    assert(vstorage);
+    assert(meta);
+
+    if (meta->GetLinkedSsts().empty() &&
+        meta->GetGarbageBlobCount() >= meta->GetTotalBlobCount()) {
+      return;
+    }
+
+    vstorage->AddBlobFile(std::forward<Meta>(meta));
+  }
+
+  // Merge the blob file metadata from the base version with the changes (edits)
+  // applied, and save the result into *vstorage.
+  void SaveBlobFilesTo(VersionStorageInfo* vstorage) const {
+    assert(vstorage);
+
+    const uint64_t oldest_blob_file_with_linked_ssts =
+        GetMinOldestBlobFileNumber();
+
+    auto process_base =
+        [vstorage](const std::shared_ptr<BlobFileMetaData>& base_meta) {
+          assert(base_meta);
+
+          AddBlobFileIfNeeded(vstorage, base_meta);
+
+          return true;
+        };
+
+    auto process_mutable =
+        [vstorage](const MutableBlobFileMetaData& mutable_meta) {
+          AddBlobFileIfNeeded(vstorage, CreateBlobFileMetaData(mutable_meta));
+
+          return true;
+        };
+
+    auto process_both = [vstorage](
+                            const std::shared_ptr<BlobFileMetaData>& base_meta,
+                            const MutableBlobFileMetaData& mutable_meta) {
+      assert(base_meta);
+      assert(base_meta->GetSharedMeta() == mutable_meta.GetSharedMeta());
+
+      if (!mutable_meta.HasDelta()) {
+        assert(base_meta->GetGarbageBlobCount() ==
+               mutable_meta.GetGarbageBlobCount());
+        assert(base_meta->GetGarbageBlobBytes() ==
+               mutable_meta.GetGarbageBlobBytes());
+        assert(base_meta->GetLinkedSsts() == mutable_meta.GetLinkedSsts());
+
+        AddBlobFileIfNeeded(vstorage, base_meta);
+
+        return true;
+      }
+
+      AddBlobFileIfNeeded(vstorage, CreateBlobFileMetaData(mutable_meta));
+
+      return true;
+    };
+
+    MergeBlobFileMetas(oldest_blob_file_with_linked_ssts, process_base,
+                       process_mutable, process_both);
+  }
+
  void MaybeAddFile(VersionStorageInfo* vstorage, int level,
                    FileMetaData* f) const {
    const uint64_t file_number = f->fd.GetNumber();
@@ -999,7 +1079,7 @@ class VersionBuilder::Rep {
    }
  }

-  template <class Cmp>
+  template <typename Cmp>
  void SaveSSTFilesTo(VersionStorageInfo* vstorage, int level, Cmp cmp) const {
    // Merge the set of added files with the set of pre-existing files.
    // Drop any deleted files.  Store the result in *vstorage.
@@ -69,7 +69,8 @@ class VersionBuilderTest : public testing::Test {
        GetInternalKey(largest, largest_seq), smallest_seqno, largest_seqno,
        /* marked_for_compact */ false, oldest_blob_file_number,
        kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-        kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+        kUnknownFileChecksum, kUnknownFileChecksumFuncName,
+        kDisableUserTimestamp, kDisableUserTimestamp);
    f->compensated_file_size = file_size;
    f->num_entries = num_entries;
    f->num_deletions = num_deletions;
@@ -130,7 +131,8 @@ class VersionBuilderTest : public testing::Test {
                  smallest_seqno, largest_seqno, marked_for_compaction,
                  blob_file_number, kUnknownOldestAncesterTime,
                  kUnknownFileCreationTime, kUnknownFileChecksum,
-                  kUnknownFileChecksumFuncName);
+                  kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                  kDisableUserTimestamp);
  }

  static std::shared_ptr<BlobFileMetaData> GetBlobFileMetaData(
@@ -190,7 +192,8 @@ TEST_F(VersionBuilderTest, ApplyAndSaveTo) {
                       GetInternalKey("350"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit.DeleteFile(3, 27U);

  EnvOptions env_options;
@@ -230,7 +233,8 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) {
                       GetInternalKey("350"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit.DeleteFile(0, 1U);
  version_edit.DeleteFile(0, 88U);

@@ -273,7 +277,8 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) {
                       GetInternalKey("350"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit.DeleteFile(0, 1U);
  version_edit.DeleteFile(0, 88U);
  version_edit.DeleteFile(4, 6U);
@@ -307,27 +312,32 @@ TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) {
                       GetInternalKey("350"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
                       GetInternalKey("450"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
                       GetInternalKey("650"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
                       GetInternalKey("550"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
                       GetInternalKey("750"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);

  EnvOptions env_options;
  constexpr TableCache* table_cache = nullptr;
@@ -364,27 +374,32 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
                       GetInternalKey("350"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit.AddFile(2, 676, 0, 100U, GetInternalKey("401"),
                       GetInternalKey("450"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit.AddFile(2, 636, 0, 100U, GetInternalKey("601"),
                       GetInternalKey("650"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit.AddFile(2, 616, 0, 100U, GetInternalKey("501"),
                       GetInternalKey("550"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit.AddFile(2, 606, 0, 100U, GetInternalKey("701"),
                       GetInternalKey("750"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  ASSERT_OK(version_builder.Apply(&version_edit));

  VersionEdit version_edit2;
@@ -392,14 +407,16 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
                       GetInternalKey("950"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);
  version_edit2.DeleteFile(2, 616);
  version_edit2.DeleteFile(2, 636);
  version_edit.AddFile(2, 806, 0, 100U, GetInternalKey("801"),
                       GetInternalKey("850"), 200, 200, false,
                       kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                       kUnknownFileCreationTime, kUnknownFileChecksum,
-                       kUnknownFileChecksumFuncName);
+                       kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                       kDisableUserTimestamp);

  ASSERT_OK(version_builder.Apply(&version_edit2));
  ASSERT_OK(version_builder.SaveTo(&new_vstorage));
@@ -500,7 +517,8 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) {
                   GetInternalKey(largest, largest_seq), smallest_seqno,
                   largest_seqno, marked_for_compaction, kInvalidBlobFileNumber,
                   kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-                   kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+                   kUnknownFileChecksum, kUnknownFileChecksumFuncName,
+                   kDisableUserTimestamp, kDisableUserTimestamp);

  ASSERT_OK(builder.Apply(&addition));

@@ -544,7 +562,8 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyInBase) {
               smallest_seqno, largest_seqno, marked_for_compaction,
               kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
+               kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+               kDisableUserTimestamp);

  const Status s = builder.Apply(&edit);
  ASSERT_TRUE(s.IsCorruption());
@@ -577,7 +596,8 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) {
               GetInternalKey(largest), smallest_seqno, largest_seqno,
               marked_for_compaction, kInvalidBlobFileNumber,
               kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-               kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+               kUnknownFileChecksum, kUnknownFileChecksumFuncName,
+               kDisableUserTimestamp, kDisableUserTimestamp);

  ASSERT_OK(builder.Apply(&edit));

@@ -590,7 +610,8 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) {
                     smallest_seqno, largest_seqno, marked_for_compaction,
                     kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                     kUnknownFileCreationTime, kUnknownFileChecksum,
-                     kUnknownFileChecksumFuncName);
+                     kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                     kDisableUserTimestamp);

  const Status s = builder.Apply(&other_edit);
  ASSERT_TRUE(s.IsCorruption());
@@ -624,7 +645,8 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAndDeletion) {
                   smallest_seqno, largest_seqno, marked_for_compaction,
                   kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
                   kUnknownFileCreationTime, kUnknownFileChecksum,
-                   kUnknownFileChecksumFuncName);
+                   kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+                   kDisableUserTimestamp);

  ASSERT_OK(builder.Apply(&addition));

@@ -1154,7 +1176,8 @@ TEST_F(VersionBuilderTest, SaveBlobFilesToConcurrentJobs) {
               GetInternalKey(smallest), GetInternalKey(largest),
               smallest_seqno, largest_seqno, marked_for_compaction,
               blob_file_number, kUnknownOldestAncesterTime,
-               kUnknownFileCreationTime, checksum_value, checksum_method);
+               kUnknownFileCreationTime, checksum_value, checksum_method,
+               kDisableUserTimestamp, kDisableUserTimestamp);
  edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
                   checksum_method, checksum_value);

@@ -1238,7 +1261,8 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
               /* largest_seqno */ 200, /* marked_for_compaction */ false,
               /* oldest_blob_file_number */ 16, kUnknownOldestAncesterTime,
               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
+               kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+               kDisableUserTimestamp);

  edit.AddFile(/* level */ 1, /* file_number */ 700, /* path_id */ 0,
               /* file_size */ 100, /* smallest */ GetInternalKey("801"),
@@ -1246,7 +1270,8 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
               /* largest_seqno */ 200, /* marked_for_compaction */ false,
               /* oldest_blob_file_number */ 1000, kUnknownOldestAncesterTime,
               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
+               kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+               kDisableUserTimestamp);
  edit.AddBlobFile(/* blob_file_number */ 1000, /* total_blob_count */ 2000,
                   /* total_blob_bytes */ 200000,
                   /* checksum_method */ std::string(),
@@ -1464,7 +1489,8 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
      /* largest_seqno */ 2100, /* marked_for_compaction */ false,
      /* oldest_blob_file_number */ 1, kUnknownOldestAncesterTime,
      kUnknownFileCreationTime, kUnknownFileChecksum,
-      kUnknownFileChecksumFuncName);
+      kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+      kDisableUserTimestamp);

  // Add an SST that does not reference any blob files.
  edit.AddFile(
@@ -1474,7 +1500,8 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
      /* largest_seqno */ 2200, /* marked_for_compaction */ false,
      kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
      kUnknownFileCreationTime, kUnknownFileChecksum,
-      kUnknownFileChecksumFuncName);
+      kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+      kDisableUserTimestamp);

  // Delete a file that references a blob file.
  edit.DeleteFile(/* level */ 1, /* file_number */ 6);
@@ -1496,7 +1523,8 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
               /* largest_seqno */ 300, /* marked_for_compaction */ false,
               /* oldest_blob_file_number */ 3, kUnknownOldestAncesterTime,
               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
+               kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+               kDisableUserTimestamp);

  // Trivially move a file that does not reference any blob files.
  edit.DeleteFile(/* level */ 1, /* file_number */ 13);
@@ -1507,7 +1535,8 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
               /* largest_seqno */ 1300, /* marked_for_compaction */ false,
               kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
+               kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+               kDisableUserTimestamp);

  // Add one more SST file that references a blob file, then promptly
  // delete it in a second version edit before the new version gets saved.
@@ -1519,7 +1548,8 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
               /* largest_seqno */ 2300, /* marked_for_compaction */ false,
               /* oldest_blob_file_number */ 5, kUnknownOldestAncesterTime,
               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
+               kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+               kDisableUserTimestamp);

  VersionEdit edit2;

@@ -186,6 +186,16 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
    PutVarint32(dst, NewFileCustomTag::kFileChecksumFuncName);
    PutLengthPrefixedSlice(dst, Slice(f.file_checksum_func_name));

+    if (f.max_timestamp != kDisableUserTimestamp) {
+      if (f.min_timestamp.size() != f.max_timestamp.size()) {
+        assert(false);
+        return false;
+      }
+      PutVarint32(dst, NewFileCustomTag::kMinTimestamp);
+      PutLengthPrefixedSlice(dst, Slice(f.min_timestamp));
+      PutVarint32(dst, NewFileCustomTag::kMaxTimestamp);
+      PutLengthPrefixedSlice(dst, Slice(f.max_timestamp));
+    }
    if (f.fd.GetPathId() != 0) {
      PutVarint32(dst, NewFileCustomTag::kPathId);
      char p = static_cast<char>(f.fd.GetPathId());
@@ -315,6 +325,10 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
        return "new-file4 custom field";
      }
      if (custom_tag == kTerminate) {
+        if (f.min_timestamp.size() != f.max_timestamp.size()) {
+          assert(false);
+          return "new-file4 custom field timestamp size mismatch error";
+        }
        break;
      }
      if (!GetLengthPrefixedSlice(input, &field)) {
@@ -375,6 +389,12 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
            }
          }
          break;
+        case kMinTimestamp:
+          f.min_timestamp = field.ToString();
+          break;
+        case kMaxTimestamp:
+          f.max_timestamp = field.ToString();
+          break;
        default:
          if ((custom_tag & kCustomTagNonSafeIgnoreMask) != 0) {
            // Should not proceed if cannot understand it
@@ -781,6 +801,10 @@ std::string VersionEdit::DebugString(bool hex_key) const {
      r.append(" blob_file:");
      AppendNumberTo(&r, f.oldest_blob_file_number);
    }
+    r.append(" min_timestamp:");
+    r.append(Slice(f.min_timestamp).ToString(true));
+    r.append(" max_timestamp:");
+    r.append(Slice(f.max_timestamp).ToString(true));
    r.append(" oldest_ancester_time:");
    AppendNumberTo(&r, f.oldest_ancester_time);
    r.append(" file_creation_time:");
@@ -894,6 +918,11 @@ std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
      jw << "FileSize" << f.fd.GetFileSize();
      jw << "SmallestIKey" << f.smallest.DebugString(hex_key);
      jw << "LargestIKey" << f.largest.DebugString(hex_key);
+      if (f.min_timestamp != kDisableUserTimestamp) {
+        assert(f.max_timestamp != kDisableUserTimestamp);
+        jw << "MinTimestamp" << Slice(f.min_timestamp).ToString(true);
+        jw << "MaxTimestamp" << Slice(f.max_timestamp).ToString(true);
+      }
      if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
        jw << "OldestBlobFile" << f.oldest_blob_file_number;
      }
@@ -82,6 +82,8 @@ enum NewFileCustomTag : uint32_t {
  kFileChecksum = 7,
  kFileChecksumFuncName = 8,
  kTemperature = 9,
+  kMinTimestamp = 10,
+  kMaxTimestamp = 11,

  // If this bit for the custom tag is set, opening DB should fail if
  // we don't know this field.
@@ -209,6 +211,10 @@ struct FileMetaData {

  // File checksum function name
  std::string file_checksum_func_name = kUnknownFileChecksumFuncName;
+  // Min (oldest) timestamp of keys in this file
+  std::string min_timestamp;
+  // Max (newest) timestamp of keys in this file
+  std::string max_timestamp;

  FileMetaData() = default;

@@ -218,7 +224,8 @@ struct FileMetaData {
               const SequenceNumber& largest_seq, bool marked_for_compact,
               uint64_t oldest_blob_file, uint64_t _oldest_ancester_time,
               uint64_t _file_creation_time, const std::string& _file_checksum,
-               const std::string& _file_checksum_func_name)
+               const std::string& _file_checksum_func_name,
+               std::string _min_timestamp, std::string _max_timestamp)
      : fd(file, file_path_id, file_size, smallest_seq, largest_seq),
        smallest(smallest_key),
        largest(largest_key),
@@ -227,7 +234,9 @@ struct FileMetaData {
        oldest_ancester_time(_oldest_ancester_time),
        file_creation_time(_file_creation_time),
        file_checksum(_file_checksum),
-        file_checksum_func_name(_file_checksum_func_name) {
+        file_checksum_func_name(_file_checksum_func_name),
+        min_timestamp(std::move(_min_timestamp)),
+        max_timestamp(std::move(_max_timestamp)) {
    TEST_SYNC_POINT_CALLBACK("FileMetaData::FileMetaData", this);
  }

@@ -394,14 +403,17 @@ class VersionEdit {
               const SequenceNumber& largest_seqno, bool marked_for_compaction,
               uint64_t oldest_blob_file_number, uint64_t oldest_ancester_time,
               uint64_t file_creation_time, const std::string& file_checksum,
-               const std::string& file_checksum_func_name) {
+               const std::string& file_checksum_func_name,
+               const std::string& min_timestamp,
+               const std::string& max_timestamp) {
    assert(smallest_seqno <= largest_seqno);
    new_files_.emplace_back(
-        level, FileMetaData(file, file_path_id, file_size, smallest, largest,
-                            smallest_seqno, largest_seqno,
-                            marked_for_compaction, oldest_blob_file_number,
-                            oldest_ancester_time, file_creation_time,
-                            file_checksum, file_checksum_func_name));
+        level,
+        FileMetaData(file, file_path_id, file_size, smallest, largest,
+                     smallest_seqno, largest_seqno, marked_for_compaction,
+                     oldest_blob_file_number, oldest_ancester_time,
+                     file_creation_time, file_checksum, file_checksum_func_name,
+                     min_timestamp, max_timestamp));
  }

  void AddFile(int level, const FileMetaData& f) {
@@ -517,7 +517,11 @@ Status VersionEditHandler::MaybeCreateVersion(const VersionEdit& /*edit*/,
 Status VersionEditHandler::LoadTables(ColumnFamilyData* cfd,
                                      bool prefetch_index_and_filter_in_cache,
                                      bool is_initial_load) {
-  if (skip_load_table_files_) {
+  bool skip_load_table_files = skip_load_table_files_;
+  TEST_SYNC_POINT_CALLBACK(
+      "VersionEditHandler::LoadTables:skip_load_table_files",
+      &skip_load_table_files);
+  if (skip_load_table_files) {
    return Status::OK();
  }
  assert(cfd != nullptr);
@@ -634,6 +638,11 @@ void VersionEditHandlerPointInTime::CheckIterationResult(
        versions_.erase(v_iter);
      }
    }
+  } else {
+    for (const auto& elem : versions_) {
+      delete elem.second;
+    }
+    versions_.clear();
  }
 }

@@ -870,7 +879,7 @@ Status ManifestTailer::OnColumnFamilyAdd(VersionEdit& edit,

 #ifndef NDEBUG
  auto version_iter = versions_.find(edit.GetColumnFamily());
-  assert(version_iter != versions_.end());
+  assert(version_iter == versions_.end());
 #endif  // !NDEBUG
  return Status::OK();
 }
@@ -40,7 +40,7 @@ TEST_F(VersionEditTest, EncodeDecode) {
                 InternalKey("foo", kBig + 500 + i, kTypeValue),
                 InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
                 kBig + 500 + i, kBig + 600 + i, false, kInvalidBlobFileNumber,
-                 888, 678, "234", "crc32c");
+                 888, 678, "234", "crc32c", "123", "345");
    edit.DeleteFile(4, kBig + 700 + i);
  }

@@ -59,21 +59,24 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
               InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
               kBig + 600, true, kInvalidBlobFileNumber,
               kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-               kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+               kUnknownFileChecksum, kUnknownFileChecksumFuncName, "123",
+               "234");
  edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
               InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
               kBig + 601, false, kInvalidBlobFileNumber,
               kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-               kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+               kUnknownFileChecksum, kUnknownFileChecksumFuncName, "345",
+               "543");
  edit.AddFile(5, 302, 0, 100, InternalKey("foo", kBig + 502, kTypeValue),
               InternalKey("zoo", kBig + 602, kTypeDeletion), kBig + 502,
               kBig + 602, true, kInvalidBlobFileNumber, 666, 888,
-               kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+               kUnknownFileChecksum, kUnknownFileChecksumFuncName, "456",
+               "567");
  edit.AddFile(5, 303, 0, 100, InternalKey("foo", kBig + 503, kTypeBlobIndex),
               InternalKey("zoo", kBig + 603, kTypeBlobIndex), kBig + 503,
               kBig + 603, true, 1001, kUnknownOldestAncesterTime,
               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
+               kUnknownFileChecksumFuncName, "678", "789");
  ;

  edit.DeleteFile(4, 700);
@@ -105,6 +108,14 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
  ASSERT_EQ(kInvalidBlobFileNumber,
            new_files[2].second.oldest_blob_file_number);
  ASSERT_EQ(1001, new_files[3].second.oldest_blob_file_number);
+  ASSERT_EQ("123", new_files[0].second.min_timestamp);
+  ASSERT_EQ("234", new_files[0].second.max_timestamp);
+  ASSERT_EQ("345", new_files[1].second.min_timestamp);
+  ASSERT_EQ("543", new_files[1].second.max_timestamp);
+  ASSERT_EQ("456", new_files[2].second.min_timestamp);
+  ASSERT_EQ("567", new_files[2].second.max_timestamp);
+  ASSERT_EQ("678", new_files[3].second.min_timestamp);
+  ASSERT_EQ("789", new_files[3].second.max_timestamp);
 }

 TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
@@ -114,11 +125,12 @@ TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
               InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
               kBig + 600, true, kInvalidBlobFileNumber,
               kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-               kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+               kUnknownFileChecksum, kUnknownFileChecksumFuncName, "123",
+               "234");
  edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
               InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
               kBig + 601, false, kInvalidBlobFileNumber, 686, 868, "234",
-               "crc32c");
+               "crc32c", kDisableUserTimestamp, kDisableUserTimestamp);
  edit.DeleteFile(4, 700);

  edit.SetComparatorName("foo");
@@ -157,6 +169,10 @@ TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
  ASSERT_EQ(3u, new_files[0].second.fd.GetPathId());
  ASSERT_EQ(3u, new_files[1].second.fd.GetPathId());
  ASSERT_EQ(1u, parsed.GetDeletedFiles().size());
+  ASSERT_EQ("123", new_files[0].second.min_timestamp);
+  ASSERT_EQ("234", new_files[0].second.max_timestamp);
+  ASSERT_EQ(kDisableUserTimestamp, new_files[1].second.min_timestamp);
+  ASSERT_EQ(kDisableUserTimestamp, new_files[1].second.max_timestamp);
 }

 TEST_F(VersionEditTest, NewFile4NotSupportedField) {
@@ -166,7 +182,8 @@ TEST_F(VersionEditTest, NewFile4NotSupportedField) {
               InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
               kBig + 600, true, kInvalidBlobFileNumber,
               kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-               kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+               kUnknownFileChecksum, kUnknownFileChecksumFuncName,
+               kDisableUserTimestamp, kDisableUserTimestamp);

  edit.SetComparatorName("foo");
  edit.SetLogNumber(kBig + 100);
@@ -196,7 +213,8 @@ TEST_F(VersionEditTest, EncodeEmptyFile) {
  edit.AddFile(0, 0, 0, 0, InternalKey(), InternalKey(), 0, 0, false,
               kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
+               kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+               kDisableUserTimestamp);
  std::string buffer;
  ASSERT_TRUE(!edit.EncodeTo(&buffer));
 }
@@ -880,7 +880,8 @@ class LevelIterator final : public InternalIterator {
        level_(level),
        range_del_agg_(range_del_agg),
        pinned_iters_mgr_(nullptr),
-        compaction_boundaries_(compaction_boundaries) {
+        compaction_boundaries_(compaction_boundaries),
+        is_next_read_sequential_(false) {
    // Empty level is not supported.
    assert(flevel_ != nullptr && flevel_->num_files > 0);
  }
@@ -1027,6 +1028,8 @@ class LevelIterator final : public InternalIterator {
  // To be propagated to RangeDelAggregator in order to safely truncate range
  // tombstones.
  const std::vector<AtomicCompactionUnitBoundary>* compaction_boundaries_;
+
+  bool is_next_read_sequential_;
 };

 void LevelIterator::Seek(const Slice& target) {
@@ -1128,7 +1131,9 @@ bool LevelIterator::NextAndGetResult(IterateResult* result) {
  assert(Valid());
  bool is_valid = file_iter_.NextAndGetResult(result);
  if (!is_valid) {
+    is_next_read_sequential_ = true;
    SkipEmptyFileForward();
+    is_next_read_sequential_ = false;
    is_valid = Valid();
    if (is_valid) {
      result->key = key();
@@ -1195,6 +1200,12 @@ void LevelIterator::SetFileIterator(InternalIterator* iter) {
  }

  InternalIterator* old_iter = file_iter_.Set(iter);
+
+  // Update the read pattern for PrefetchBuffer.
+  if (is_next_read_sequential_) {
+    file_iter_.UpdateReadaheadState(old_iter);
+  }
+
  if (pinned_iters_mgr_ && pinned_iters_mgr_->PinningEnabled()) {
    pinned_iters_mgr_->PinIterator(old_iter);
  } else {
@@ -1260,7 +1271,6 @@ Status Version::GetTableProperties(std::shared_ptr<const TableProperties>* tp,
    return s;
  }

-  TableProperties* raw_table_properties;
  // By setting the magic number to kInvalidTableMagicNumber, we can by
  // pass the magic number check in the footer.
  std::unique_ptr<RandomAccessFileReader> file_reader(
@@ -1268,16 +1278,16 @@ Status Version::GetTableProperties(std::shared_ptr<const TableProperties>* tp,
          std::move(file), file_name, nullptr /* env */, io_tracer_,
          nullptr /* stats */, 0 /* hist_type */, nullptr /* file_read_hist */,
          nullptr /* rate_limiter */, ioptions->listeners));
+  std::unique_ptr<TableProperties> props;
  s = ReadTableProperties(
      file_reader.get(), file_meta->fd.GetFileSize(),
      Footer::kInvalidTableMagicNumber /* table's magic number */, *ioptions,
-      &raw_table_properties, false /* compression_type_missing */);
+      &props);
  if (!s.ok()) {
    return s;
  }
+  *tp = std::move(props);
  RecordTick(ioptions->stats, NUMBER_DIRECT_LOAD_TABLE_PROPERTIES);
-
-  *tp = std::shared_ptr<const TableProperties>(raw_table_properties);
  return s;
 }

@@ -1780,12 +1790,9 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset,
      io_tracer_(io_tracer) {}

 Status Version::GetBlob(const ReadOptions& read_options, const Slice& user_key,
-                        const Slice& blob_index_slice, PinnableSlice* value,
-                        uint64_t* bytes_read) const {
-  if (read_options.read_tier == kBlockCacheTier) {
-    return Status::Incomplete("Cannot read blob: no disk I/O allowed");
-  }
-
+                        const Slice& blob_index_slice,
+                        FilePrefetchBuffer* prefetch_buffer,
+                        PinnableSlice* value, uint64_t* bytes_read) const {
  BlobIndex blob_index;

  {
@@ -1795,14 +1802,20 @@ Status Version::GetBlob(const ReadOptions& read_options, const Slice& user_key,
    }
  }

-  return GetBlob(read_options, user_key, blob_index, value, bytes_read);
+  return GetBlob(read_options, user_key, blob_index, prefetch_buffer, value,
+                 bytes_read);
 }

 Status Version::GetBlob(const ReadOptions& read_options, const Slice& user_key,
-                        const BlobIndex& blob_index, PinnableSlice* value,
-                        uint64_t* bytes_read) const {
+                        const BlobIndex& blob_index,
+                        FilePrefetchBuffer* prefetch_buffer,
+                        PinnableSlice* value, uint64_t* bytes_read) const {
  assert(value);

+  if (read_options.read_tier == kBlockCacheTier) {
+    return Status::Incomplete("Cannot read blob: no disk I/O allowed");
+  }
+
  if (blob_index.HasTTL() || blob_index.IsInlined()) {
    return Status::Corruption("Unexpected TTL/inlined blob index");
  }
@@ -1830,7 +1843,7 @@ Status Version::GetBlob(const ReadOptions& read_options, const Slice& user_key,
  assert(blob_file_reader.GetValue());
  const Status s = blob_file_reader.GetValue()->GetBlob(
      read_options, user_key, blob_index.offset(), blob_index.size(),
-      blob_index.compression(), value, bytes_read);
+      blob_index.compression(), prefetch_buffer, value, bytes_read);

  return s;
 }
@@ -2057,10 +2070,11 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,

        if (is_blob_index) {
          if (do_merge && value) {
+            constexpr FilePrefetchBuffer* prefetch_buffer = nullptr;
            constexpr uint64_t* bytes_read = nullptr;

-            *status =
-                GetBlob(read_options, user_key, *value, value, bytes_read);
+            *status = GetBlob(read_options, user_key, *value, prefetch_buffer,
+                              value, bytes_read);
            if (!status->ok()) {
              if (status->IsIncomplete()) {
                get_context.MarkKeyMayExist();
@@ -3044,8 +3058,7 @@ void VersionStorageInfo::AddBlobFile(
  auto it = blob_files_.lower_bound(blob_file_number);
  assert(it == blob_files_.end() || it->first != blob_file_number);

-  blob_files_.insert(
-      it, BlobFiles::value_type(blob_file_number, std::move(blob_file_meta)));
+  blob_files_.emplace_hint(it, blob_file_number, std::move(blob_file_meta));
 }

 // Version::PrepareApply() need to be called before calling the function, or
@@ -5377,7 +5390,8 @@ Status VersionSet::WriteCurrentStateToManifest(
                       f->fd.smallest_seqno, f->fd.largest_seqno,
                       f->marked_for_compaction, f->oldest_blob_file_number,
                       f->oldest_ancester_time, f->file_creation_time,
-                       f->file_checksum, f->file_checksum_func_name);
+                       f->file_checksum, f->file_checksum_func_name,
+                       f->min_timestamp, f->max_timestamp);
        }
      }

@@ -723,13 +723,15 @@ class Version {
  // saves it in *value.
  // REQUIRES: blob_index_slice stores an encoded blob reference
  Status GetBlob(const ReadOptions& read_options, const Slice& user_key,
-                 const Slice& blob_index_slice, PinnableSlice* value,
+                 const Slice& blob_index_slice,
+                 FilePrefetchBuffer* prefetch_buffer, PinnableSlice* value,
                 uint64_t* bytes_read) const;

  // Retrieves a blob using a blob reference and saves it in *value,
  // assuming the corresponding blob file is part of this Version.
  Status GetBlob(const ReadOptions& read_options, const Slice& user_key,
-                 const BlobIndex& blob_index, PinnableSlice* value,
+                 const BlobIndex& blob_index,
+                 FilePrefetchBuffer* prefetch_buffer, PinnableSlice* value,
                 uint64_t* bytes_read) const;

  using BlobReadRequest =
@@ -804,6 +806,7 @@ class Version {
  int TEST_refs() const { return refs_; }

  VersionStorageInfo* storage_info() { return &storage_info_; }
+  const VersionStorageInfo* storage_info() const { return &storage_info_; }

  VersionSet* version_set() { return vset_; }

@@ -46,7 +46,8 @@ class GenerateLevelFilesBriefTest : public testing::Test {
        InternalKey(largest, largest_seq, kTypeValue), smallest_seq,
        largest_seq, /* marked_for_compact */ false, kInvalidBlobFileNumber,
        kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-        kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+        kUnknownFileChecksum, kUnknownFileChecksumFuncName,
+        kDisableUserTimestamp, kDisableUserTimestamp);
    files_.push_back(f);
  }

@@ -154,7 +155,8 @@ class VersionStorageInfoTestBase : public testing::Test {
        /* largest_seq */ 0, /* marked_for_compact */ false,
        oldest_blob_file_number, kUnknownOldestAncesterTime,
        kUnknownFileCreationTime, kUnknownFileChecksum,
-        kUnknownFileChecksumFuncName);
+        kUnknownFileChecksumFuncName, kDisableUserTimestamp,
+        kDisableUserTimestamp);
    f->compensated_file_size = file_size;
    vstorage_.AddFile(level, f);
  }
@@ -2996,7 +2998,8 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
      ASSERT_NE(0, file_size);
      file_metas->emplace_back(file_num, /*file_path_id=*/0, file_size, ikey,
                               ikey, 0, 0, false, 0, 0, 0, kUnknownFileChecksum,
-                               kUnknownFileChecksumFuncName);
+                               kUnknownFileChecksumFuncName,
+                               kDisableUserTimestamp, kDisableUserTimestamp);
    }
  }

@@ -3051,7 +3054,8 @@ TEST_F(VersionSetTestMissingFiles, ManifestFarBehindSst) {
    FileMetaData meta =
        FileMetaData(file_num, /*file_path_id=*/0, /*file_size=*/12,
                     smallest_ikey, largest_ikey, 0, 0, false, 0, 0, 0,
-                     kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+                     kUnknownFileChecksum, kUnknownFileChecksumFuncName,
+                     kDisableUserTimestamp, kDisableUserTimestamp);
    added_files.emplace_back(0, meta);
  }
  WriteFileAdditionAndDeletionToManifest(
@@ -3106,7 +3110,8 @@ TEST_F(VersionSetTestMissingFiles, ManifestAheadofSst) {
    FileMetaData meta =
        FileMetaData(file_num, /*file_path_id=*/0, /*file_size=*/12,
                     smallest_ikey, largest_ikey, 0, 0, false, 0, 0, 0,
-                     kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+                     kUnknownFileChecksum, kUnknownFileChecksumFuncName,
+                     kDisableUserTimestamp, kDisableUserTimestamp);
    added_files.emplace_back(0, meta);
  }
  WriteFileAdditionAndDeletionToManifest(
@@ -253,6 +253,7 @@ DECLARE_string(blob_compression_type);
 DECLARE_bool(enable_blob_garbage_collection);
 DECLARE_double(blob_garbage_collection_age_cutoff);
 DECLARE_double(blob_garbage_collection_force_threshold);
+DECLARE_uint64(blob_compaction_readahead_size);

 DECLARE_int32(approximate_size_one_in);
 DECLARE_bool(sync_fault_injection);
@@ -266,6 +267,9 @@ DECLARE_uint64(batch_protection_bytes_per_key);

 DECLARE_uint64(user_timestamp_size);
 DECLARE_string(secondary_cache_uri);
+DECLARE_int32(secondary_cache_fault_one_in);
+
+DECLARE_int32(prepopulate_block_cache);

 constexpr long KB = 1024;
 constexpr int kRandomValueMaxFactor = 3;
@@ -405,6 +405,11 @@ DEFINE_double(blob_garbage_collection_force_threshold,
              "[Integrated BlobDB] The threshold for the ratio of garbage in "
              "the oldest blob files for forcing garbage collection.");

+DEFINE_uint64(blob_compaction_readahead_size,
+              ROCKSDB_NAMESPACE::AdvancedColumnFamilyOptions()
+                  .blob_compaction_readahead_size,
+              "[Integrated BlobDB] Compaction readahead for blob files.");
+
 static const bool FLAGS_subcompactions_dummy __attribute__((__unused__)) =
    RegisterFlagValidator(&FLAGS_subcompactions, &ValidateUint32Range);

@@ -841,6 +846,9 @@ DEFINE_int32(open_metadata_write_fault_one_in, 0,
 #ifndef ROCKSDB_LITE
 DEFINE_string(secondary_cache_uri, "",
              "Full URI for creating a customized secondary cache object");
+DEFINE_int32(secondary_cache_fault_one_in, 0,
+             "On non-zero, enables fault injection in secondary cache inserts"
+             " and lookups");
 #endif  // ROCKSDB_LITE
 DEFINE_int32(open_write_fault_one_in, 0,
             "On non-zero, enables fault injection on file writes "
@@ -852,5 +860,10 @@ DEFINE_int32(injest_error_severity, 1,
             "The severity of the injested IO Error. 1 is soft error (e.g. "
             "retryable error), 2 is fatal error, and the default is "
             "retryable error.");
+DEFINE_int32(prepopulate_block_cache,
+             static_cast<int32_t>(ROCKSDB_NAMESPACE::BlockBasedTableOptions::
+                                      PrepopulateBlockCache::kDisable),
+             "Options related to cache warming (see `enum "
+             "PrepopulateBlockCache` in table.h)");

 #endif  // GFLAGS
@@ -22,6 +22,7 @@
 #include "util/cast_util.h"
 #include "utilities/backupable/backupable_db_impl.h"
 #include "utilities/fault_injection_fs.h"
+#include "utilities/fault_injection_secondary_cache.h"

 namespace ROCKSDB_NAMESPACE {

@@ -148,6 +149,10 @@ std::shared_ptr<Cache> StressTest::NewCache(size_t capacity,
                FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str());
        exit(1);
      }
+      if (FLAGS_secondary_cache_fault_one_in > 0) {
+        secondary_cache = std::make_shared<FaultInjectionSecondaryCache>(
+            secondary_cache, FLAGS_seed, FLAGS_secondary_cache_fault_one_in);
+      }
      opts.secondary_cache = secondary_cache;
    }
 #endif
@@ -264,6 +269,8 @@ bool StressTest::BuildOptionsTable() {
        std::vector<std::string>{"0.0", "0.25", "0.5", "0.75", "1.0"});
    options_tbl.emplace("blob_garbage_collection_force_threshold",
                        std::vector<std::string>{"0.5", "0.75", "1.0"});
+    options_tbl.emplace("blob_compaction_readahead_size",
+                        std::vector<std::string>{"0", "1M", "4M"});
  }

  options_table_ = std::move(options_tbl);
@@ -2219,6 +2226,9 @@ void StressTest::Open() {
        FLAGS_optimize_filters_for_memory;
    block_based_options.index_type =
        static_cast<BlockBasedTableOptions::IndexType>(FLAGS_index_type);
+    block_based_options.prepopulate_block_cache =
+        static_cast<BlockBasedTableOptions::PrepopulateBlockCache>(
+            FLAGS_prepopulate_block_cache);
    options_.table_factory.reset(
        NewBlockBasedTableFactory(block_based_options));
    options_.db_write_buffer_size = FLAGS_db_write_buffer_size;
@@ -2318,6 +2328,8 @@ void StressTest::Open() {
        FLAGS_blob_garbage_collection_age_cutoff;
    options_.blob_garbage_collection_force_threshold =
        FLAGS_blob_garbage_collection_force_threshold;
+    options_.blob_compaction_readahead_size =
+        FLAGS_blob_compaction_readahead_size;
  } else {
 #ifdef ROCKSDB_LITE
    fprintf(stderr, "--options_file not supported in lite mode\n");
@@ -2417,21 +2429,18 @@ void StressTest::Open() {
    exit(1);
  }

-  if (options_.enable_blob_files) {
-    fprintf(stdout,
-            "Integrated BlobDB: blob files enabled, min blob size %" PRIu64
-            ", blob file size %" PRIu64 ", blob compression type %s\n",
-            options_.min_blob_size, options_.blob_file_size,
-            CompressionTypeToString(options_.blob_compression_type).c_str());
-  }
-
-  if (options_.enable_blob_garbage_collection) {
-    fprintf(
-        stdout,
-        "Integrated BlobDB: blob GC enabled, cutoff %f, force threshold %f\n",
-        options_.blob_garbage_collection_age_cutoff,
-        options_.blob_garbage_collection_force_threshold);
-  }
+  fprintf(stdout,
+          "Integrated BlobDB: blob files enabled %d, min blob size %" PRIu64
+          ", blob file size %" PRIu64
+          ", blob compression type %s, blob GC enabled %d, cutoff %f, force "
+          "threshold %f, blob compaction readahead size %" PRIu64 "\n",
+          options_.enable_blob_files, options_.min_blob_size,
+          options_.blob_file_size,
+          CompressionTypeToString(options_.blob_compression_type).c_str(),
+          options_.enable_blob_garbage_collection,
+          options_.blob_garbage_collection_age_cutoff,
+          options_.blob_garbage_collection_force_threshold,
+          options_.blob_compaction_readahead_size);

  fprintf(stdout, "DB path: [%s]\n", FLAGS_db.c_str());

@@ -37,6 +37,8 @@
 #include <sys/uio.h>
 #endif
 #include <time.h>
+#include <unistd.h>
+
 #include <algorithm>
 // Get nano time includes
 #if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_GNU_KFREEBSD)
@@ -302,19 +304,22 @@ class PosixEnv : public CompositeEnv {
    return thread_status_updater_->GetThreadList(thread_list);
  }

-  static uint64_t gettid(pthread_t tid) {
+  uint64_t GetThreadID() const override {
    uint64_t thread_id = 0;
+#if defined(_GNU_SOURCE) && defined(__GLIBC_PREREQ)
+#if __GLIBC_PREREQ(2, 30)
+    thread_id = ::gettid();
+#else   // __GLIBC_PREREQ(2, 30)
+    pthread_t tid = pthread_self();
    memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid)));
+#endif  // __GLIBC_PREREQ(2, 30)
+#else   // defined(_GNU_SOURCE) && defined(__GLIBC_PREREQ)
+    pthread_t tid = pthread_self();
+    memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid)));
+#endif  // defined(_GNU_SOURCE) && defined(__GLIBC_PREREQ)
    return thread_id;
  }

-  static uint64_t gettid() {
-    pthread_t tid = pthread_self();
-    return gettid(tid);
-  }
-
-  uint64_t GetThreadID() const override { return gettid(pthread_self()); }
-
  Status GetHostName(char* name, uint64_t len) override {
    int ret = gethostname(name, static_cast<size_t>(len));
    if (ret < 0) {
@@ -1543,7 +1543,8 @@ PosixDirectory::PosixDirectory(int fd) : fd_(fd) {
 #ifdef OS_LINUX
  struct statfs buf;
  int ret = fstatfs(fd, &buf);
-  is_btrfs_ = (ret == 0 && buf.f_type == BTRFS_SUPER_MAGIC);
+  is_btrfs_ = (ret == 0 && buf.f_type == static_cast<decltype(buf.f_type)>(
+                                             BTRFS_SUPER_MAGIC));
 #endif
 }

@@ -108,6 +108,7 @@ Status FilePrefetchBuffer::Prefetch(const IOOptions& opts,
 }

 bool FilePrefetchBuffer::TryReadFromCache(const IOOptions& opts,
+                                          RandomAccessFileReader* reader,
                                          uint64_t offset, size_t n,
                                          Slice* result, Status* status,
                                          bool for_compaction) {
@@ -122,13 +123,15 @@ bool FilePrefetchBuffer::TryReadFromCache(const IOOptions& opts,
  //    If readahead is enabled: prefetch the remaining bytes + readahead bytes
  //        and satisfy the request.
  //    If readahead is not enabled: return false.
+  TEST_SYNC_POINT_CALLBACK("FilePrefetchBuffer::TryReadFromCache",
+                           &readahead_size_);
  if (offset + n > buffer_offset_ + buffer_.CurrentSize()) {
    if (readahead_size_ > 0) {
-      assert(file_reader_ != nullptr);
+      assert(reader != nullptr);
      assert(max_readahead_size_ >= readahead_size_);
      Status s;
      if (for_compaction) {
-        s = Prefetch(opts, file_reader_, offset, std::max(n, readahead_size_),
+        s = Prefetch(opts, reader, offset, std::max(n, readahead_size_),
                     for_compaction);
      } else {
        if (implicit_auto_readahead_) {
@@ -149,8 +152,7 @@ bool FilePrefetchBuffer::TryReadFromCache(const IOOptions& opts,
            return false;
          }
        }
-        s = Prefetch(opts, file_reader_, offset, n + readahead_size_,
-                     for_compaction);
+        s = Prefetch(opts, reader, offset, n + readahead_size_, for_compaction);
      }
      if (!s.ok()) {
        if (status) {
@@ -8,11 +8,12 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

 #pragma once
+#include <algorithm>
 #include <atomic>
 #include <sstream>
 #include <string>

-#include "file/random_access_file_reader.h"
+#include "file/readahead_file_info.h"
 #include "port/port.h"
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
@@ -20,14 +21,20 @@

 namespace ROCKSDB_NAMESPACE {

+#define DEAFULT_DECREMENT 8 * 1024
+
+struct IOOptions;
+class RandomAccessFileReader;
+
 // FilePrefetchBuffer is a smart buffer to store and read data from a file.
 class FilePrefetchBuffer {
 public:
  static const int kMinNumFileReadsToStartAutoReadahead = 2;
+  static const size_t kInitAutoReadaheadSize = 8 * 1024;
+
  // Constructor.
  //
  // All arguments are optional.
-  // file_reader        : the file reader to use. Can be a nullptr.
  // readahead_size     : the initial readahead size.
  // max_readahead_size : the maximum readahead size.
  //   If max_readahead_size > readahead_size, the readahead size will be
@@ -42,21 +49,16 @@ class FilePrefetchBuffer {
  // implicit_auto_readahead : Readahead is enabled implicitly by rocksdb after
  //   doing sequential scans for two times.
  //
-  // Automatic readhead is enabled for a file if file_reader, readahead_size,
+  // Automatic readhead is enabled for a file if readahead_size
  // and max_readahead_size are passed in.
-  // If file_reader is a nullptr, setting readahead_size and max_readahead_size
-  // does not make any sense. So it does nothing.
  // A user can construct a FilePrefetchBuffer without any arguments, but use
  // `Prefetch` to load data into the buffer.
-  FilePrefetchBuffer(RandomAccessFileReader* file_reader = nullptr,
-                     size_t readahead_size = 0, size_t max_readahead_size = 0,
+  FilePrefetchBuffer(size_t readahead_size = 0, size_t max_readahead_size = 0,
                     bool enable = true, bool track_min_offset = false,
                     bool implicit_auto_readahead = false)
      : buffer_offset_(0),
-        file_reader_(file_reader),
        readahead_size_(readahead_size),
        max_readahead_size_(max_readahead_size),
-        initial_readahead_size_(readahead_size),
        min_offset_read_(port::kMaxSizet),
        enable_(enable),
        track_min_offset_(track_min_offset),
@@ -73,24 +75,36 @@ class FilePrefetchBuffer {
  Status Prefetch(const IOOptions& opts, RandomAccessFileReader* reader,
                  uint64_t offset, size_t n, bool for_compaction = false);

-  // Tries returning the data for a file raed from this buffer, if that data is
+  // Tries returning the data for a file read from this buffer if that data is
  // in the buffer.
  // It handles tracking the minimum read offset if track_min_offset = true.
  // It also does the exponential readahead when readahead_size is set as part
  // of the constructor.
  //
-  // offset : the file offset.
-  // n      : the number of bytes.
-  // result : output buffer to put the data into.
-  // for_compaction : if cache read is done for compaction read.
-  bool TryReadFromCache(const IOOptions& opts, uint64_t offset, size_t n,
-                        Slice* result, Status* s, bool for_compaction = false);
+  // opts           : the IO options to use.
+  // reader         : the file reader.
+  // offset         : the file offset.
+  // n              : the number of bytes.
+  // result         : output buffer to put the data into.
+  // s              : output status.
+  // for_compaction : true if cache read is done for compaction read.
+  bool TryReadFromCache(const IOOptions& opts, RandomAccessFileReader* reader,
+                        uint64_t offset, size_t n, Slice* result, Status* s,
+                        bool for_compaction = false);

  // The minimum `offset` ever passed to TryReadFromCache(). This will nly be
  // tracked if track_min_offset = true.
  size_t min_offset_read() const { return min_offset_read_; }

-  void UpdateReadPattern(const size_t& offset, const size_t& len) {
+  // Called in case of implicit auto prefetching.
+  void UpdateReadPattern(const uint64_t& offset, const size_t& len,
+                         bool is_adaptive_readahead = false) {
+    if (is_adaptive_readahead) {
+      // Since this block was eligible for prefetch but it was found in
+      // cache, so check and decrease the readahead_size by 8KB (default)
+      // if eligible.
+      DecreaseReadAheadIfEligible(offset, len);
+    }
    prev_offset_ = offset;
    prev_len_ = len;
  }
@@ -99,18 +113,47 @@ class FilePrefetchBuffer {
    return (prev_len_ == 0 || (prev_offset_ + prev_len_ == offset));
  }

+  // Called in case of implicit auto prefetching.
  void ResetValues() {
    num_file_reads_ = 1;
-    readahead_size_ = initial_readahead_size_;
+    readahead_size_ = kInitAutoReadaheadSize;
+  }
+
+  void GetReadaheadState(ReadaheadFileInfo::ReadaheadInfo* readahead_info) {
+    readahead_info->readahead_size = readahead_size_;
+    readahead_info->num_file_reads = num_file_reads_;
+  }
+
+  void DecreaseReadAheadIfEligible(uint64_t offset, size_t size,
+                                   size_t value = DEAFULT_DECREMENT) {
+    // Decrease the readahead_size if
+    // - its enabled internally by RocksDB (implicit_auto_readahead_) and,
+    // - readahead_size is greater than 0 and,
+    // - this block would have called prefetch API if not found in cache for
+    //   which conditions are:
+    //   - few/no bytes are in buffer and,
+    //   - block is sequential with the previous read and,
+    //   - num_file_reads_ + 1 (including this read) >
+    //   kMinNumFileReadsToStartAutoReadahead
+    if (implicit_auto_readahead_ && readahead_size_ > 0) {
+      if ((offset + size > buffer_offset_ + buffer_.CurrentSize()) &&
+          IsBlockSequential(offset) &&
+          (num_file_reads_ + 1 > kMinNumFileReadsToStartAutoReadahead)) {
+        size_t initial_auto_readahead_size = kInitAutoReadaheadSize;
+        readahead_size_ =
+            std::max(initial_auto_readahead_size,
+                     (readahead_size_ >= value ? readahead_size_ - value : 0));
+      }
+    }
  }

 private:
  AlignedBuffer buffer_;
  uint64_t buffer_offset_;
-  RandomAccessFileReader* file_reader_;
  size_t readahead_size_;
+  // FilePrefetchBuffer object won't be created from Iterator flow if
+  // max_readahead_size_ = 0.
  size_t max_readahead_size_;
-  size_t initial_readahead_size_;
  // The minimum `offset` ever passed to TryReadFromCache().
  size_t min_offset_read_;
  // if false, TryReadFromCache() always return false, and we only take stats
@@ -120,11 +163,11 @@ class FilePrefetchBuffer {
  // can be fetched from min_offset_read().
  bool track_min_offset_;

-  // implicit_auto_readahead is enabled by rocksdb internally after 2 sequential
-  // IOs.
+  // implicit_auto_readahead is enabled by rocksdb internally after 2
+  // sequential IOs.
  bool implicit_auto_readahead_;
-  size_t prev_offset_;
+  uint64_t prev_offset_;
  size_t prev_len_;
-  int num_file_reads_;
+  int64_t num_file_reads_;
 };
 }  // namespace ROCKSDB_NAMESPACE
@@ -183,9 +183,9 @@ IOStatus GenerateOneFileChecksum(
                              ? verify_checksums_readahead_size
                              : default_max_read_ahead_size;

-  FilePrefetchBuffer prefetch_buffer(
-      reader.get(), readahead_size /* readahead_size */,
-      readahead_size /* max_readahead_size */, !allow_mmap_reads /* enable */);
+  FilePrefetchBuffer prefetch_buffer(readahead_size /* readahead_size */,
+                                     readahead_size /* max_readahead_size */,
+                                     !allow_mmap_reads /* enable */);

  Slice slice;
  uint64_t offset = 0;
@@ -193,8 +193,9 @@ IOStatus GenerateOneFileChecksum(
  while (size > 0) {
    size_t bytes_to_read =
        static_cast<size_t>(std::min(uint64_t{readahead_size}, size));
-    if (!prefetch_buffer.TryReadFromCache(opts, offset, bytes_to_read, &slice,
-                                          nullptr, false)) {
+    if (!prefetch_buffer.TryReadFromCache(
+            opts, reader.get(), offset, bytes_to_read, &slice,
+            nullptr /* status */, false /* for_compaction */)) {
      return IOStatus::Corruption("file read failed");
    }
    if (slice.size() == 0) {