Merge pull request #10609 from facebook/revert-10606-7.6.1

Revert "Patch 7.6.1"

.circleci/config.yml

@@ -3,11 +3,6 @@ version: 2.1
 orbs:
   win: circleci/windows@2.4.0
 
-aliases:
-  - &notify-on-main-failure
-    fail_only: true
-    only_for_branches: main
-
 commands:
   install-cmake-on-macos:
     steps:
@@ -61,35 +56,21 @@ commands:
       - store_artifacts: # store LOG for debugging if there's any
           path: LOG
       - run: # on fail, compress Test Logs for diagnosing the issue
-         name: Compress Test Logs
-         command: tar -cvzf t.tar.gz t
-         when: on_fail
+          name: Compress Test Logs
+          command: tar -cvzf t.tar.gz t
+          when: on_fail
       - store_artifacts: # on fail, store Test Logs for diagnosing the issue
           path: t.tar.gz
           destination: test_logs
           when: on_fail
-
-  install-clang-10:
-    steps:
-      - run:
-          name: Install Clang 10
+      - run: # store core dumps if there's any
           command: |
-            wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
-            echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list
-            echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list
-            echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
-            sudo apt-get update -y && sudo apt-get install -y clang-10
-
-  install-clang-13:
-    steps:
-      - run:
-          name: Install Clang 13
-          command: |
-            echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
-            echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
-            echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
-            wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-            sudo apt-get update -y && sudo apt-get install -y clang-13
+            mkdir -p /tmp/core_dumps
+            cp core.* /tmp/core_dumps
+          when: on_fail
+      - store_artifacts:
+          path: /tmp/core_dumps
+          when: on_fail
 
   install-gflags:
     steps:
@@ -98,33 +79,6 @@ commands:
           command: |
             sudo apt-get update -y && sudo apt-get install -y libgflags-dev
 
-  install-benchmark:
-    steps:
-      - run:
-          name: Install ninja build
-          command: sudo apt-get update -y && sudo apt-get install -y ninja-build
-      - run:
-          name: Install benchmark
-          command: |
-            git clone --depth 1 --branch v1.6.1 https://github.com/google/benchmark.git ~/benchmark
-            cd ~/benchmark && mkdir build && cd build
-            cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0
-            ninja && sudo ninja install
-
-  install-valgrind:
-    steps:
-      - run:
-          name: Install valgrind
-          command: sudo apt-get update -y && sudo apt-get install -y valgrind
-
-  upgrade-cmake:
-    steps:
-      - run:
-          name: Upgrade cmake
-          command: |
-            sudo apt remove --purge cmake
-            sudo snap install cmake --classic
-
   install-gflags-on-macos:
     steps:
       - run:
@@ -132,48 +86,8 @@ commands:
           command: |
             HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
 
-  install-gtest-parallel:
-    steps:
-      - run:
-          name: Install gtest-parallel
-          command: |
-            git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
-            echo 'export PATH=$HOME/gtest-parallel:$PATH' >> $BASH_ENV
-
-  install-compression-libs:
-    steps:
-      - run:
-          name: Install compression libs
-          command: |
-            sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
-
-  install-streaming-compress-libs:
-    steps:
-      - run:
-          name: Install streaming compression libs
-          command: |
-            sudo apt-get update -y && sudo apt-get install -y libzstd-dev
-
-  install-libprotobuf-mutator:
-    steps:
-      - run:
-          name: Install libprotobuf-mutator libs
-          command: |
-            git clone -b v1.0 git@github.com:google/libprotobuf-mutator.git ~/libprotobuf-mutator
-            cd ~/libprotobuf-mutator && git checkout ffd86a32874e5c08a143019aad1aaf0907294c9f && mkdir build && cd build
-            cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON
-            ninja && sudo ninja install
-      - run:
-          name: Setup environment variables
-          command: |
-            echo "export PKG_CONFIG_PATH=/usr/local/OFF/:~/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/" >> $BASH_ENV
-            echo "export PROTOC_BIN=~/libprotobuf-mutator/build/external.protobuf/bin/protoc" >> $BASH_ENV
   setup-folly:
     steps:
-      - run:
-          name: Install folly dependencies
-          command: |
-            sudo apt-get install libgoogle-glog-dev
       - run:
           name: Checkout folly sources
           command: |
@@ -190,14 +104,17 @@ commands:
   perform-benchmarks:
     steps:
       - run:
-          name: "Run basic benchmark, 5 min"
-          command: ./tools/benchmark.sh fillseq_enable_wal
+          name: "Test low-variance benchmarks"
+          command: ./tools/benchmark_ci.py --db_dir /tmp/rocksdb-benchmark-datadir --output_dir /tmp/benchmark-results --num_keys 10000000
           environment:
             LD_LIBRARY_PATH: /usr/local/lib
-            DB_DIR: /tmp/rocksdb-benchmark-datadir
-            WAL_DIR: /tmp/rocksdb-benchmark-waldir
-            OUTPUT_DIR: /tmp/benchmark-results
-            NUM_KEYS: 10000000
+            # How long to run parts of the test(s)
+            DURATION_RO: 450
+            DURATION_RW: 450
+            # The benchmark host has 32GB memory
+            # The folllowing values are tailored to work with that
+            # Note, tests may not exercise the targeted issues if the memory is increased on new test hosts.
+
 
   post-benchmarks:
     steps:
@@ -209,7 +126,7 @@ commands:
           command: |
             set +e
             set +o pipefail
-            ./build_tools/benchmark_log_tool.py --tsvfile /tmp/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3/_doc
+            ./build_tools/benchmark_log_tool.py --tsvfile /tmp/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc
             true
 
 executors:
@@ -218,6 +135,21 @@ executors:
       image: 'windows-server-2019-vs2019:stable'
       resource_class: windows.2xlarge
       shell: bash.exe
+  linux-docker:
+    docker:
+      # The image configuration is build_tools/ubuntu20_image/Dockerfile
+      # To update and build the image:
+      #  $ cd build_tools/ubuntu20_image
+      #  $ docker build -t zjay437/rocksdb:0.5 .
+      #  $ docker push zjay437/rocksdb:0.5
+      # `zjay437` is the account name for zjay@meta.com which readwrite token is shared internally. To login:
+      #  $ docker login --username zjay437
+      # Or please feel free to change it to your docker hub account for hosting the image, meta employee should already have the account and able to login with SSO.
+      # To avoid impacting the existing CI runs, please bump the version every time creating a new image
+      # to run the CI image environment locally:
+      #  $ docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined -it zjay437/rocksdb:0.5 bash
+      # option `--cap-add=SYS_PTRACE --security-opt seccomp=unconfined` is used to enable gdb to attach an existing process
+      - image: zjay437/rocksdb:0.5
 
 jobs:
   build-macos:
@@ -269,172 +201,132 @@ jobs:
       - post-steps
 
   build-linux:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-gflags
-      - install-streaming-compress-libs
       - run: make V=1 J=32 -j32 check
       - post-steps
 
   build-linux-encrypted_env-no_compression:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-gflags
       - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check
       - run: |
           ./sst_dump --help | egrep -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression
       - post-steps
 
   build-linux-shared_lib-alt_namespace-status_checked:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-gflags
       - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 check
       - post-steps
 
   build-linux-release:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - checkout # check out the code in the project directory
       - run: make V=1 -j32 release
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - install-gflags
-      - run: make V=1 -j32 release
       - run: ./db_stress --version # ensure with gflags
+      - run: make clean
+      - run: apt-get remove -y libgflags-dev
+      - run: make V=1 -j32 release
+      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
       - post-steps
 
   build-linux-release-rtti:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: xlarge
     steps:
       - checkout # check out the code in the project directory
-      - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
-      - run: make clean
       - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
       - run: ./db_stress --version # ensure with gflags
+      - run: make clean
+      - run: apt-get remove -y libgflags-dev
+      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
+      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
 
   build-linux-lite:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: large
     steps:
       - pre-steps
-      - install-gflags
       - run: LITE=1 make V=1 J=8 -j8 check
       - post-steps
 
   build-linux-lite-release:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: large
     steps:
       - checkout # check out the code in the project directory
       - run: LITE=1 make V=1 -j8 release
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - install-gflags
-      - run: LITE=1 make V=1 -j8 release
       - run: ./db_stress --version # ensure with gflags
+      - run: make clean
+      - run: apt-get remove -y libgflags-dev
+      - run: LITE=1 make V=1 -j8 release
+      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
       - post-steps
 
   build-linux-clang-no_test_run:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: xlarge
     steps:
       - checkout # check out the code in the project directory
-      - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev libtbb-dev
       - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all
       - post-steps
 
   build-linux-clang10-asan:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-gflags
-      - install-clang-10
       - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out
       - post-steps
 
   build-linux-clang10-mini-tsan:
-    machine:
-      image: ubuntu-2004:202111-02
-    resource_class: 2xlarge
-    # find test list by `make list_all_tests`
-    parameters:
-      start_test:
-        default: ""
-        type: string
-      end_test:
-        default: ""
-        type: string
+    executor: linux-docker
+    resource_class: 2xlarge+
     steps:
       - pre-steps
-      - install-gflags
-      - install-clang-10
-      - install-gtest-parallel
-      - run:
-          name: "Build unit tests"
-          command: |
-            echo "env: $(env)"
-            ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 --output-sync=target build_subset_tests
-      - run:
-          name: "Run unit tests in parallel"
-          command: |
-            sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
-            cat /tmp/test_list
-            gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat  # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.
+      - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check
       - post-steps
 
   build-linux-clang10-ubsan:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-gflags
-      - install-clang-10
       - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check # aligned new doesn't work for reason we haven't figured out
       - post-steps
 
   build-linux-valgrind:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-gflags
-      - install-valgrind
       - run: PORTABLE=1 make V=1 -j32 valgrind_test
       - post-steps
 
   build-linux-clang10-clang-analyze:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: sudo apt-get update -y && sudo apt-get install -y clang-tools-10
       - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
       - post-steps
+      - run:
+          name: "compress test report"
+          command: tar -cvzf scan_build_report.tar.gz scan_build_report
+          when: on_fail
+      - store_artifacts:
+          path: scan_build_report.tar.gz
+          destination: scan_build_report
+          when: on_fail
 
   build-linux-runner:
     machine: true
@@ -449,26 +341,20 @@ jobs:
       - post-steps
 
   build-linux-cmake-with-folly:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-gflags
-      - upgrade-cmake
       - setup-folly
       - run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20)
       - post-steps
 
   build-linux-cmake-with-benchmark:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-gflags
-      - install-benchmark
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20)
+      - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20
       - post-steps
 
   build-linux-unity-and-headers:
@@ -485,93 +371,95 @@ jobs:
       - post-steps
 
   build-linux-gcc-7-with-folly:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get update -y && sudo apt-get install gcc-7 g++-7 libgflags-dev
       - setup-folly
       - run: USE_FOLLY=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 check
       - post-steps
 
   build-linux-gcc-8-no_test_run:
-    machine:
-      image: ubuntu-2004:202111-02
-    resource_class: xlarge
+    executor: linux-docker
+    resource_class: 2xlarge
     steps:
       - pre-steps
-      - run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get update -y && sudo apt-get install gcc-8 g++-8 libgflags-dev
-      - run: CC=gcc-8 CXX=g++-8 V=1 make -j16 all
+      - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all
       - post-steps
 
   build-linux-gcc-10-cxx20-no_test_run:
-    machine:
-      image: ubuntu-2004:202111-02
-    resource_class: xlarge
+    executor: linux-docker
+    resource_class: 2xlarge
     steps:
       - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-10 g++-10 libgflags-dev
-      - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all
+      - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all
       - post-steps
 
   build-linux-gcc-11-no_test_run:
-    machine:
-      image: ubuntu-2004:202111-02
-    resource_class: xlarge
+    executor: linux-docker
+    resource_class: 2xlarge
     steps:
       - pre-steps
-      - run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get update -y && sudo apt-get install gcc-11 g++-11 libgflags-dev
-      - install-benchmark
-      - run: CC=gcc-11 CXX=g++-11 V=1 make -j16 all microbench
+      - run: CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench
       - post-steps
 
   build-linux-clang-13-no_test_run:
-    machine:
-      image: ubuntu-2004:202111-02
-    resource_class: xlarge
+    executor: linux-docker
+    resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-clang-13
-      - install-benchmark
-      - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j16 all microbench
+      - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench
       - post-steps
 
   # Ensure ASAN+UBSAN with folly, and full testsuite with clang 13
   build-linux-clang-13-asan-ubsan-with-folly:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-clang-13
-      - install-gflags
       - setup-folly
       - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check
       - post-steps
 
   # This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing.
   build-linux-run-microbench:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-benchmark
       - run: DEBUG_LEVEL=0 make -j32 run_microbench
       - post-steps
 
   build-linux-mini-crashtest:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: large
     steps:
       - pre-steps
-      - install-gflags
-      - install-compression-libs
       - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS=--duration=960 blackbox_crash_test_with_atomic_flush
       - post-steps
 
+  build-linux-crashtest-tiered-storage-bb:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run:
+          name: "run crashtest"
+          command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS=--duration=10800 blackbox_crash_test_with_tiered_storage
+          no_output_timeout: 100m
+      - post-steps
+
+  build-linux-crashtest-tiered-storage-wb:
+    executor: linux-docker
+    resource_class: 2xlarge
+    steps:
+      - pre-steps
+      - run:
+          name: "run crashtest"
+          command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS=--duration=10800 whitebox_crash_test_with_tiered_storage
+          no_output_timeout: 100m
+      - post-steps
+
   build-windows:
     executor: windows-2xlarge
     parameters:
@@ -641,14 +529,10 @@ jobs:
             build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
 
   build-linux-java:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: large
-    environment:
-      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
     steps:
       - pre-steps
-      - install-gflags
       - run:
           name: "Set Java Environment"
           command: |
@@ -662,14 +546,10 @@ jobs:
       - post-steps
 
   build-linux-java-static:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: large
-    environment:
-      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
     steps:
       - pre-steps
-      - install-gflags
       - run:
           name: "Set Java Environment"
           command: |
@@ -755,34 +635,25 @@ jobs:
       - post-steps
 
   build-examples:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: large
     steps:
       - pre-steps
-      - install-gflags
       - run:
           name: "Build examples"
           command: |
-            OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4
+            make V=1 -j4 static_lib && cd examples && make V=1 -j4
       - post-steps
 
   build-cmake-mingw:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: large
     steps:
       - pre-steps
-      - install-gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y mingw-w64
-      - run: sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
+      - run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
       - run:
           name: "Build cmake-mingw"
           command: |
-            sudo apt-get install snapd && sudo snap install cmake --beta --classic
-            export PATH=/snap/bin:$PATH
-            sudo apt-get install -y openjdk-8-jdk
-            export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
             export PATH=$JAVA_HOME/bin:$PATH
             echo "JAVA_HOME=${JAVA_HOME}"
             which java && java -version
@@ -791,14 +662,12 @@ jobs:
       - post-steps
 
   build-linux-non-shm:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     environment:
       TEST_TMPDIR: /tmp/rocksdb_test_tmp
     steps:
       - pre-steps
-      - install-gflags
       - run: make V=1 -j32 check
       - post-steps
 
@@ -856,13 +725,10 @@ jobs:
       - post-steps
 
   build-format-compatible:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: 2xlarge
     steps:
       - pre-steps
-      - install-gflags
-      - install-compression-libs
       - run:
           name: "test"
           command: |
@@ -873,14 +739,10 @@ jobs:
       - post-steps
 
   build-fuzzers:
-    machine:
-      image: ubuntu-2004:202111-02
+    executor: linux-docker
     resource_class: large
     steps:
       - pre-steps
-      - install-clang-13
-      - run: sudo apt-get update -y && sudo apt-get install -y cmake ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool
-      - install-libprotobuf-mutator
       - run:
           name: "Build rocksdb lib"
           command: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
@@ -911,12 +773,7 @@ workflows:
     jobs:
       - build-linux-clang10-asan
       - build-linux-clang10-ubsan
-      - build-linux-clang10-mini-tsan:
-          start_test: ""
-          end_test: "env_test"
-      - build-linux-clang10-mini-tsan:
-          start_test: "env_test"
-          end_test: ""
+      - build-linux-clang10-mini-tsan
       - build-linux-shared_lib-alt_namespace-status_checked
   jobs-linux-no-test-run:
     jobs:

.travis.yml

@@ -1,280 +0,0 @@
-dist: xenial
-language: cpp
-os:
-  - linux
-arch:
-  - arm64
-  - ppc64le
-  - s390x
-compiler:
-  - clang
-  - gcc
-cache:
-  - ccache
-
-addons:
-  apt:
-    update: true
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      - libgflags-dev
-      - libbz2-dev
-      - liblz4-dev
-      - libsnappy-dev
-      - liblzma-dev  # xv
-      - libzstd-dev
-      - zlib1g-dev
-
-env:
-  - TEST_GROUP=platform_dependent # 16-18 minutes
-  - TEST_GROUP=1 # 33-35 minutes
-  - TEST_GROUP=2 # 18-20 minutes
-  - TEST_GROUP=3 # 20-22 minutes
-  - TEST_GROUP=4 # 12-14 minutes
-  # Run java tests
-  - JOB_NAME=java_test # 4-11 minutes
-  # Build ROCKSDB_LITE
-  - JOB_NAME=lite_build # 3-4 minutes
-  # Build examples
-  - JOB_NAME=examples # 5-7 minutes
-  - JOB_NAME=cmake # 3-5 minutes
-  - JOB_NAME=cmake-gcc8 # 3-5 minutes
-  - JOB_NAME=cmake-gcc9 # 3-5 minutes
-  - JOB_NAME=cmake-gcc9-c++20 # 3-5 minutes
-  - JOB_NAME=cmake-mingw # 3 minutes
-
-matrix:
-  exclude:
-  - os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-mingw
-  - os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-mingw
-  - os: linux
-    arch: s390x
-    env: JOB_NAME=cmake-mingw
-  - os: linux
-    compiler: clang
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: arm64
-    env: TEST_GROUP=platform_dependent
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: TEST_GROUP=1
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=1
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: s390x
-    env: TEST_GROUP=1
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: TEST_GROUP=2
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=2
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: s390x
-    env: TEST_GROUP=2
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: TEST_GROUP=3
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=3
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: s390x
-    env: TEST_GROUP=3
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: TEST_GROUP=4
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=4
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: s390x
-    env: TEST_GROUP=4
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake
-  - if: type = pull_request AND commit_message !~ /FULL_CI/ AND commit_message !~ /java/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=java_test
-  - if: type = pull_request AND commit_message !~ /FULL_CI/ AND commit_message !~ /java/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=java_test
-  - if: type = pull_request AND commit_message !~ /FULL_CI/ AND commit_message !~ /java/
-    os: linux
-    arch: s390x
-    env: JOB_NAME=java_test
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=lite_build
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=lite_build
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: s390x
-    env: JOB_NAME=lite_build
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=examples
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=examples
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: s390x
-    env: JOB_NAME=examples
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-gcc8
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-gcc8
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: s390x
-    env: JOB_NAME=cmake-gcc8
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-gcc9
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-gcc9
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: s390x
-    env: JOB_NAME=cmake-gcc9
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-gcc9-c++20
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-gcc9-c++20
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: s390x
-    env: JOB_NAME=cmake-gcc9-c++20
-
-install:
-  -  CC=gcc-7 && CXX=g++-7
-  - if [ "${JOB_NAME}" == cmake-gcc8 ]; then
-      sudo apt-get install -y g++-8 || exit $?;
-      CC=gcc-8 && CXX=g++-8;
-    fi
-  - if [ "${JOB_NAME}" == cmake-gcc9 ] || [ "${JOB_NAME}" == cmake-gcc9-c++20 ]; then
-      sudo apt-get install -y g++-9 || exit $?;
-      CC=gcc-9 && CXX=g++-9;
-    fi
-  - if [ "${JOB_NAME}" == cmake-mingw ]; then
-      sudo apt-get install -y mingw-w64 || exit $?;
-    fi
-  - if [ "${CXX}" == "g++-7" ]; then
-      sudo apt-get install -y g++-7 || exit $?;
-    fi
-  - |
-    if [[ "${JOB_NAME}" == cmake* ]]; then
-      sudo apt-get remove -y cmake cmake-data
-      export CMAKE_DEB="cmake-3.14.5-Linux-$(uname -m).deb"
-      export CMAKE_DEB_URL="https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/${CMAKE_DEB}"
-      curl --silent --fail --show-error --location --output "${CMAKE_DEB}" "${CMAKE_DEB_URL}" || exit $?
-      sudo dpkg -i "${CMAKE_DEB}" || exit $?
-      which cmake && cmake --version
-    fi
-  - |
-    if [[ "${JOB_NAME}" == java_test || "${JOB_NAME}" == cmake* ]]; then
-      # Ensure JDK 8
-      sudo apt-get install -y openjdk-8-jdk || exit $?
-      export PATH=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin:$PATH
-      export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)
-      echo "JAVA_HOME=${JAVA_HOME}"
-      which java && java -version
-      which javac && javac -version
-    fi
-
-before_script:
-  # Increase the maximum number of open file descriptors, since some tests use
-  # more FDs than the default limit.
-  - ulimit -n 8192
-
-script:
-  - date; ${CXX} --version
-  - if [ `command -v ccache` ]; then ccache -C; fi
-  - export MK_PARALLEL=4;
-    if [[ "$TRAVIS_CPU_ARCH" == s390x ]]; then
-      export MK_PARALLEL=1;
-    fi
-  - case $TEST_GROUP in
-    platform_dependent)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=only make -j$MK_PARALLEL all_but_some_tests check_some
-      ;;
-    1)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_END=backup_engine_test make -j$MK_PARALLEL check_some
-      ;;
-    2)
-      OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 make -j$MK_PARALLEL tools && OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=backup_engine_test ROCKSDBTESTS_END=db_universal_compaction_test make -j$MK_PARALLEL check_some
-      ;;
-    3)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=db_universal_compaction_test ROCKSDBTESTS_END=table_properties_collector_test make -j$MK_PARALLEL check_some
-      ;;
-    4)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=table_properties_collector_test make -j$MK_PARALLEL check_some
-      ;;
-    esac
-  - case $JOB_NAME in
-    java_test)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 make rocksdbjava jtest
-      ;;
-    lite_build)
-      OPT='-DTRAVIS -DROCKSDB_LITE' LIB_MODE=shared V=1 make -j$MK_PARALLEL all
-      ;;
-    examples)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 make -j$MK_PARALLEL static_lib && cd examples && make -j$MK_PARALLEL
-      ;;
-    cmake-mingw)
-      sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix;
-      mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
-      ;;
-    cmake*)
-      case $JOB_NAME in
-        *-c++20)
-          OPT=-DCMAKE_CXX_STANDARD=20
-          ;;
-      esac
-
-      mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=0 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. && make -j$MK_PARALLEL && cd  .. && rm -rf build && mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release $OPT && make -j$MK_PARALLEL rocksdb rocksdbjni
-      ;;
-    esac
-notifications:
-    email:
-      - leveldb@fb.com

CMakeLists.txt

@@ -598,6 +598,7 @@ set(SOURCES
         cache/cache_entry_roles.cc
         cache/cache_key.cc
         cache/cache_reservation_manager.cc
+        cache/charged_cache.cc
         cache/clock_cache.cc
         cache/compressed_secondary_cache.cc
         cache/fast_lru_cache.cc
@@ -670,6 +671,7 @@ set(SOURCES
         db/range_del_aggregator.cc
         db/range_tombstone_fragmenter.cc
         db/repair.cc
+        db/seqno_to_time_mapping.cc
         db/snapshot_impl.cc
         db/table_cache.cc
         db/table_properties_collector.cc
@@ -682,6 +684,7 @@ set(SOURCES
         db/wal_edit.cc
         db/wal_manager.cc
         db/wide/wide_column_serialization.cc
+        db/wide/wide_columns.cc
         db/write_batch.cc
         db/write_batch_base.cc
         db/write_controller.cc
@@ -1007,7 +1010,7 @@ set(ROCKSDB_PLUGIN_BUILTINS "")
 message(STATUS "ROCKSDB PLUGINS TO BUILD ${ROCKSDB_PLUGINS}")
 list(APPEND PLUGINS ${ROCKSDB_PLUGINS})
 foreach(PLUGIN IN LISTS PLUGINS)
-  set(PLUGIN_ROOT "${CMAKE_SOURCE_DIR}/plugin/${PLUGIN}/")
+  set(PLUGIN_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/plugin/${PLUGIN}/")
   message("including rocksb plugin ${PLUGIN_ROOT}")
   set(PLUGINMKFILE "${PLUGIN_ROOT}${PLUGIN}.mk")
   if (NOT EXISTS ${PLUGINMKFILE})
@@ -1296,6 +1299,7 @@ if(WITH_TESTS)
         db/perf_context_test.cc
         db/periodic_work_scheduler_test.cc
         db/plain_table_db_test.cc
+        db/seqno_time_test.cc
         db/prefix_test.cc
         db/range_del_aggregator_test.cc
         db/range_tombstone_fragmenter_test.cc
@@ -1506,6 +1510,20 @@ if(WITH_BENCHMARK_TOOLS)
     ${ROCKSDB_LIB} ${GFLAGS_LIB})
 endif()
 
+option(WITH_TRACE_TOOLS "build with trace tools" ON)
+if(WITH_TRACE_TOOLS)
+  add_executable(block_cache_trace_analyzer_tool${ARTIFACT_SUFFIX}
+    tools/block_cache_analyzer/block_cache_trace_analyzer_tool.cc)
+  target_link_libraries(block_cache_trace_analyzer_tool${ARTIFACT_SUFFIX}
+    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+
+  add_executable(trace_analyzer${ARTIFACT_SUFFIX}
+    tools/trace_analyzer.cc)
+  target_link_libraries(trace_analyzer${ARTIFACT_SUFFIX}
+    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+
+endif()
+
 if(WITH_CORE_TOOLS OR WITH_TOOLS)
   add_subdirectory(tools)
   add_custom_target(core_tools

HISTORY.md

@@ -1,5 +1,52 @@
 # Rocksdb Change Log
-## Unreleased
+## 7.6.0 (08/19/2022)
+### New Features
+* Added `prepopulate_blob_cache` to ColumnFamilyOptions. If enabled, prepopulate warm/hot blobs which are already in memory into blob cache at the time of flush. On a flush, the blob that is in memory (in memtables) get flushed to the device. If using Direct IO, additional IO is incurred to read this blob back into memory again, which is avoided by enabling this option. This further helps if the workload exhibits high temporal locality, where most of the reads go to recently written data. This also helps in case of the remote file system since it involves network traffic and higher latencies.
+* Support using secondary cache with the blob cache. When creating a blob cache, the user can set a secondary blob cache by configuring `secondary_cache` in LRUCacheOptions.
+* Charge memory usage of blob cache when the backing cache of the blob cache and the block cache are different. If an operation reserving memory for blob cache exceeds the avaible space left in the block cache at some point (i.e, causing a cache full under `LRUCacheOptions::strict_capacity_limit` = true), creation will fail with `Status::MemoryLimit()`. To opt in this feature, enable charging `CacheEntryRole::kBlobCache` in `BlockBasedTableOptions::cache_usage_options`.
+* Improve subcompaction range partition so that it is likely to be more even. More evenly distribution of subcompaction will improve compaction throughput for some workloads. All input files' index blocks to sample some anchor key points from which we pick positions to partition the input range. This would introduce some CPU overhead in compaction preparation phase, if subcompaction is enabled, but it should be a small fraction of the CPU usage of the whole compaction process. This also brings a behavier change: subcompaction number is much more likely to maxed out than before.
+* Add CompactionPri::kRoundRobin, a compaction picking mode that cycles through all the files with a compact cursor in a round-robin manner. This feature is available since 7.5.
+* Provide support for subcompactions for user_defined_timestamp.
+* Added an option `memtable_protection_bytes_per_key` that turns on memtable per key-value checksum protection. Each memtable entry will be suffixed by a checksum that is computed during writes, and verified in reads/compaction. Detected corruption will be logged and with corruption status returned to user.
+* Added a blob-specific cache priority level - bottom level. Blobs are typically lower-value targets for caching than data blocks, since 1) with BlobDB, data blocks containing blob references conceptually form an index structure which has to be consulted before we can read the blob value, and 2) cached blobs represent only a single key-value, while cached data blocks generally contain multiple KVs. The user can specify the new option `low_pri_pool_ratio` in `LRUCacheOptions` to configure the ratio of capacity reserved for low priority cache entries (and therefore the remaining ratio is the space reserved for the bottom level), or configuring the new argument `low_pri_pool_ratio` in `NewLRUCache()` to achieve the same effect.
+
+### Public API changes
+* Removed Customizable support for RateLimiter and removed its CreateFromString() and Type() functions.
+* `CompactRangeOptions::exclusive_manual_compaction` is now false by default. This ensures RocksDB does not introduce artificial parallelism limitations by default.
+* Tiered Storage: change `bottommost_temperture` to `last_level_temperture`. The old option name is kept only for migration, please use the new option. The behavior is changed to apply temperature for the `last_level` SST files only.
+* Added a new experimental ReadOption flag called optimize_multiget_for_io, which when set attempts to reduce MultiGet latency by spawning coroutines for keys in multiple levels.
+
+### Bug Fixes
+* Fix a bug starting in 7.4.0 in which some fsync operations might be skipped in a DB after any DropColumnFamily on that DB, until it is re-opened. This can lead to data loss on power loss. (For custom FileSystem implementations, this could lead to `FSDirectory::Fsync` or `FSDirectory::Close` after the first `FSDirectory::Close`; Also, valgrind could report call to `close()` with `fd=-1`.)
+* Fix a bug where `GenericRateLimiter` could revert the bandwidth set dynamically using `SetBytesPerSecond()` when a user configures a structure enclosing it, e.g., using `GetOptionsFromString()` to configure an `Options` that references an existing `RateLimiter` object.
+* Fix race conditions in `GenericRateLimiter`.
+* Fix a bug in `FIFOCompactionPicker::PickTTLCompaction` where total_size calculating might cause underflow
+* Fix data race bug in hash linked list memtable. With this bug, read request might temporarily miss an old record in the memtable in a race condition to the hash bucket.
+* Fix a bug that `best_efforts_recovery` may fail to open the db with mmap read.
+* Fixed a bug where blobs read during compaction would pollute the cache.
+* Fixed a data race in LRUCache when used with a secondary_cache.
+* Fixed a bug where blobs read by iterators would be inserted into the cache even with the `fill_cache` read option set to false.
+* Fixed the segfault caused by `AllocateData()` in `CompressedSecondaryCache::SplitValueIntoChunks()` and `MergeChunksIntoValueTest`.
+* Fixed a bug in BlobDB where a mix of inlined and blob values could result in an incorrect value being passed to the compaction filter (see #10391).
+* Fixed a memory leak bug in stress tests caused by `FaultInjectionSecondaryCache`.
+
+### Behavior Change
+* Added checksum handshake during the copying of decompressed WAL fragment. This together with #9875, #10037, #10212, #10114 and #10319 provides end-to-end integrity protection for write batch during recovery.
+* To minimize the internal fragmentation caused by the variable size of the compressed blocks in `CompressedSecondaryCache`, the original block is split according to the jemalloc bin size in `Insert()` and then merged back in `Lookup()`.
+* PosixLogger is removed and by default EnvLogger will be used for info logging. The behavior of the two loggers should be very similar when using the default Posix Env.
+* Remove [min|max]_timestamp from VersionEdit for now since they are not tracked in MANIFEST anyway but consume two empty std::string (up to 64 bytes) for each file. Should they be added back in the future, we should store them more compactly.
+* Improve universal tiered storage compaction picker to avoid extra major compaction triggered by size amplification. If `preclude_last_level_data_seconds` is enabled, the size amplification is calculated within non last_level data only which skip the last level and use the penultimate level as the size base.
+* If an error is hit when writing to a file (append, sync, etc), RocksDB is more strict with not issuing more operations to it, except closing the file, with exceptions of some WAL file operations in error recovery path.
+* A `WriteBufferManager` constructed with `allow_stall == false` will no longer trigger write stall implicitly by thrashing until memtable count limit is reached. Instead, a column family can continue accumulating writes while that CF is flushing, which means memory may increase. Users who prefer stalling writes must now explicitly set `allow_stall == true`.
+* Add `CompressedSecondaryCache` into the stress tests.
+* Block cache keys have changed, which will cause any persistent caches to miss between versions.
+
+### Performance Improvements
+* Instead of constructing `FragmentedRangeTombstoneList` during every read operation, it is now constructed once and stored in immutable memtables. This improves speed of querying range tombstones from immutable memtables.
+* When using iterators with the integrated BlobDB implementation, blob cache handles are now released immediately when the iterator's position changes.
+* MultiGet can now do more IO in parallel by reading data blocks from SST files in multiple levels, if the optimize_multiget_for_io ReadOption flag is set.
+
+## 7.5.0 (07/15/2022)
 ### New Features
 * Mempurge option flag `experimental_mempurge_threshold` is now a ColumnFamilyOptions and can now be dynamically configured using `SetOptions()`.
 * Support backward iteration when `ReadOptions::iter_start_ts` is set.
@@ -8,6 +55,7 @@
   * User can configure the new ColumnFamilyOptions `blob_cache` to enable/disable blob caching.
   * Either sharing the backend cache with the block cache or using a completely separate cache is supported.
   * A new abstraction interface called `BlobSource` for blob read logic gives all users access to blobs, whether they are in the blob cache, secondary cache, or (remote) storage. Blobs can be potentially read both while handling user reads (`Get`, `MultiGet`, or iterator) and during compaction (while dealing with compaction filters, Merges, or garbage collection) but eventually all blob reads go through `Version::GetBlob` or, for MultiGet, `Version::MultiGetBlob` (and then get dispatched to the interface -- `BlobSource`).
+* Add experimental tiered compaction feature `AdvancedColumnFamilyOptions::preclude_last_level_data_seconds`, which makes sure the new data inserted within preclude_last_level_data_seconds won't be placed on cold tier (the feature is not complete).
 
 ### Public API changes
 * Add metadata related structs and functions in C API, including
@@ -19,6 +67,8 @@
 * When using block cache strict capacity limit (`LRUCache` with `strict_capacity_limit=true`), DB operations now fail with Status code `kAborted` subcode `kMemoryLimit` (`IsMemoryLimit()`) instead of `kIncomplete` (`IsIncomplete()`) when the capacity limit is reached, because Incomplete can mean other specific things for some operations. In more detail, `Cache::Insert()` now returns the updated Status code and this usually propagates through RocksDB to the user on failure.
 * NewClockCache calls temporarily return an LRUCache (with similar characteristics as the desired ClockCache). This is because ClockCache is being replaced by a new version (the old one had unknown bugs) but this is still under development.
 * Add two functions `int ReserveThreads(int threads_to_be_reserved)` and `int ReleaseThreads(threads_to_be_released)` into `Env` class. In the default implementation, both return 0. Newly added `xxxEnv` class that inherits `Env` should implement these two functions for thread reservation/releasing features.
+* Add `rocksdb_options_get_prepopulate_blob_cache` and `rocksdb_options_set_prepopulate_blob_cache` to C API.
+* Add `prepopulateBlobCache` and `setPrepopulateBlobCache` to Java API.
 
 ### Bug Fixes
 * Fix a bug in which backup/checkpoint can include a WAL deleted by RocksDB.
@@ -31,6 +81,11 @@
 * For track_and_verify_wals_in_manifest, revert to the original behavior before #10087: syncing of live WAL file is not tracked, and we track only the synced sizes of **closed** WALs. (PR #10330).
 * WAL compression now computes/verifies checksum during compression/decompression.
 
+### Performance Improvements
+* Rather than doing total sort against all files in a level, SortFileByOverlappingRatio() to only find the top 50 files based on score. This can improve write throughput for the use cases where data is loaded in increasing key order and there are a lot of files in one LSM-tree, where applying compaction results is the bottleneck.
+* In leveled compaction, L0->L1 trivial move will allow more than one file to be moved in one compaction. This would allow L0 files to be moved down faster when data is loaded in sequential order, making slowdown or stop condition harder to hit. Also seek L0->L1 trivial move when only some files qualify.
+* In leveled compaction, try to trivial move more than one files if possible, up to 4 files or max_compaction_bytes. This is to allow higher write throughput for some use cases where data is loaded in sequential order, where appying compaction results is the bottleneck.
+
 ## 7.4.0 (06/19/2022)
 ### Bug Fixes
 * Fixed a bug in calculating key-value integrity protection for users of in-place memtable updates. In particular, the affected users would be those who configure `protection_bytes_per_key > 0` on `WriteBatch` or `WriteOptions`, and configure `inplace_callback != nullptr`.
@@ -79,17 +134,13 @@
 
 ### Behavior changes
 * DB::Open(), DB::OpenAsSecondary() will fail if a Logger cannot be created (#9984)
+* DB::Write does not hold global `mutex_` if this db instance does not need to switch wal and mem-table (#7516).
 * Removed support for reading Bloom filters using obsolete block-based filter format. (Support for writing such filters was dropped in 7.0.) For good read performance on old DBs using these filters, a full compaction is required.
 * Per KV checksum in write batch is verified before a write batch is written to WAL to detect any corruption to the write batch (#10114).
 
 ### Performance Improvements
 * When compiled with folly (Meta-internal integration; experimental in open source build), improve the locking performance (CPU efficiency) of LRUCache by using folly DistributedMutex in place of standard mutex.
 
-### Performance Improvements
-* Rather than doing total sort against all files in a level, SortFileByOverlappingRatio() to only find the top 50 files based on score. This can improve write throughput for the use cases where data is loaded in increasing key order and there are a lot of files in one LSM-tree, where applying compaction results is the bottleneck.
-* In leveled compaction, L0->L1 trivial move will allow more than one file to be moved in one compaction. This would allow L0 files to be moved down faster when data is loaded in sequential order, making slowdown or stop condition harder to hit. Also seek L0->L1 trivial move when only some files qualify.
-* In leveled compaction, try to trivial move more than one files if possible, up to 4 files or max_compaction_bytes. This is to allow higher write throughput for some use cases where data is loaded in sequential order, where appying compaction results is the bottleneck.
-
 ## 7.3.0 (05/20/2022)
 ### Bug Fixes
 * Fixed a bug where manual flush would block forever even though flush options had wait=false.

Makefile

@@ -1504,6 +1504,9 @@ db_table_properties_test: $(OBJ_DIR)/db/db_table_properties_test.o $(TEST_LIBRAR
 log_write_bench: $(OBJ_DIR)/util/log_write_bench.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK) $(PROFILING_FLAGS)
 
+seqno_time_test: $(OBJ_DIR)/db/seqno_time_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 plain_table_db_test: $(OBJ_DIR)/db/plain_table_db_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 

README.md

@@ -1,7 +1,6 @@
 ## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage
 
 [![CircleCI Status](https://circleci.com/gh/facebook/rocksdb.svg?style=svg)](https://circleci.com/gh/facebook/rocksdb)
-[![TravisCI Status](https://api.travis-ci.com/facebook/rocksdb.svg?branch=main)](https://travis-ci.com/github/facebook/rocksdb)
 [![Appveyor Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/main?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/main)
 [![PPC64le Build Status](http://140-211-168-68-openstack.osuosl.org:8080/buildStatus/icon?job=rocksdb&style=plastic)](http://140-211-168-68-openstack.osuosl.org:8080/job/rocksdb)
 

TARGETS

@@ -13,6 +13,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "cache/cache_entry_roles.cc",
         "cache/cache_key.cc",
         "cache/cache_reservation_manager.cc",
+        "cache/charged_cache.cc",
         "cache/clock_cache.cc",
         "cache/compressed_secondary_cache.cc",
         "cache/fast_lru_cache.cc",
@@ -85,6 +86,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/range_del_aggregator.cc",
         "db/range_tombstone_fragmenter.cc",
         "db/repair.cc",
+        "db/seqno_to_time_mapping.cc",
         "db/snapshot_impl.cc",
         "db/table_cache.cc",
         "db/table_properties_collector.cc",
@@ -97,6 +99,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/wal_edit.cc",
         "db/wal_manager.cc",
         "db/wide/wide_column_serialization.cc",
+        "db/wide/wide_columns.cc",
         "db/write_batch.cc",
         "db/write_batch_base.cc",
         "db/write_controller.cc",
@@ -347,6 +350,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "cache/cache_entry_roles.cc",
         "cache/cache_key.cc",
         "cache/cache_reservation_manager.cc",
+        "cache/charged_cache.cc",
         "cache/clock_cache.cc",
         "cache/compressed_secondary_cache.cc",
         "cache/fast_lru_cache.cc",
@@ -419,6 +423,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "db/range_del_aggregator.cc",
         "db/range_tombstone_fragmenter.cc",
         "db/repair.cc",
+        "db/seqno_to_time_mapping.cc",
         "db/snapshot_impl.cc",
         "db/table_cache.cc",
         "db/table_properties_collector.cc",
@@ -431,6 +436,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "db/wal_edit.cc",
         "db/wal_manager.cc",
         "db/wide/wide_column_serialization.cc",
+        "db/wide/wide_columns.cc",
         "db/write_batch.cc",
         "db/write_batch_base.cc",
         "db/write_controller.cc",
@@ -5688,6 +5694,12 @@ cpp_unittest_wrapper(name="ribbon_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="seqno_time_test",
+            srcs=["db/seqno_time_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="sim_cache_test",
             srcs=["utilities/simulator_cache/sim_cache_test.cc"],
             deps=[":rocksdb_test_lib"],

build_tools/benchmark_log_tool.py

@@ -62,7 +62,11 @@ class BenchmarkUtils:
 
     def conform_opensearch(row):
         (dt, _) = parser.parse(row['date'], fuzzy_with_tokens=True)
+        # create a test_date field, which was previously what was expected
+        # repair the date field, which has what can be a WRONG ISO FORMAT, (no leading 0 on single-digit day-of-month)
+        # e.g. 2022-07-1T00:14:55 should be 2022-07-01T00:14:55
         row['test_date'] = dt.isoformat()
+        row['date'] = dt.isoformat()
         return dict((key.replace('.', '_'), value)
                     for (key, value) in row.items())
 

build_tools/format-diff.sh

@@ -170,11 +170,6 @@ echo "$diffs" |
   sed -e "s/\(^-.*$\)/`echo -e \"$COLOR_RED\1$COLOR_END\"`/" |
   sed -e "s/\(^+.*$\)/`echo -e \"$COLOR_GREEN\1$COLOR_END\"`/"
 
-if [[ "$OPT" == *"-DTRAVIS"* ]]
-then
-  exit 1
-fi
-
 echo -e "Would you like to fix the format automatically (y/n): \c"
 
 # Make sure under any mode, we can read user input.

build_tools/ubuntu20_image/Dockerfile

@@ -0,0 +1,56 @@
+# from official ubuntu 20.04
+FROM ubuntu:20.04
+# update system
+RUN apt-get update && apt-get upgrade -y
+# install basic tools
+RUN apt-get install -y vim wget curl
+# install tzdata noninteractive
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
+# install git and default compilers
+RUN apt-get install -y git gcc g++ clang clang-tools
+# install basic package
+RUN apt-get install -y lsb-release software-properties-common gnupg
+# install gflags, tbb
+RUN apt-get install -y libgflags-dev libtbb-dev
+# install compression libs
+RUN apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+# install cmake
+RUN apt-get install -y cmake
+# install clang-13
+WORKDIR /root
+RUN wget https://apt.llvm.org/llvm.sh
+RUN chmod +x llvm.sh
+RUN ./llvm.sh 13 all
+# install gcc-7, 8, 10, 11, default is 9
+RUN apt-get install -y gcc-7 g++-7
+RUN apt-get install -y gcc-8 g++-8
+RUN apt-get install -y gcc-10 g++-10
+RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test
+RUN apt-get install -y gcc-11 g++-11
+# install apt-get install -y valgrind
+RUN apt-get install -y valgrind
+# install folly depencencies
+RUN apt-get install -y libgoogle-glog-dev
+# install openjdk 8
+RUN apt-get install -y openjdk-8-jdk
+ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64
+# install mingw
+RUN apt-get install -y mingw-w64
+
+# install gtest-parallel package
+RUN git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
+ENV PATH $PATH:/root/gtest-parallel
+
+# install libprotobuf for fuzzers test
+RUN apt-get install -y ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool
+RUN git clone --branch v1.0 https://github.com/google/libprotobuf-mutator.git ~/libprotobuf-mutator && cd ~/libprotobuf-mutator && git checkout ffd86a32874e5c08a143019aad1aaf0907294c9f && mkdir build && cd build && cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON && ninja && ninja install
+ENV PKG_CONFIG_PATH /usr/local/OFF/:/root/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/
+ENV PROTOC_BIN /root/libprotobuf-mutator/build/external.protobuf/bin/protoc
+
+# install the latest google benchmark
+RUN git clone --depth 1 --branch v1.7.0 https://github.com/google/benchmark.git ~/benchmark
+RUN cd ~/benchmark && mkdir build && cd build && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0 && ninja && ninja install
+
+# clean up
+RUN rm -rf /var/lib/apt/lists/*
+RUN rm -rf /root/benchmark

cache/cache.cc

@@ -33,6 +33,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
          {offsetof(struct LRUCacheOptions, high_pri_pool_ratio),
           OptionType::kDouble, OptionVerificationType::kNormal,
           OptionTypeFlags::kMutable}},
+        {"low_pri_pool_ratio",
+         {offsetof(struct LRUCacheOptions, low_pri_pool_ratio),
+          OptionType::kDouble, OptionVerificationType::kNormal,
+          OptionTypeFlags::kMutable}},
 };
 
 static std::unordered_map<std::string, OptionTypeInfo>

cache/cache_bench_tool.cc

@@ -74,6 +74,11 @@ DEFINE_uint32(
 DEFINE_uint32(gather_stats_entries_per_lock, 256,
               "For Cache::ApplyToAllEntries");
 DEFINE_bool(skewed, false, "If true, skew the key access distribution");
+
+DEFINE_bool(lean, false,
+            "If true, no additional computation is performed besides cache "
+            "operations.");
+
 #ifndef ROCKSDB_LITE
 DEFINE_string(secondary_cache_uri, "",
               "Full URI for creating a custom secondary cache object");
@@ -109,6 +114,8 @@ DEFINE_uint32(
     "(-stress_cache_key) Simulated file size in MiB, for accounting purposes");
 DEFINE_uint32(sck_reopen_nfiles, 100,
               "(-stress_cache_key) Simulate DB re-open average every n files");
+DEFINE_uint32(sck_newdb_nreopen, 1000,
+              "(-stress_cache_key) Simulate new DB average every n re-opens");
 DEFINE_uint32(sck_restarts_per_day, 24,
               "(-stress_cache_key) Average simulated process restarts per day "
               "(across DBs)");
@@ -297,7 +304,9 @@ class CacheBench {
           FLAGS_cache_size, FLAGS_value_bytes, FLAGS_num_shard_bits,
           false /*strict_capacity_limit*/, kDefaultCacheMetadataChargePolicy);
     } else if (FLAGS_cache_type == "lru_cache") {
-      LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits, false, 0.5);
+      LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits,
+                           false /* strict_capacity_limit */,
+                           0.5 /* high_pri_pool_ratio */);
 #ifndef ROCKSDB_LITE
       if (!FLAGS_secondary_cache_uri.empty()) {
         Status s = SecondaryCache::CreateFromString(
@@ -522,7 +531,6 @@ class CacheBench {
     StopWatchNano timer(clock);
 
     for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
-      timer.Start();
       Slice key = gen.GetRand(thread->rnd, max_key_, max_log_);
       uint64_t random_op = thread->rnd.Next();
       Cache::CreateCallback create_cb = [](const void* buf, size_t size,
@@ -534,6 +542,8 @@ class CacheBench {
         return Status::OK();
       };
 
+      timer.Start();
+
       if (random_op < lookup_insert_threshold_) {
         if (handle) {
           cache_->Release(handle);
@@ -543,9 +553,11 @@ class CacheBench {
         handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
                                 true);
         if (handle) {
-          // do something with the data
-          result += NPHash64(static_cast<char*>(cache_->Value(handle)),
-                             FLAGS_value_bytes);
+          if (!FLAGS_lean) {
+            // do something with the data
+            result += NPHash64(static_cast<char*>(cache_->Value(handle)),
+                               FLAGS_value_bytes);
+          }
         } else {
           // do insert
           Status s = cache_->Insert(key, createValue(thread->rnd), &helper2,
@@ -570,9 +582,11 @@ class CacheBench {
         handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
                                 true);
         if (handle) {
-          // do something with the data
-          result += NPHash64(static_cast<char*>(cache_->Value(handle)),
-                             FLAGS_value_bytes);
+          if (!FLAGS_lean) {
+            // do something with the data
+            result += NPHash64(static_cast<char*>(cache_->Value(handle)),
+                               FLAGS_value_bytes);
+          }
         }
       } else if (random_op < erase_threshold_) {
         // do erase
@@ -770,7 +784,7 @@ class StressCacheKey {
 
   void RunOnce() {
     // Re-initialized simulated state
-    const size_t db_count = FLAGS_sck_db_count;
+    const size_t db_count = std::max(size_t{FLAGS_sck_db_count}, size_t{1});
     dbs_.reset(new TableProperties[db_count]{});
     const size_t table_mask = (size_t{1} << FLAGS_sck_table_bits) - 1;
     table_.reset(new uint64_t[table_mask + 1]{});
@@ -787,13 +801,13 @@ class StressCacheKey {
 
     process_count_ = 0;
     session_count_ = 0;
-    ResetProcess();
+    newdb_count_ = 0;
+    ResetProcess(/*newdbs*/ true);
 
     Random64 r{std::random_device{}()};
 
     uint64_t max_file_count =
         uint64_t{FLAGS_sck_files_per_day} * FLAGS_sck_days_per_run;
-    uint64_t file_size = FLAGS_sck_file_size_mb * uint64_t{1024} * 1024U;
     uint32_t report_count = 0;
     uint32_t collisions_this_run = 0;
     size_t db_i = 0;
@@ -806,9 +820,9 @@ class StressCacheKey {
       }
       // Any other periodic actions before simulating next file
       if (!FLAGS_sck_footer_unique_id && r.OneIn(FLAGS_sck_reopen_nfiles)) {
-        ResetSession(db_i);
+        ResetSession(db_i, /*newdb*/ r.OneIn(FLAGS_sck_newdb_nreopen));
       } else if (r.OneIn(restart_nfiles_)) {
-        ResetProcess();
+        ResetProcess(/*newdbs*/ false);
       }
       // Simulate next file
       OffsetableCacheKey ock;
@@ -821,8 +835,7 @@ class StressCacheKey {
       }
       bool is_stable;
       BlockBasedTable::SetupBaseCacheKey(&dbs_[db_i], /* ignored */ "",
-                                         /* ignored */ 42, file_size, &ock,
-                                         &is_stable);
+                                         /* ignored */ 42, &ock, &is_stable);
       assert(is_stable);
       // Get a representative cache key, which later we analytically generalize
       // to a range.
@@ -832,13 +845,11 @@ class StressCacheKey {
         reduced_key = GetSliceHash64(ck.AsSlice()) >> shift_away;
       } else if (FLAGS_sck_footer_unique_id) {
         // Special case: keep only file number, not session counter
-        uint32_t a = DecodeFixed32(ck.AsSlice().data() + 4) >> shift_away_a;
-        uint32_t b = DecodeFixed32(ck.AsSlice().data() + 12) >> shift_away_b;
-        reduced_key = (uint64_t{a} << 32) + b;
+        reduced_key = DecodeFixed64(ck.AsSlice().data()) >> shift_away;
       } else {
         // Try to keep file number and session counter (shift away other bits)
         uint32_t a = DecodeFixed32(ck.AsSlice().data()) << shift_away_a;
-        uint32_t b = DecodeFixed32(ck.AsSlice().data() + 12) >> shift_away_b;
+        uint32_t b = DecodeFixed32(ck.AsSlice().data() + 4) >> shift_away_b;
         reduced_key = (uint64_t{a} << 32) + b;
       }
       if (reduced_key == 0) {
@@ -860,7 +871,7 @@ class StressCacheKey {
         // Our goal is to predict probability of no collisions, not expected
         // number of collisions. To make the distinction, we have to get rid
         // of observing correlated collisions, which this takes care of:
-        ResetProcess();
+        ResetProcess(/*newdbs*/ false);
       } else {
         // Replace (end of lifetime for file that was in this slot)
         table_[pos] = reduced_key;
@@ -878,10 +889,11 @@ class StressCacheKey {
         }
         // Report
         printf(
-            "%" PRIu64 " days, %" PRIu64 " proc, %" PRIu64
-            " sess, %u coll, occ %g%%, ejected %g%%   \r",
+            "%" PRIu64 " days, %" PRIu64 " proc, %" PRIu64 " sess, %" PRIu64
+            " newdb, %u coll, occ %g%%, ejected %g%%      \r",
             file_count / FLAGS_sck_files_per_day, process_count_,
-            session_count_, collisions_this_run, 100.0 * sampled_count / 1000.0,
+            session_count_, newdb_count_ - FLAGS_sck_db_count,
+            collisions_this_run, 100.0 * sampled_count / 1000.0,
             100.0 * (1.0 - sampled_count / 1000.0 * table_mask / file_count));
         fflush(stdout);
       }
@@ -889,16 +901,27 @@ class StressCacheKey {
     collisions_ += collisions_this_run;
   }
 
-  void ResetSession(size_t i) {
+  void ResetSession(size_t i, bool newdb) {
     dbs_[i].db_session_id = DBImpl::GenerateDbSessionId(nullptr);
+    if (newdb) {
+      ++newdb_count_;
+      if (FLAGS_sck_footer_unique_id) {
+        // Simulate how footer id would behave
+        dbs_[i].db_id = "none";
+      } else {
+        // db_id might be ignored, depending on the implementation details
+        dbs_[i].db_id = std::to_string(newdb_count_);
+        dbs_[i].orig_file_number = 0;
+      }
+    }
     session_count_++;
   }
 
-  void ResetProcess() {
+  void ResetProcess(bool newdbs) {
     process_count_++;
     DBImpl::TEST_ResetDbSessionIdGen();
     for (size_t i = 0; i < FLAGS_sck_db_count; ++i) {
-      ResetSession(i);
+      ResetSession(i, newdbs);
     }
     if (FLAGS_sck_footer_unique_id) {
       // For footer unique ID, this tracks process-wide generated SST file
@@ -913,6 +936,7 @@ class StressCacheKey {
   std::unique_ptr<uint64_t[]> table_;
   uint64_t process_count_ = 0;
   uint64_t session_count_ = 0;
+  uint64_t newdb_count_ = 0;
   uint64_t collisions_ = 0;
   uint32_t restart_nfiles_ = 0;
   double multiplier_ = 0.0;

cache/cache_entry_roles.cc

@@ -23,6 +23,7 @@ std::array<std::string, kNumCacheEntryRoles> kCacheEntryRoleToCamelString{{
     "FilterConstruction",
     "BlockBasedTableReader",
     "FileMetadata",
+    "BlobCache",
     "Misc",
 }};
 
@@ -38,6 +39,7 @@ std::array<std::string, kNumCacheEntryRoles> kCacheEntryRoleToHyphenString{{
     "filter-construction",
     "block-based-table-reader",
     "file-metadata",
+    "blob-cache",
     "misc",
 }};
 

cache/cache_key.cc

@@ -17,7 +17,7 @@ namespace ROCKSDB_NAMESPACE {
 
 // Value space plan for CacheKey:
 //
-// session_etc64_ | offset_etc64_ | Only generated by
+// file_num_etc64_ | offset_etc64_ | Only generated by
 // ---------------+---------------+------------------------------------------
 //              0 |             0 | Reserved for "empty" CacheKey()
 //              0 |  > 0, < 1<<63 | CreateUniqueForCacheLifetime
@@ -44,7 +44,7 @@ CacheKey CacheKey::CreateUniqueForProcessLifetime() {
   return CacheKey(0, id);
 }
 
-// Value plan for CacheKeys from OffsetableCacheKey, assuming that
+// How we generate CacheKeys and base OffsetableCacheKey, assuming that
 // db_session_ids are generated from a base_session_id and
 // session_id_counter (by SemiStructuredUniqueIdGen+EncodeSessionId
 // in DBImpl::GenerateDbSessionId):
@@ -56,63 +56,108 @@ CacheKey CacheKey::CreateUniqueForProcessLifetime() {
 //   base_session_id         (unstructured, from GenerateRawUniqueId)
 //   session_id_counter      (structured)
 //                           * usually much smaller than 2**24
-//   file_number             (structured)
+//   orig_file_number        (structured)
 //                           * usually smaller than 2**24
 //   offset_in_file          (structured, might skip lots of values)
 //                           * usually smaller than 2**32
-//   max_offset              determines placement of file_number to prevent
-//                           overlapping with offset
 //
-// Outputs come from bitwise-xor of the constituent pieces, low bits on left:
+// Overall approach (see https://github.com/pdillinger/unique_id for
+// background):
 //
-// |------------------------- session_etc64 -------------------------|
-// | +++++++++++++++ base_session_id (lower 64 bits) +++++++++++++++ |
+// First, we have three "structured" values, up to 64 bits each, that we
+// need to fit, without losses, into 128 bits. In practice, the values will
+// be small enough that they should fit. For example, applications generating
+// large SST files (large offsets) will naturally produce fewer files (small
+// file numbers). But we don't know ahead of time what bounds the values will
+// have.
+//
+// Second, we have unstructured inputs that enable distinct RocksDB processes
+// to pick a random point in space, likely very different from others. Xoring
+// the structured with the unstructured give us a cache key that is
+// structurally distinct between related keys (e.g. same file or same RocksDB
+// process) and distinct with high probability between unrelated keys.
+//
+// The problem of packing three structured values into the space for two is
+// complicated by the fact that we want to derive cache keys from SST unique
+// IDs, which have already combined structured and unstructured inputs in a
+// practically inseparable way. And we want a base cache key that works
+// with an offset of any size. So basically, we need to encode these three
+// structured values, each up to 64 bits, into 128 bits without knowing any
+// of their sizes. The DownwardInvolution() function gives us a mechanism to
+// accomplish this. (See its properties in math.h.) Specifically, for inputs
+// a, b, and c:
+//   lower64 = DownwardInvolution(a) ^ ReverseBits(b);
+//   upper64 = c ^ ReverseBits(a);
+// The 128-bit output is unique assuming there exist some i, j, and k
+// where a < 2**i, b < 2**j, c < 2**k, i <= 64, j <= 64, k <= 64, and
+// i + j + k <= 128. In other words, as long as there exist some bounds
+// that would allow us to pack the bits of a, b, and c into the output
+// if we know the bound, we can generate unique outputs without knowing
+// those bounds. To validate this claim, the inversion function (given
+// the bounds) has been implemented in CacheKeyDecoder in
+// db_block_cache_test.cc.
+//
+// With that in mind, the outputs in terms of the conceptual inputs look
+// like this, using bitwise-xor of the constituent pieces, low bits on left:
+//
+// |------------------------- file_num_etc64 -------------------------|
+// | +++++++++ base_session_id (lower 64 bits, involution) +++++++++ |
 // |-----------------------------------------------------------------|
-// | session_id_counter ...|                                         |
+// | session_id_counter (involution) ..... |                         |
 // |-----------------------------------------------------------------|
-// |                                               | ... file_number |
-// |                                               | overflow & meta |
-// |-----------------------------------------------------------------|
-//
-//
-// |------------------------- offset_etc64 --------------------------|
 // | hash of: ++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
 // |  * base_session_id (upper ~39 bits)                             |
 // |  * db_id (~122 bits entropy)                                    |
 // |-----------------------------------------------------------------|
+// |                             | ..... orig_file_number (reversed) |
+// |-----------------------------------------------------------------|
+//
+//
+// |------------------------- offset_etc64 --------------------------|
+// | ++++++++++ base_session_id (lower 64 bits, reversed) ++++++++++ |
+// |-----------------------------------------------------------------|
+// |                           | ..... session_id_counter (reversed) |
+// |-----------------------------------------------------------------|
 // | offset_in_file ............... |                                |
 // |-----------------------------------------------------------------|
-// |                                              | file_number, 0-3 |
-// |                                              | lower bytes      |
-// |-----------------------------------------------------------------|
 //
-// Based on max_offset, a maximal number of bytes 0..3 is chosen for
-// including from lower bits of file_number in offset_etc64. The choice
-// is encoded in two bits of metadata going into session_etc64, though
-// the common case of 3 bytes is encoded as 0 so that session_etc64
-// is unmodified by file_number concerns in the common case.
+// Some oddities or inconveniences of this layout are due to deriving
+// the "base" cache key (without offset) from the SST unique ID (see
+// GetSstInternalUniqueId). Specifically,
+// * Lower 64 of base_session_id occurs in both output words (ok but
+//   weird)
+// * The inclusion of db_id is bad for the conditions under which we
+//   can guarantee uniqueness, but could be useful in some cases with
+//   few small files per process, to make up for db session id only having
+//   ~103 bits of entropy.
 //
-// There is nothing preventing "file number overflow & meta" from meeting
-// and overlapping with session_id_counter, but reaching such a case requires
-// an intractable combination of large file offsets (thus at least some large
-// files), large file numbers (thus large number of files generated), and
-// large number of session IDs generated in a single process. A trillion each
-// (2**40) of session ids, offsets, and file numbers comes to 120 bits.
-// With two bits of metadata and byte granularity, this is on the verge of
-// overlap, but even in the overlap case, it doesn't seem likely that
-// a file from billions of files or session ids ago will still be live
-// or cached.
+// In fact, if DB ids were not involved, we would be guaranteed unique
+// cache keys for files generated in a single process until total bits for
+// biggest session_id_counter, orig_file_number, and offset_in_file
+// reach 128 bits.
 //
-// In fact, if our SST files are all < 4TB (see
-// BlockBasedTable::kMaxFileSizeStandardEncoding), then SST files generated
-// in a single process are guaranteed to have unique cache keys, unless/until
-// number session ids * max file number = 2**86, e.g. 1 trillion DB::Open in
-// a single process and 64 trillion files generated. Even at that point, to
-// see a collision we would need a miraculous re-synchronization of session
-// id and file number, along with a live file or stale cache entry from
-// trillions of files ago.
+// With the DB id limitation, we only have nice guaranteed unique cache
+// keys for files generated in a single process until biggest
+// session_id_counter and offset_in_file reach combined 64 bits. This
+// is quite good in practice because we can have millions of DB Opens
+// with terabyte size SST files, or billions of DB Opens with gigabyte
+// size SST files.
 //
-// How https://github.com/pdillinger/unique_id applies here:
+// One of the considerations in the translation between existing SST unique
+// IDs and base cache keys is supporting better SST unique IDs in a future
+// format_version. If we use a process-wide file counter instead of
+// session counter and file numbers, we only need to combine two 64-bit values
+// instead of three. But we don't want to track unique ID versions in the
+// manifest, so we want to keep the same translation layer between SST unique
+// IDs and base cache keys, even with updated SST unique IDs. If the new
+// unique IDs put the file counter where the orig_file_number was, and
+// use no structured field where session_id_counter was, then our translation
+// layer works fine for two structured fields as well as three (for
+// compatibility). The small computation for the translation (one
+// DownwardInvolution(), two ReverseBits(), both ~log(64) instructions deep)
+// is negligible for computing as part of SST file reader open.
+//
+// More on how https://github.com/pdillinger/unique_id applies here:
 // Every bit of output always includes "unstructured" uniqueness bits and
 // often combines with "structured" uniqueness bits. The "unstructured" bits
 // change infrequently: only when we cannot guarantee our state tracking for
@@ -141,12 +186,11 @@ CacheKey CacheKey::CreateUniqueForProcessLifetime() {
 // 128 bits cache key size
 // - 55 <- ideal size for byte offsets + file numbers
 // -  2 <- bits for offsets and file numbers not exactly powers of two
-// -  2 <- bits for file number encoding metadata
 // +  2 <- bits saved not using byte offsets in BlockBasedTable::GetCacheKey
 // ----
-//   71 <- bits remaining for distinguishing session IDs
-// The probability of a collision in 71 bits of session ID data is less than
-// 1 in 2**(71 - (2 * 16)), or roughly 1 in a trillion. And this assumes all
+//   73 <- bits remaining for distinguishing session IDs
+// The probability of a collision in 73 bits of session ID data is less than
+// 1 in 2**(73 - (2 * 16)), or roughly 1 in a trillion. And this assumes all
 // data from the last 180 days is in cache for potential collision, and that
 // cache keys under each session id exhaustively cover the remaining 57 bits
 // while in reality they'll only cover a small fraction of it.
@@ -160,7 +204,7 @@ CacheKey CacheKey::CreateUniqueForProcessLifetime() {
 // Now suppose we have many DBs per host, say 2**10, with same host-wide write
 // rate and process/session lifetime. File numbers will be ~10 bits smaller
 // and we will have 2**10 times as many session IDs because of simultaneous
-// lifetimes. So now collision chance is less than 1 in 2**(81 - (2 * 26)),
+// lifetimes. So now collision chance is less than 1 in 2**(83 - (2 * 26)),
 // or roughly 1 in a billion.
 //
 // Suppose instead we generated random or hashed cache keys for each
@@ -176,17 +220,17 @@ CacheKey CacheKey::CreateUniqueForProcessLifetime() {
 // activity over many months, by making some pessimistic simplifying
 // assumptions. See class StressCacheKey in cache_bench_tool.cc for details.
 // Here is some sample output with
-// `./cache_bench -stress_cache_key -sck_keep_bits=40`:
+// `./cache_bench -stress_cache_key -sck_keep_bits=43`:
 //
 //   Total cache or DBs size: 32TiB  Writing 925.926 MiB/s or 76.2939TiB/day
-//   Multiply by 9.22337e+18 to correct for simulation losses (but still
+//   Multiply by 1.15292e+18 to correct for simulation losses (but still
 //   assume whole file cached)
 //
 // These come from default settings of 2.5M files per day of 32 MB each, and
-// `-sck_keep_bits=40` means that to represent a single file, we are only
-// keeping 40 bits of the 128-bit (base) cache key.  With file size of 2**25
-// contiguous keys (pessimistic), our simulation is about 2\*\*(128-40-25) or
-// about 9 billion billion times more prone to collision than reality.
+// `-sck_keep_bits=43` means that to represent a single file, we are only
+// keeping 43 bits of the 128-bit (base) cache key.  With file size of 2**25
+// contiguous keys (pessimistic), our simulation is about 2\*\*(128-43-25) or
+// about 1 billion billion times more prone to collision than reality.
 //
 // More default assumptions, relatively pessimistic:
 // * 100 DBs in same process (doesn't matter much)
@@ -194,49 +238,55 @@ CacheKey CacheKey::CreateUniqueForProcessLifetime() {
 // average every 100 files generated
 // * Restart process (all new session IDs unrelated to old) 24 times per day
 //
-// After enough data, we get a result at the end (-sck_keep_bits=40):
+// After enough data, we get a result at the end (-sck_keep_bits=43):
 //
-//   (keep 40 bits)  17 collisions after 2 x 90 days, est 10.5882 days between
-//                   (9.76592e+19 corrected)
+//   (keep 43 bits)  18 collisions after 2 x 90 days, est 10 days between
+//                   (1.15292e+19 corrected)
 //
 // If we believe the (pessimistic) simulation and the mathematical
-// extrapolation, we would need to run a billion machines all for 97 billion
+// extrapolation, we would need to run a billion machines all for 11 billion
 // days to expect a cache key collision. To help verify that our extrapolation
-// ("corrected") is robust, we can make our simulation more precise with
-// `-sck_keep_bits=41` and `42`, which takes more running time to get enough
+// ("corrected") is robust, we can make our simulation more precise by
+// increasing the "keep" bits, which takes more running time to get enough
 // collision data:
 //
-//   (keep 41 bits)  16 collisions after 4 x 90 days, est 22.5 days between
-//                   (1.03763e+20 corrected)
-//   (keep 42 bits)  19 collisions after 10 x 90 days, est 47.3684 days between
-//                   (1.09224e+20 corrected)
+//   (keep 44 bits)  16 collisions after 5 x 90 days, est 28.125 days between
+//                   (1.6213e+19 corrected)
+//   (keep 45 bits)  15 collisions after 7 x 90 days, est 42 days between
+//                   (1.21057e+19 corrected)
+//   (keep 46 bits)  15 collisions after 17 x 90 days, est 102 days between
+//                   (1.46997e+19 corrected)
+//   (keep 47 bits)  15 collisions after 49 x 90 days, est 294 days between
+//                   (2.11849e+19 corrected)
 //
-// The extrapolated prediction is very close. If anything, we might have some
-// very small losses of structured data (see class StressCacheKey in
-// cache_bench_tool.cc) leading to more accurate & more attractive prediction
-// with more bits kept.
+// The extrapolated prediction seems to be within noise (sampling error).
 //
 // With the `-sck_randomize` option, we can see that typical workloads like
 // above have lower collision probability than "random" cache keys (note:
-// offsets still non-randomized) by a modest amount (roughly 20x less collision
-// prone than random), which should make us reasonably comfortable even in
-// "degenerate" cases (e.g. repeatedly launch a process to generate 1 file
-// with SstFileWriter):
+// offsets still non-randomized) by a modest amount (roughly 2-3x less
+// collision prone than random), which should make us reasonably comfortable
+// even in "degenerate" cases (e.g. repeatedly launch a process to generate
+// one file with SstFileWriter):
 //
-//   (rand 40 bits) 197 collisions after 1 x 90 days, est 0.456853 days between
-//                  (4.21372e+18 corrected)
+//   (rand 43 bits) 22 collisions after 1 x 90 days, est 4.09091 days between
+//                  (4.7165e+18 corrected)
 //
-// We can see that with more frequent process restarts (all new session IDs),
-// we get closer to the "random" cache key performance:
+// We can see that with more frequent process restarts,
+// -sck_restarts_per_day=5000, which means more all-new session IDs, we get
+// closer to the "random" cache key performance:
 //
-//   (-sck_restarts_per_day=5000): 140 collisions after 1 x 90 days, ...
-//                  (5.92931e+18 corrected)
+// 15 collisions after 1 x 90 days, est 6 days between (6.91753e+18 corrected)
+//
+// And with less frequent process restarts and re-opens,
+// -sck_restarts_per_day=1 -sck_reopen_nfiles=1000, we get lower collision
+// probability:
+//
+// 18 collisions after 8 x 90 days, est 40 days between (4.61169e+19 corrected)
 //
 // Other tests have been run to validate other conditions behave as expected,
 // never behaving "worse than random" unless we start chopping off structured
 // data.
 //
-//
 // Conclusion: Even in extreme cases, rapidly burning through "all new" IDs
 // that only arise when a new process is started, the chance of any cache key
 // collisions in a giant fleet of machines is negligible. Especially when
@@ -249,96 +299,66 @@ CacheKey CacheKey::CreateUniqueForProcessLifetime() {
 // quantify) block cache corruptions, including collisions, should be added.
 OffsetableCacheKey::OffsetableCacheKey(const std::string &db_id,
                                        const std::string &db_session_id,
-                                       uint64_t file_number,
-                                       uint64_t max_offset) {
+                                       uint64_t file_number) {
+  UniqueId64x2 internal_id;
+  Status s = GetSstInternalUniqueId(db_id, db_session_id, file_number,
+                                    &internal_id, /*force=*/true);
+  assert(s.ok());
+  *this = FromInternalUniqueId(&internal_id);
+}
+
+OffsetableCacheKey OffsetableCacheKey::FromInternalUniqueId(UniqueIdPtr id) {
+  uint64_t session_lower = id.ptr[0];
+  uint64_t file_num_etc = id.ptr[1];
+
 #ifndef NDEBUG
-  max_offset_ = max_offset;
+  bool is_empty = session_lower == 0 && file_num_etc == 0;
 #endif
-  // Closely related to GetSstInternalUniqueId, but only need 128 bits and
-  // need to include an offset within the file.
-  // See also https://github.com/pdillinger/unique_id for background.
-  uint64_t session_upper = 0;  // Assignment to appease clang-analyze
-  uint64_t session_lower = 0;  // Assignment to appease clang-analyze
-  {
-    Status s = DecodeSessionId(db_session_id, &session_upper, &session_lower);
-    if (!s.ok()) {
-      // A reasonable fallback in case malformed
-      Hash2x64(db_session_id.data(), db_session_id.size(), &session_upper,
-               &session_lower);
-    }
-  }
-
-  // Hash the session upper (~39 bits entropy) and DB id (120+ bits entropy)
-  // for more global uniqueness entropy.
-  // (It is possible that many DBs descended from one common DB id are copied
-  // around and proliferate, in which case session id is critical, but it is
-  // more common for different DBs to have different DB ids.)
-  uint64_t db_hash = Hash64(db_id.data(), db_id.size(), session_upper);
-
-  // This establishes the db+session id part of the cache key.
-  //
-  // Exactly preserve (in common cases; see modifiers below) session lower to
-  // ensure that session ids generated during the same process lifetime are
-  // guaranteed unique.
-  //
-  // We put this first for CommonPrefixSlice(), so that a small-ish set of
-  // cache key prefixes to cover entries relevant to any DB.
-  session_etc64_ = session_lower;
-  // This provides extra entopy in case of different DB id or process
-  // generating a session id, but is also partly/variably obscured by
-  // file_number and offset (see below).
-  offset_etc64_ = db_hash;
-
-  // Into offset_etc64_ we are (eventually) going to pack & xor in an offset and
-  // a file_number, but we might need the file_number to overflow into
-  // session_etc64_. (There must only be one session_etc64_ value per
-  // file, and preferably shared among many files.)
-  //
-  // Figure out how many bytes of file_number we are going to be able to
-  // pack in with max_offset, though our encoding will only support packing
-  // in up to 3 bytes of file_number. (16M file numbers is enough for a new
-  // file number every second for half a year.)
-  int file_number_bytes_in_offset_etc =
-      (63 - FloorLog2(max_offset | 0x100000000U)) / 8;
-  int file_number_bits_in_offset_etc = file_number_bytes_in_offset_etc * 8;
-
-  // Assert two bits of metadata
-  assert(file_number_bytes_in_offset_etc >= 0 &&
-         file_number_bytes_in_offset_etc <= 3);
-  // Assert we couldn't have used a larger allowed number of bytes (shift
-  // would chop off bytes).
-  assert(file_number_bytes_in_offset_etc == 3 ||
-         (max_offset << (file_number_bits_in_offset_etc + 8) >>
-          (file_number_bits_in_offset_etc + 8)) != max_offset);
-
-  uint64_t mask = (uint64_t{1} << (file_number_bits_in_offset_etc)) - 1;
-  // Pack into high bits of etc so that offset can go in low bits of etc
-  // TODO: could be EndianSwapValue?
-  uint64_t offset_etc_modifier = ReverseBits(file_number & mask);
-  assert(offset_etc_modifier << file_number_bits_in_offset_etc == 0U);
-
-  // Overflow and 3 - byte count (likely both zero) go into session_id part
-  uint64_t session_etc_modifier =
-      (file_number >> file_number_bits_in_offset_etc << 2) |
-      static_cast<uint64_t>(3 - file_number_bytes_in_offset_etc);
-  // Packed into high bits to minimize interference with session id counter.
-  session_etc_modifier = ReverseBits(session_etc_modifier);
-
-  // Assert session_id part is only modified in extreme cases
-  assert(session_etc_modifier == 0 || file_number > /*3 bytes*/ 0xffffffU ||
-         max_offset > /*5 bytes*/ 0xffffffffffU);
-
-  // Xor in the modifiers
-  session_etc64_ ^= session_etc_modifier;
-  offset_etc64_ ^= offset_etc_modifier;
 
   // Although DBImpl guarantees (in recent versions) that session_lower is not
-  // zero, that's not entirely sufficient to guarantee that session_etc64_ is
+  // zero, that's not entirely sufficient to guarantee that file_num_etc64_ is
   // not zero (so that the 0 case can be used by CacheKey::CreateUnique*)
-  if (session_etc64_ == 0U) {
-    session_etc64_ = session_upper | 1U;
+  // However, if we are given an "empty" id as input, then we should produce
+  // "empty" as output.
+  // As a consequence, this function is only bijective assuming
+  // id[0] == 0 only if id[1] == 0.
+  if (session_lower == 0U) {
+    session_lower = file_num_etc;
   }
-  assert(session_etc64_ != 0);
+
+  // See comments above for how DownwardInvolution and ReverseBits
+  // make this function invertible under various assumptions.
+  OffsetableCacheKey rv;
+  rv.file_num_etc64_ =
+      DownwardInvolution(session_lower) ^ ReverseBits(file_num_etc);
+  rv.offset_etc64_ = ReverseBits(session_lower);
+
+  // Because of these transformations and needing to allow arbitrary
+  // offset (thus, second 64 bits of cache key might be 0), we need to
+  // make some correction to ensure the first 64 bits is not 0.
+  // Fortunately, the transformation ensures the second 64 bits is not 0
+  // for non-empty base key, so we can swap in the case one is 0 without
+  // breaking bijectivity (assuming condition above).
+  assert(is_empty || rv.offset_etc64_ > 0);
+  if (rv.file_num_etc64_ == 0) {
+    std::swap(rv.file_num_etc64_, rv.offset_etc64_);
+  }
+  assert(is_empty || rv.file_num_etc64_ > 0);
+  return rv;
+}
+
+// Inverse of FromInternalUniqueId (assuming file_num_etc64 == 0 only if
+// offset_etc64 == 0)
+UniqueId64x2 OffsetableCacheKey::ToInternalUniqueId() {
+  uint64_t a = file_num_etc64_;
+  uint64_t b = offset_etc64_;
+  if (b == 0) {
+    std::swap(a, b);
+  }
+  UniqueId64x2 rv;
+  rv[0] = ReverseBits(b);
+  rv[1] = ReverseBits(a ^ DownwardInvolution(rv[0]));
+  return rv;
 }
 
 }  // namespace ROCKSDB_NAMESPACE

cache/cache_key.h

@@ -9,6 +9,7 @@
 
 #include "rocksdb/rocksdb_namespace.h"
 #include "rocksdb/slice.h"
+#include "table/unique_id_impl.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -33,10 +34,10 @@ class CacheKey {
  public:
   // For convenience, constructs an "empty" cache key that is never returned
   // by other means.
-  inline CacheKey() : session_etc64_(), offset_etc64_() {}
+  inline CacheKey() : file_num_etc64_(), offset_etc64_() {}
 
   inline bool IsEmpty() const {
-    return (session_etc64_ == 0) & (offset_etc64_ == 0);
+    return (file_num_etc64_ == 0) & (offset_etc64_ == 0);
   }
 
   // Use this cache key as a Slice (byte order is endianness-dependent)
@@ -59,9 +60,9 @@ class CacheKey {
 
  protected:
   friend class OffsetableCacheKey;
-  CacheKey(uint64_t session_etc64, uint64_t offset_etc64)
-      : session_etc64_(session_etc64), offset_etc64_(offset_etc64) {}
-  uint64_t session_etc64_;
+  CacheKey(uint64_t file_num_etc64, uint64_t offset_etc64)
+      : file_num_etc64_(file_num_etc64), offset_etc64_(offset_etc64) {}
+  uint64_t file_num_etc64_;
   uint64_t offset_etc64_;
 };
 
@@ -85,50 +86,58 @@ class OffsetableCacheKey : private CacheKey {
   inline OffsetableCacheKey() : CacheKey() {}
 
   // Constructs an OffsetableCacheKey with the given information about a file.
-  // max_offset is based on file size (see WithOffset) and is required here to
-  // choose an appropriate (sub-)encoding. This constructor never generates an
-  // "empty" base key.
+  // This constructor never generates an "empty" base key.
   OffsetableCacheKey(const std::string &db_id, const std::string &db_session_id,
-                     uint64_t file_number, uint64_t max_offset);
+                     uint64_t file_number);
+
+  // Creates an OffsetableCacheKey from an SST unique ID, so that cache keys
+  // can be derived from DB manifest data before reading the file from
+  // storage--so that every part of the file can potentially go in a persistent
+  // cache.
+  //
+  // Calling GetSstInternalUniqueId() on a db_id, db_session_id, and
+  // file_number and passing the result to this function produces the same
+  // base cache key as feeding those inputs directly to the constructor.
+  //
+  // This is a bijective transformation assuming either id is empty or
+  // lower 64 bits is non-zero:
+  // * Empty (all zeros) input -> empty (all zeros) output
+  // * Lower 64 input is non-zero -> lower 64 output (file_num_etc64_) is
+  //   non-zero
+  static OffsetableCacheKey FromInternalUniqueId(UniqueIdPtr id);
+
+  // This is the inverse transformation to the above, assuming either empty
+  // or lower 64 bits (file_num_etc64_) is non-zero. Perhaps only useful for
+  // testing.
+  UniqueId64x2 ToInternalUniqueId();
 
   inline bool IsEmpty() const {
-    bool result = session_etc64_ == 0;
+    bool result = file_num_etc64_ == 0;
     assert(!(offset_etc64_ > 0 && result));
     return result;
   }
 
-  // Construct a CacheKey for an offset within a file, which must be
-  // <= max_offset provided in constructor. An offset is not necessarily a
-  // byte offset if a smaller unique identifier of keyable offsets is used.
+  // Construct a CacheKey for an offset within a file. An offset is not
+  // necessarily a byte offset if a smaller unique identifier of keyable
+  // offsets is used.
   //
   // This class was designed to make this hot code extremely fast.
   inline CacheKey WithOffset(uint64_t offset) const {
     assert(!IsEmpty());
-    assert(offset <= max_offset_);
-    return CacheKey(session_etc64_, offset_etc64_ ^ offset);
+    return CacheKey(file_num_etc64_, offset_etc64_ ^ offset);
   }
 
-  // The "common prefix" is a shared prefix for all the returned CacheKeys,
-  // that also happens to usually be the same among many files in the same DB,
-  // so is efficient and highly accurate (not perfectly) for DB-specific cache
-  // dump selection (but not file-specific).
+  // The "common prefix" is a shared prefix for all the returned CacheKeys.
+  // It is specific to the file but the same for all offsets within the file.
   static constexpr size_t kCommonPrefixSize = 8;
   inline Slice CommonPrefixSlice() const {
-    static_assert(sizeof(session_etc64_) == kCommonPrefixSize,
+    static_assert(sizeof(file_num_etc64_) == kCommonPrefixSize,
                   "8 byte common prefix expected");
     assert(!IsEmpty());
-    assert(&this->session_etc64_ == static_cast<const void *>(this));
+    assert(&this->file_num_etc64_ == static_cast<const void *>(this));
 
     return Slice(reinterpret_cast<const char *>(this), kCommonPrefixSize);
   }
-
-  // For any max_offset <= this value, the same encoding scheme is guaranteed.
-  static constexpr uint64_t kMaxOffsetStandardEncoding = 0xffffffffffU;
-
- private:
-#ifndef NDEBUG
-  uint64_t max_offset_ = 0;
-#endif
 };
 
 }  // namespace ROCKSDB_NAMESPACE

cache/cache_reservation_manager.cc

@@ -181,4 +181,5 @@ template class CacheReservationManagerImpl<CacheEntryRole::kFilterConstruction>;
 template class CacheReservationManagerImpl<CacheEntryRole::kMisc>;
 template class CacheReservationManagerImpl<CacheEntryRole::kWriteBuffer>;
 template class CacheReservationManagerImpl<CacheEntryRole::kFileMetadata>;
+template class CacheReservationManagerImpl<CacheEntryRole::kBlobCache>;
 }  // namespace ROCKSDB_NAMESPACE

cache/cache_reservation_manager_test.cc

@@ -48,13 +48,13 @@ TEST_F(CacheReservationManagerTest, GenerateCacheKey) {
   // Next unique Cache key
   CacheKey ckey = CacheKey::CreateUniqueForCacheLifetime(cache.get());
   // Get to the underlying values
-  using PairU64 = std::array<uint64_t, 2>;
-  auto& ckey_pair = *reinterpret_cast<PairU64*>(&ckey);
+  uint64_t* ckey_data = reinterpret_cast<uint64_t*>(&ckey);
   // Back it up to the one used by CRM (using CacheKey implementation details)
-  ckey_pair[1]--;
+  ckey_data[1]--;
 
   // Specific key (subject to implementation details)
-  EXPECT_EQ(ckey_pair, PairU64({0, 2}));
+  EXPECT_EQ(ckey_data[0], 0);
+  EXPECT_EQ(ckey_data[1], 2);
 
   Cache::Handle* handle = cache->Lookup(ckey.AsSlice());
   EXPECT_NE(handle, nullptr)

cache/cache_test.cc

@@ -23,6 +23,11 @@
 #include "util/coding.h"
 #include "util/string_util.h"
 
+// FastLRUCache and ClockCache only support 16-byte keys, so some of
+// the tests originally wrote for LRUCache do not work on the other caches.
+// Those tests were adapted to use 16-byte keys. We kept the original ones.
+// TODO: Remove the original tests if they ever become unused.
+
 namespace ROCKSDB_NAMESPACE {
 
 namespace {
@@ -58,17 +63,22 @@ int DecodeValue(void* v) {
   return static_cast<int>(reinterpret_cast<uintptr_t>(v));
 }
 
-const std::string kLRU = "lru";
-const std::string kClock = "clock";
-const std::string kFast = "fast";
+void DumbDeleter(const Slice& /*key*/, void* /*value*/) {}
 
-void dumbDeleter(const Slice& /*key*/, void* /*value*/) {}
-
-void eraseDeleter(const Slice& /*key*/, void* value) {
+void EraseDeleter1(const Slice& /*key*/, void* value) {
   Cache* cache = reinterpret_cast<Cache*>(value);
   cache->Erase("foo");
 }
 
+void EraseDeleter2(const Slice& /*key*/, void* value) {
+  Cache* cache = reinterpret_cast<Cache*>(value);
+  cache->Erase(EncodeKey16Bytes(1234));
+}
+
+const std::string kLRU = "lru";
+const std::string kClock = "clock";
+const std::string kFast = "fast";
+
 }  // anonymous namespace
 
 class CacheTest : public testing::TestWithParam<std::string> {
@@ -223,13 +233,9 @@ class LRUCacheTest : public CacheTest {};
 
 TEST_P(CacheTest, UsageTest) {
   auto type = GetParam();
-  if (type == kFast || type == kClock) {
-    ROCKSDB_GTEST_BYPASS("FastLRUCache and ClockCache require 16-byte keys.");
-    return;
-  }
 
   // cache is std::shared_ptr and will be automatically cleaned up.
-  const uint64_t kCapacity = 100000;
+  const size_t kCapacity = 100000;
   auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata);
   auto precise_cache = NewCache(kCapacity, 0, false, kFullChargeCacheMetadata);
   ASSERT_EQ(0, cache->GetUsage());
@@ -239,12 +245,17 @@ TEST_P(CacheTest, UsageTest) {
   char value[10] = "abcdef";
   // make sure everything will be cached
   for (int i = 1; i < 100; ++i) {
-    std::string key(i, 'a');
+    std::string key;
+    if (type == kLRU) {
+      key = std::string(i, 'a');
+    } else {
+      key = EncodeKey(i);
+    }
     auto kv_size = key.size() + 5;
     ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
-                            dumbDeleter));
+                            DumbDeleter));
     ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    kv_size, dumbDeleter));
+                                    kv_size, DumbDeleter));
     usage += kv_size;
     ASSERT_EQ(usage, cache->GetUsage());
     ASSERT_LT(usage, precise_cache->GetUsage());
@@ -256,12 +267,17 @@ TEST_P(CacheTest, UsageTest) {
   ASSERT_EQ(0, precise_cache->GetUsage());
 
   // make sure the cache will be overloaded
-  for (uint64_t i = 1; i < kCapacity; ++i) {
-    auto key = std::to_string(i);
+  for (size_t i = 1; i < kCapacity; ++i) {
+    std::string key;
+    if (type == kLRU) {
+      key = std::to_string(i);
+    } else {
+      key = EncodeKey(static_cast<int>(1000 + i));
+    }
     ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
-                            dumbDeleter));
+                            DumbDeleter));
     ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    key.size() + 5, dumbDeleter));
+                                    key.size() + 5, DumbDeleter));
   }
 
   // the usage should be close to the capacity
@@ -271,15 +287,18 @@ TEST_P(CacheTest, UsageTest) {
   ASSERT_LT(kCapacity * 0.95, precise_cache->GetUsage());
 }
 
+// TODO: This test takes longer than expected on ClockCache. This is
+// because the values size estimate at construction is too sloppy.
+// Fix this.
+// Why is it so slow? The cache is constructed with an estimate of 1, but
+// then the charge is claimed to be 21. This will cause the hash table
+// to be extremely sparse, which in turn means clock needs to scan too
+// many slots to find victims.
 TEST_P(CacheTest, PinnedUsageTest) {
   auto type = GetParam();
-  if (type == kFast || type == kClock) {
-    ROCKSDB_GTEST_BYPASS("FastLRUCache and ClockCache require 16-byte keys.");
-    return;
-  }
 
   // cache is std::shared_ptr and will be automatically cleaned up.
-  const uint64_t kCapacity = 200000;
+  const size_t kCapacity = 200000;
   auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata);
   auto precise_cache = NewCache(kCapacity, 8, false, kFullChargeCacheMetadata);
 
@@ -292,15 +311,20 @@ TEST_P(CacheTest, PinnedUsageTest) {
   // Add entries. Unpin some of them after insertion. Then, pin some of them
   // again. Check GetPinnedUsage().
   for (int i = 1; i < 100; ++i) {
-    std::string key(i, 'a');
+    std::string key;
+    if (type == kLRU) {
+      key = std::string(i, 'a');
+    } else {
+      key = EncodeKey(i);
+    }
     auto kv_size = key.size() + 5;
     Cache::Handle* handle;
     Cache::Handle* handle_in_precise_cache;
     ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
-                            dumbDeleter, &handle));
+                            DumbDeleter, &handle));
     assert(handle);
     ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    kv_size, dumbDeleter,
+                                    kv_size, DumbDeleter,
                                     &handle_in_precise_cache));
     assert(handle_in_precise_cache);
     pinned_usage += kv_size;
@@ -334,12 +358,17 @@ TEST_P(CacheTest, PinnedUsageTest) {
   ASSERT_LT(pinned_usage, precise_cache_pinned_usage);
 
   // check that overloading the cache does not change the pinned usage
-  for (uint64_t i = 1; i < 2 * kCapacity; ++i) {
-    auto key = std::to_string(i);
+  for (size_t i = 1; i < 2 * kCapacity; ++i) {
+    std::string key;
+    if (type == kLRU) {
+      key = std::to_string(i);
+    } else {
+      key = EncodeKey(static_cast<int>(1000 + i));
+    }
     ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
-                            dumbDeleter));
+                            DumbDeleter));
     ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    key.size() + 5, dumbDeleter));
+                                    key.size() + 5, DumbDeleter));
   }
   ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
   ASSERT_EQ(precise_cache_pinned_usage, precise_cache->GetPinnedUsage());
@@ -447,7 +476,7 @@ TEST_P(CacheTest, EvictionPolicy) {
   Insert(200, 201);
 
   // Frequently used entry must be kept around
-  for (int i = 0; i < kCacheSize * 2; i++) {
+  for (int i = 0; i < 2 * kCacheSize; i++) {
     Insert(1000+i, 2000+i);
     ASSERT_EQ(101, Lookup(100));
   }
@@ -500,9 +529,7 @@ TEST_P(CacheTest, EvictionPolicyRef) {
   Insert(303, 104);
 
   // Insert entries much more than cache capacity.
-  double load_factor =
-      std::min(fast_lru_cache::kLoadFactor, clock_cache::kLoadFactor);
-  for (int i = 0; i < 2 * static_cast<int>(kCacheSize / load_factor); i++) {
+  for (int i = 0; i < 100 * kCacheSize; i++) {
     Insert(1000 + i, 2000 + i);
   }
 
@@ -533,31 +560,40 @@ TEST_P(CacheTest, EvictionPolicyRef) {
 
 TEST_P(CacheTest, EvictEmptyCache) {
   auto type = GetParam();
-  if (type == kFast || type == kClock) {
-    ROCKSDB_GTEST_BYPASS("FastLRUCache and ClockCache require 16-byte keys.");
-    return;
-  }
 
   // Insert item large than capacity to trigger eviction on empty cache.
   auto cache = NewCache(1, 0, false);
-  ASSERT_OK(cache->Insert("foo", nullptr, 10, dumbDeleter));
+  if (type == kLRU) {
+    ASSERT_OK(cache->Insert("foo", nullptr, 10, DumbDeleter));
+  } else {
+    ASSERT_OK(cache->Insert(EncodeKey(1000), nullptr, 10, DumbDeleter));
+  }
 }
 
 TEST_P(CacheTest, EraseFromDeleter) {
   auto type = GetParam();
-  if (type == kFast || type == kClock) {
-    ROCKSDB_GTEST_BYPASS("FastLRUCache and ClockCache require 16-byte keys.");
-    return;
-  }
 
   // Have deleter which will erase item from cache, which will re-enter
   // the cache at that point.
   std::shared_ptr<Cache> cache = NewCache(10, 0, false);
-  ASSERT_OK(cache->Insert("foo", nullptr, 1, dumbDeleter));
-  ASSERT_OK(cache->Insert("bar", cache.get(), 1, eraseDeleter));
-  cache->Erase("bar");
-  ASSERT_EQ(nullptr, cache->Lookup("foo"));
-  ASSERT_EQ(nullptr, cache->Lookup("bar"));
+  std::string foo, bar;
+  Cache::DeleterFn erase_deleter;
+  if (type == kLRU) {
+    foo = "foo";
+    bar = "bar";
+    erase_deleter = EraseDeleter1;
+  } else {
+    foo = EncodeKey(1234);
+    bar = EncodeKey(5678);
+    erase_deleter = EraseDeleter2;
+  }
+
+  ASSERT_OK(cache->Insert(foo, nullptr, 1, DumbDeleter));
+  ASSERT_OK(cache->Insert(bar, cache.get(), 1, erase_deleter));
+
+  cache->Erase(bar);
+  ASSERT_EQ(nullptr, cache->Lookup(foo));
+  ASSERT_EQ(nullptr, cache->Lookup(bar));
 }
 
 TEST_P(CacheTest, ErasedHandleState) {
@@ -590,9 +626,9 @@ TEST_P(CacheTest, HeavyEntries) {
   const int kHeavy = 10;
   int added = 0;
   int index = 0;
-  while (added < 2*kCacheSize) {
+  while (added < 2 * kCacheSize) {
     const int weight = (index & 1) ? kLight : kHeavy;
-    Insert(index, 1000+index, weight);
+    Insert(index, 1000 + index, weight);
     added += weight;
     index++;
   }
@@ -603,7 +639,7 @@ TEST_P(CacheTest, HeavyEntries) {
     int r = Lookup(i);
     if (r >= 0) {
       cached_weight += weight;
-      ASSERT_EQ(1000+i, r);
+      ASSERT_EQ(1000 + i, r);
     }
   }
   ASSERT_LE(cached_weight, kCacheSize + kCacheSize/10);
@@ -615,7 +651,6 @@ TEST_P(CacheTest, NewId) {
   ASSERT_NE(a, b);
 }
 
-
 class Value {
  public:
   explicit Value(int v) : v_(v) {}
@@ -664,7 +699,8 @@ TEST_P(CacheTest, SetCapacity) {
   auto type = GetParam();
   if (type == kFast || type == kClock) {
     ROCKSDB_GTEST_BYPASS(
-        "FastLRUCache and ClockCache don't support capacity adjustments.");
+        "FastLRUCache and ClockCache don't support arbitrary capacity "
+        "adjustments.");
     return;
   }
   // test1: increase capacity
@@ -716,9 +752,9 @@ TEST_P(CacheTest, SetCapacity) {
 
 TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
   auto type = GetParam();
-  if (type == kFast || type == kClock) {
+  if (type == kFast) {
     ROCKSDB_GTEST_BYPASS(
-        "FastLRUCache and ClockCache don't support an unbounded number of "
+        "FastLRUCache only supports a limited number of "
         "inserts beyond "
         "capacity.");
     return;
@@ -775,9 +811,8 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
 
 TEST_P(CacheTest, OverCapacity) {
   auto type = GetParam();
-  if (type == kFast || type == kClock) {
-    ROCKSDB_GTEST_BYPASS(
-        "FastLRUCache and ClockCache don't support capacity adjustments.");
+  if (type == kClock) {
+    ROCKSDB_GTEST_BYPASS("Requires LRU eviction policy.");
     return;
   }
   size_t n = 10;

cache/charged_cache.cc

@@ -0,0 +1,117 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "cache/charged_cache.h"
+
+#include "cache/cache_reservation_manager.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+ChargedCache::ChargedCache(std::shared_ptr<Cache> cache,
+                           std::shared_ptr<Cache> block_cache)
+    : cache_(cache),
+      cache_res_mgr_(std::make_shared<ConcurrentCacheReservationManager>(
+          std::make_shared<
+              CacheReservationManagerImpl<CacheEntryRole::kBlobCache>>(
+              block_cache))) {}
+
+Status ChargedCache::Insert(const Slice& key, void* value, size_t charge,
+                            DeleterFn deleter, Handle** handle,
+                            Priority priority) {
+  Status s = cache_->Insert(key, value, charge, deleter, handle, priority);
+  if (s.ok()) {
+    // Insert may cause the cache entry eviction if the cache is full. So we
+    // directly call the reservation manager to update the total memory used
+    // in the cache.
+    assert(cache_res_mgr_);
+    cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
+        .PermitUncheckedError();
+  }
+  return s;
+}
+
+Status ChargedCache::Insert(const Slice& key, void* value,
+                            const CacheItemHelper* helper, size_t charge,
+                            Handle** handle, Priority priority) {
+  Status s = cache_->Insert(key, value, helper, charge, handle, priority);
+  if (s.ok()) {
+    // Insert may cause the cache entry eviction if the cache is full. So we
+    // directly call the reservation manager to update the total memory used
+    // in the cache.
+    assert(cache_res_mgr_);
+    cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
+        .PermitUncheckedError();
+  }
+  return s;
+}
+
+Cache::Handle* ChargedCache::Lookup(const Slice& key, Statistics* stats) {
+  return cache_->Lookup(key, stats);
+}
+
+Cache::Handle* ChargedCache::Lookup(const Slice& key,
+                                    const CacheItemHelper* helper,
+                                    const CreateCallback& create_cb,
+                                    Priority priority, bool wait,
+                                    Statistics* stats) {
+  auto handle = cache_->Lookup(key, helper, create_cb, priority, wait, stats);
+  // Lookup may promote the KV pair from the secondary cache to the primary
+  // cache. So we directly call the reservation manager to update the total
+  // memory used in the cache.
+  assert(cache_res_mgr_);
+  cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
+      .PermitUncheckedError();
+  return handle;
+}
+
+bool ChargedCache::Release(Cache::Handle* handle, bool useful,
+                           bool erase_if_last_ref) {
+  size_t memory_used_delta = cache_->GetUsage(handle);
+  bool erased = cache_->Release(handle, useful, erase_if_last_ref);
+  if (erased) {
+    assert(cache_res_mgr_);
+    cache_res_mgr_
+        ->UpdateCacheReservation(memory_used_delta, /* increase */ false)
+        .PermitUncheckedError();
+  }
+  return erased;
+}
+
+bool ChargedCache::Release(Cache::Handle* handle, bool erase_if_last_ref) {
+  size_t memory_used_delta = cache_->GetUsage(handle);
+  bool erased = cache_->Release(handle, erase_if_last_ref);
+  if (erased) {
+    assert(cache_res_mgr_);
+    cache_res_mgr_
+        ->UpdateCacheReservation(memory_used_delta, /* increase */ false)
+        .PermitUncheckedError();
+  }
+  return erased;
+}
+
+void ChargedCache::Erase(const Slice& key) {
+  cache_->Erase(key);
+  assert(cache_res_mgr_);
+  cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
+      .PermitUncheckedError();
+}
+
+void ChargedCache::EraseUnRefEntries() {
+  cache_->EraseUnRefEntries();
+  assert(cache_res_mgr_);
+  cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
+      .PermitUncheckedError();
+}
+
+void ChargedCache::SetCapacity(size_t capacity) {
+  cache_->SetCapacity(capacity);
+  // SetCapacity can result in evictions when the cache capacity is decreased,
+  // so we would want to update the cache reservation here as well.
+  assert(cache_res_mgr_);
+  cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
+      .PermitUncheckedError();
+}
+
+}  // namespace ROCKSDB_NAMESPACE

cache/charged_cache.h

@@ -0,0 +1,121 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <string>
+
+#include "port/port.h"
+#include "rocksdb/cache.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+class ConcurrentCacheReservationManager;
+
+// A cache interface which wraps around another cache and takes care of
+// reserving space in block cache towards a single global memory limit, and
+// forwards all the calls to the underlying cache.
+class ChargedCache : public Cache {
+ public:
+  ChargedCache(std::shared_ptr<Cache> cache,
+               std::shared_ptr<Cache> block_cache);
+  ~ChargedCache() override = default;
+
+  Status Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter,
+                Handle** handle, Priority priority) override;
+  Status Insert(const Slice& key, void* value, const CacheItemHelper* helper,
+                size_t charge, Handle** handle = nullptr,
+                Priority priority = Priority::LOW) override;
+
+  Cache::Handle* Lookup(const Slice& key, Statistics* stats) override;
+  Cache::Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
+                        const CreateCallback& create_cb, Priority priority,
+                        bool wait, Statistics* stats = nullptr) override;
+
+  bool Release(Cache::Handle* handle, bool useful,
+               bool erase_if_last_ref = false) override;
+  bool Release(Cache::Handle* handle, bool erase_if_last_ref = false) override;
+
+  void Erase(const Slice& key) override;
+  void EraseUnRefEntries() override;
+
+  static const char* kClassName() { return "ChargedCache"; }
+  const char* Name() const override { return kClassName(); }
+
+  uint64_t NewId() override { return cache_->NewId(); }
+
+  void SetCapacity(size_t capacity) override;
+
+  void SetStrictCapacityLimit(bool strict_capacity_limit) override {
+    cache_->SetStrictCapacityLimit(strict_capacity_limit);
+  }
+
+  bool HasStrictCapacityLimit() const override {
+    return cache_->HasStrictCapacityLimit();
+  }
+
+  void* Value(Cache::Handle* handle) override { return cache_->Value(handle); }
+
+  bool IsReady(Cache::Handle* handle) override {
+    return cache_->IsReady(handle);
+  }
+
+  void Wait(Cache::Handle* handle) override { cache_->Wait(handle); }
+
+  void WaitAll(std::vector<Handle*>& handles) override {
+    cache_->WaitAll(handles);
+  }
+
+  bool Ref(Cache::Handle* handle) override { return cache_->Ref(handle); }
+
+  size_t GetCapacity() const override { return cache_->GetCapacity(); }
+
+  size_t GetUsage() const override { return cache_->GetUsage(); }
+
+  size_t GetUsage(Cache::Handle* handle) const override {
+    return cache_->GetUsage(handle);
+  }
+
+  size_t GetPinnedUsage() const override { return cache_->GetPinnedUsage(); }
+
+  size_t GetCharge(Cache::Handle* handle) const override {
+    return cache_->GetCharge(handle);
+  }
+
+  Cache::DeleterFn GetDeleter(Cache::Handle* handle) const override {
+    return cache_->GetDeleter(handle);
+  }
+
+  void ApplyToAllEntries(
+      const std::function<void(const Slice& key, void* value, size_t charge,
+                               Cache::DeleterFn deleter)>& callback,
+      const Cache::ApplyToAllEntriesOptions& opts) override {
+    cache_->ApplyToAllEntries(callback, opts);
+  }
+
+  void ApplyToAllCacheEntries(void (*callback)(void* value, size_t charge),
+                              bool thread_safe) override {
+    cache_->ApplyToAllCacheEntries(callback, thread_safe);
+  }
+
+  std::string GetPrintableOptions() const override {
+    return cache_->GetPrintableOptions();
+  }
+
+  void DisownData() override { return cache_->DisownData(); }
+
+  inline Cache* GetCache() const { return cache_.get(); }
+
+  inline ConcurrentCacheReservationManager* TEST_GetCacheReservationManager()
+      const {
+    return cache_res_mgr_.get();
+  }
+
+ private:
+  std::shared_ptr<Cache> cache_;
+  std::shared_ptr<ConcurrentCacheReservationManager> cache_res_mgr_;
+};
+
+}  // namespace ROCKSDB_NAMESPACE

cache/clock_cache.cc

@@ -17,7 +17,6 @@
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/statistics.h"
 #include "port/lang.h"
-#include "util/distributed_mutex.h"
 #include "util/hash.h"
 #include "util/math.h"
 #include "util/random.h"
@@ -26,184 +25,358 @@ namespace ROCKSDB_NAMESPACE {
 
 namespace clock_cache {
 
-ClockHandleTable::ClockHandleTable(int hash_bits)
+ClockHandleTable::ClockHandleTable(size_t capacity, int hash_bits)
     : length_bits_(hash_bits),
       length_bits_mask_((uint32_t{1} << length_bits_) - 1),
-      occupancy_(0),
       occupancy_limit_(static_cast<uint32_t>((uint32_t{1} << length_bits_) *
                                              kStrictLoadFactor)),
-      array_(new ClockHandle[size_t{1} << length_bits_]) {
+      capacity_(capacity),
+      array_(new ClockHandle[size_t{1} << length_bits_]),
+      clock_pointer_(0),
+      occupancy_(0),
+      usage_(0) {
   assert(hash_bits <= 32);
 }
 
 ClockHandleTable::~ClockHandleTable() {
-  ApplyToEntriesRange([](ClockHandle* h) { h->FreeData(); }, 0, GetTableSize());
+  // Assumes there are no references (of any type) to any slot in the table.
+  for (uint32_t i = 0; i < GetTableSize(); i++) {
+    ClockHandle* h = &array_[i];
+    if (h->IsElement()) {
+      h->FreeData();
+    }
+  }
 }
 
-ClockHandle* ClockHandleTable::Lookup(const Slice& key) {
-  int probe = 0;
-  int slot = FindVisibleElement(key, probe, 0);
-  return (slot == -1) ? nullptr : &array_[slot];
+ClockHandle* ClockHandleTable::Lookup(const Slice& key, uint32_t hash) {
+  uint32_t probe = 0;
+  ClockHandle* e = FindSlot(
+      key,
+      [&](ClockHandle* h) {
+        if (h->TryInternalRef()) {
+          if (h->IsElement() && h->Matches(key, hash)) {
+            return true;
+          }
+          h->ReleaseInternalRef();
+        }
+        return false;
+      },
+      [&](ClockHandle* h) { return h->displacements == 0; },
+      [&](ClockHandle* /*h*/) {}, probe);
+
+  if (e != nullptr) {
+    // TODO(Guido) Comment from #10347: Here it looks like we have three atomic
+    // updates where it would be possible to combine into one CAS (more metadata
+    // under one atomic field) or maybe two atomic updates (one arithmetic, one
+    // bitwise). Something to think about optimizing.
+    e->SetHit();
+    // The handle is now referenced, so we take it out of clock.
+    ClockOff(e);
+    e->InternalToExternalRef();
+  }
+
+  return e;
 }
 
-ClockHandle* ClockHandleTable::Insert(ClockHandle* h, ClockHandle** old) {
-  int probe = 0;
-  int slot =
-      FindVisibleElementOrAvailableSlot(h->key(), probe, 1 /*displacement*/);
-  *old = nullptr;
-  if (slot == -1) {
+ClockHandle* ClockHandleTable::Insert(ClockHandle* h,
+                                      autovector<ClockHandle>* deleted,
+                                      bool take_reference) {
+  uint32_t probe = 0;
+  ClockHandle* e = FindAvailableSlot(h->key(), h->hash, probe, deleted);
+  if (e == nullptr) {
+    // No available slot to place the handle.
     return nullptr;
   }
 
-  if (array_[slot].IsEmpty() || array_[slot].IsTombstone()) {
-    bool empty = array_[slot].IsEmpty();
-    Assign(slot, h);
-    ClockHandle* new_entry = &array_[slot];
-    if (empty) {
-      // This used to be an empty slot.
-      return new_entry;
-    }
-    // It used to be a tombstone, so there may already be a copy of the
+  // The slot is empty or is a tombstone. And we have an exclusive ref.
+  Assign(e, h);
+  // TODO(Guido) The following RemoveAll can probably be run outside of
+  // the exclusive ref. I had a bad case in mind: multiple inserts could
+  // annihilate each. Although I think this is impossible, I'm not sure
+  // my mental proof covers every case.
+  if (e->displacements != 0) {
+    // It used to be a tombstone, so there may already be copies of the
     // key in the table.
-    slot = FindVisibleElement(h->key(), probe, 0 /*displacement*/);
-    if (slot == -1) {
-      // No existing copy of the key.
-      return new_entry;
-    }
-    *old = &array_[slot];
-    return new_entry;
-  } else {
-    // There is an existing copy of the key.
-    *old = &array_[slot];
-    // Find an available slot for the new element.
-    array_[slot].displacements++;
-    slot = FindAvailableSlot(h->key(), probe, 1 /*displacement*/);
-    if (slot == -1) {
-      // No available slots. Roll back displacements.
-      probe = 0;
-      slot = FindVisibleElement(h->key(), probe, -1);
-      array_[slot].displacements--;
-      FindAvailableSlot(h->key(), probe, -1);
-      return nullptr;
-    }
-    Assign(slot, h);
-    return &array_[slot];
+    RemoveAll(h->key(), h->hash, probe, deleted);
   }
+
+  if (take_reference) {
+    // The user wants to take a reference.
+    e->ExclusiveToExternalRef();
+  } else {
+    // The user doesn't want to immediately take a reference, so we make
+    // it evictable.
+    ClockOn(e);
+    e->ReleaseExclusiveRef();
+  }
+  return e;
 }
 
-void ClockHandleTable::Remove(ClockHandle* h) {
-  assert(!h->IsInClockList());  // Already off the clock list.
-  int probe = 0;
-  FindSlot(
-      h->key(), [&h](ClockHandle* e) { return e == h; }, probe,
-      -1 /*displacement*/);
-  h->SetIsVisible(false);
-  h->SetIsElement(false);
-  occupancy_--;
-}
-
-void ClockHandleTable::Assign(int slot, ClockHandle* h) {
-  ClockHandle* dst = &array_[slot];
-  uint32_t disp = dst->displacements;
-  *dst = *h;
-  dst->displacements = disp;
-  dst->SetIsVisible(true);
+void ClockHandleTable::Assign(ClockHandle* dst, ClockHandle* src) {
+  // DON'T touch displacements and refs.
+  dst->value = src->value;
+  dst->deleter = src->deleter;
+  dst->hash = src->hash;
+  dst->total_charge = src->total_charge;
+  dst->key_data = src->key_data;
+  dst->flags.store(0);
   dst->SetIsElement(true);
-  dst->SetClockPriority(ClockHandle::ClockPriority::NONE);
+  dst->SetCachePriority(src->GetCachePriority());
+  usage_ += dst->total_charge;
   occupancy_++;
 }
 
-void ClockHandleTable::Exclude(ClockHandle* h) { h->SetIsVisible(false); }
-
-int ClockHandleTable::FindVisibleElement(const Slice& key, int& probe,
-                                         int displacement) {
-  return FindSlot(
-      key, [&](ClockHandle* h) { return h->Matches(key) && h->IsVisible(); },
-      probe, displacement);
+bool ClockHandleTable::TryRemove(ClockHandle* h,
+                                 autovector<ClockHandle>* deleted) {
+  if (h->TryExclusiveRef()) {
+    if (h->WillBeDeleted()) {
+      Remove(h, deleted);
+      return true;
+    }
+    h->ReleaseExclusiveRef();
+  }
+  return false;
 }
 
-int ClockHandleTable::FindAvailableSlot(const Slice& key, int& probe,
-                                        int displacement) {
-  return FindSlot(
-      key, [](ClockHandle* h) { return h->IsEmpty() || h->IsTombstone(); },
-      probe, displacement);
+bool ClockHandleTable::SpinTryRemove(ClockHandle* h,
+                                     autovector<ClockHandle>* deleted) {
+  if (h->SpinTryExclusiveRef()) {
+    if (h->WillBeDeleted()) {
+      Remove(h, deleted);
+      return true;
+    }
+    h->ReleaseExclusiveRef();
+  }
+  return false;
 }
 
-int ClockHandleTable::FindVisibleElementOrAvailableSlot(const Slice& key,
-                                                        int& probe,
-                                                        int displacement) {
-  return FindSlot(
+void ClockHandleTable::ClockOff(ClockHandle* h) {
+  h->SetClockPriority(ClockHandle::ClockPriority::NONE);
+}
+
+void ClockHandleTable::ClockOn(ClockHandle* h) {
+  assert(!h->IsInClock());
+  bool is_high_priority =
+      h->HasHit() || h->GetCachePriority() == Cache::Priority::HIGH;
+  h->SetClockPriority(static_cast<ClockHandle::ClockPriority>(
+      is_high_priority ? ClockHandle::ClockPriority::HIGH
+                       : ClockHandle::ClockPriority::MEDIUM));
+}
+
+void ClockHandleTable::Remove(ClockHandle* h,
+                              autovector<ClockHandle>* deleted) {
+  deleted->push_back(*h);
+  ClockOff(h);
+  uint32_t probe = 0;
+  FindSlot(
+      h->key(), [&](ClockHandle* e) { return e == h; },
+      [&](ClockHandle* /*e*/) { return false; },
+      [&](ClockHandle* e) { e->displacements--; }, probe);
+  h->SetWillBeDeleted(false);
+  h->SetIsElement(false);
+}
+
+void ClockHandleTable::RemoveAll(const Slice& key, uint32_t hash,
+                                 uint32_t& probe,
+                                 autovector<ClockHandle>* deleted) {
+  FindSlot(
       key,
       [&](ClockHandle* h) {
-        return h->IsEmpty() || h->IsTombstone() ||
-               (h->Matches(key) && h->IsVisible());
+        if (h->TryInternalRef()) {
+          if (h->IsElement() && h->Matches(key, hash)) {
+            h->SetWillBeDeleted(true);
+            h->ReleaseInternalRef();
+            if (TryRemove(h, deleted)) {
+              h->ReleaseExclusiveRef();
+            }
+            return false;
+          }
+          h->ReleaseInternalRef();
+        }
+        return false;
       },
-      probe, displacement);
+      [&](ClockHandle* h) { return h->displacements == 0; },
+      [&](ClockHandle* /*h*/) {}, probe);
 }
 
-inline int ClockHandleTable::FindSlot(const Slice& key,
-                                      std::function<bool(ClockHandle*)> cond,
-                                      int& probe, int displacement) {
+void ClockHandleTable::Free(autovector<ClockHandle>* deleted) {
+  if (deleted->size() == 0) {
+    // Avoid unnecessarily reading usage_ and occupancy_.
+    return;
+  }
+
+  size_t deleted_charge = 0;
+  for (auto& h : *deleted) {
+    deleted_charge += h.total_charge;
+    h.FreeData();
+  }
+  assert(usage_ >= deleted_charge);
+  usage_ -= deleted_charge;
+  occupancy_ -= static_cast<uint32_t>(deleted->size());
+}
+
+ClockHandle* ClockHandleTable::FindAvailableSlot(
+    const Slice& key, uint32_t hash, uint32_t& probe,
+    autovector<ClockHandle>* deleted) {
+  ClockHandle* e = FindSlot(
+      key,
+      [&](ClockHandle* h) {
+        // To read the handle, first acquire a shared ref.
+        if (h->TryInternalRef()) {
+          if (h->IsElement()) {
+            // The slot is not available.
+            // TODO(Guido) Is it worth testing h->WillBeDeleted()?
+            if (h->WillBeDeleted() || h->Matches(key, hash)) {
+              // The slot can be freed up, or the key we're inserting is already
+              // in the table, so we try to delete it. When the attempt is
+              // successful, the slot becomes available, so we stop probing.
+              // Notice that in that case TryRemove returns an exclusive ref.
+              h->SetWillBeDeleted(true);
+              h->ReleaseInternalRef();
+              if (TryRemove(h, deleted)) {
+                return true;
+              }
+              return false;
+            }
+            h->ReleaseInternalRef();
+            return false;
+          }
+
+          // Available slot.
+          h->ReleaseInternalRef();
+          // Try to acquire an exclusive ref. If we fail, continue probing.
+          if (h->SpinTryExclusiveRef()) {
+            // Check that the slot is still available.
+            if (!h->IsElement()) {
+              return true;
+            }
+            h->ReleaseExclusiveRef();
+          }
+        }
+        return false;
+      },
+      [&](ClockHandle* /*h*/) { return false; },
+      [&](ClockHandle* h) { h->displacements++; }, probe);
+  if (e == nullptr) {
+    Rollback(key, probe);
+  }
+  return e;
+}
+
+ClockHandle* ClockHandleTable::FindSlot(
+    const Slice& key, std::function<bool(ClockHandle*)> match,
+    std::function<bool(ClockHandle*)> abort,
+    std::function<void(ClockHandle*)> update, uint32_t& probe) {
+  // We use double-hashing probing. Every probe in the sequence is a
+  // pseudorandom integer, computed as a linear function of two random hashes,
+  // which we call base and increment. Specifically, the i-th probe is base + i
+  // * increment modulo the table size.
   uint32_t base = ModTableSize(Hash(key.data(), key.size(), kProbingSeed1));
+  // We use an odd increment, which is relatively prime with the power-of-two
+  // table size. This implies that we cycle back to the first probe only
+  // after probing every slot exactly once.
   uint32_t increment =
       ModTableSize((Hash(key.data(), key.size(), kProbingSeed2) << 1) | 1);
   uint32_t current = ModTableSize(base + probe * increment);
   while (true) {
     ClockHandle* h = &array_[current];
-    probe++;
-    if (current == base && probe > 1) {
+    if (current == base && probe > 0) {
       // We looped back.
-      return -1;
+      return nullptr;
     }
-    if (cond(h)) {
-      return current;
+    if (match(h)) {
+      probe++;
+      return h;
     }
-    if (h->IsEmpty()) {
-      // We check emptyness after the condition, because
-      // the condition may be emptyness.
-      return -1;
+    if (abort(h)) {
+      return nullptr;
     }
-    h->displacements += displacement;
+    probe++;
+    update(h);
     current = ModTableSize(current + increment);
   }
 }
 
+void ClockHandleTable::Rollback(const Slice& key, uint32_t probe) {
+  uint32_t current = ModTableSize(Hash(key.data(), key.size(), kProbingSeed1));
+  uint32_t increment =
+      ModTableSize((Hash(key.data(), key.size(), kProbingSeed2) << 1) | 1);
+  for (uint32_t i = 0; i < probe; i++) {
+    array_[current].displacements--;
+    current = ModTableSize(current + increment);
+  }
+}
+
+void ClockHandleTable::ClockRun(size_t charge) {
+  // TODO(Guido) When an element is in the probe sequence of a
+  // hot element, it will be hard to get an exclusive ref.
+  // Do we need a mechanism to prevent an element from sitting
+  // for a long time in cache waiting to be evicted?
+  autovector<ClockHandle> deleted;
+  uint32_t max_iterations =
+      ClockHandle::ClockPriority::HIGH *
+      (1 +
+       static_cast<uint32_t>(
+           GetTableSize() *
+           kLoadFactor));  // It may take up to HIGH passes to evict an element.
+  size_t usage_local = usage_;
+  size_t capacity_local = capacity_;
+  while (usage_local + charge > capacity_local && max_iterations--) {
+    uint32_t steps = 1 + static_cast<uint32_t>(1 / kLoadFactor);
+    uint32_t clock_pointer_local = (clock_pointer_ += steps) - steps;
+    for (uint32_t i = 0; i < steps; i++) {
+      ClockHandle* h = &array_[ModTableSize(clock_pointer_local + i)];
+      if (h->TryExclusiveRef()) {
+        if (h->WillBeDeleted()) {
+          Remove(h, &deleted);
+          usage_local -= h->total_charge;
+        } else {
+          if (!h->IsInClock() && h->IsElement()) {
+            // We adjust the clock priority to make the element evictable again.
+            // Why? Elements that are not in clock are either currently
+            // externally referenced or used to be. Because we are holding an
+            // exclusive ref, we know we are in the latter case. This can only
+            // happen when the last external reference to an element was
+            // released, and the element was not immediately removed.
+            ClockOn(h);
+          }
+          ClockHandle::ClockPriority priority = h->GetClockPriority();
+          if (priority == ClockHandle::ClockPriority::LOW) {
+            Remove(h, &deleted);
+            usage_local -= h->total_charge;
+          } else if (priority > ClockHandle::ClockPriority::LOW) {
+            h->DecreaseClockPriority();
+          }
+        }
+        h->ReleaseExclusiveRef();
+      }
+    }
+  }
+
+  Free(&deleted);
+}
+
 ClockCacheShard::ClockCacheShard(
     size_t capacity, size_t estimated_value_size, bool strict_capacity_limit,
     CacheMetadataChargePolicy metadata_charge_policy)
-    : capacity_(capacity),
-      strict_capacity_limit_(strict_capacity_limit),
-      clock_pointer_(0),
-      table_(
-          CalcHashBits(capacity, estimated_value_size, metadata_charge_policy)),
-      usage_(0),
-      clock_usage_(0) {
+    : strict_capacity_limit_(strict_capacity_limit),
+      detached_usage_(0),
+      table_(capacity, CalcHashBits(capacity, estimated_value_size,
+                                    metadata_charge_policy)) {
   set_metadata_charge_policy(metadata_charge_policy);
 }
 
 void ClockCacheShard::EraseUnRefEntries() {
-  autovector<ClockHandle> last_reference_list;
-  {
-    DMutexLock l(mutex_);
-    uint32_t slot = 0;
-    do {
-      ClockHandle* old = &(table_.array_[slot]);
-      if (!old->IsInClockList()) {
-        continue;
-      }
-      ClockRemove(old);
-      table_.Remove(old);
-      assert(usage_ >= old->total_charge);
-      usage_ -= old->total_charge;
-      last_reference_list.push_back(*old);
-      slot = table_.ModTableSize(slot + 1);
-    } while (slot != 0);
-  }
+  autovector<ClockHandle> deleted;
 
-  // Free the entries here outside of mutex for performance reasons.
-  for (auto& h : last_reference_list) {
-    h.FreeData();
-  }
+  table_.ApplyToEntriesRange(
+      [this, &deleted](ClockHandle* h) {
+        // Externally unreferenced element.
+        table_.Remove(h, &deleted);
+      },
+      0, table_.GetTableSize(), true);
+
+  table_.Free(&deleted);
 }
 
 void ClockCacheShard::ApplyToSomeEntries(
@@ -213,7 +386,6 @@ void ClockCacheShard::ApplyToSomeEntries(
   // The state is essentially going to be the starting hash, which works
   // nicely even if we resize between calls because we use upper-most
   // hash bits for table indexes.
-  DMutexLock l(mutex_);
   uint32_t length_bits = table_.GetLengthBits();
   uint32_t length = table_.GetTableSize();
 
@@ -225,59 +397,29 @@ void ClockCacheShard::ApplyToSomeEntries(
   uint32_t index_begin = *state >> (32 - length_bits);
   uint32_t index_end = index_begin + average_entries_per_lock;
   if (index_end >= length) {
-    // Going to end
+    // Going to end.
     index_end = length;
     *state = UINT32_MAX;
   } else {
     *state = index_end << (32 - length_bits);
   }
 
-  table_.ApplyToEntriesRange(
+  table_.ConstApplyToEntriesRange(
       [callback,
-       metadata_charge_policy = metadata_charge_policy_](ClockHandle* h) {
+       metadata_charge_policy = metadata_charge_policy_](const ClockHandle* h) {
         callback(h->key(), h->value, h->GetCharge(metadata_charge_policy),
                  h->deleter);
       },
-      index_begin, index_end);
+      index_begin, index_end, false);
 }
 
-void ClockCacheShard::ClockRemove(ClockHandle* h) {
-  assert(h->IsInClockList());
-  h->SetClockPriority(ClockHandle::ClockPriority::NONE);
-  assert(clock_usage_ >= h->total_charge);
-  clock_usage_ -= h->total_charge;
-}
-
-void ClockCacheShard::ClockInsert(ClockHandle* h) {
-  assert(!h->IsInClockList());
-  bool is_high_priority =
-      h->HasHit() || h->GetCachePriority() == Cache::Priority::HIGH;
-  h->SetClockPriority(static_cast<ClockHandle::ClockPriority>(
-      is_high_priority * ClockHandle::ClockPriority::HIGH +
-      (1 - is_high_priority) * ClockHandle::ClockPriority::MEDIUM));
-  clock_usage_ += h->total_charge;
-}
-
-void ClockCacheShard::EvictFromClock(size_t charge,
-                                     autovector<ClockHandle>* deleted) {
-  assert(charge <= capacity_);
-  while (clock_usage_ > 0 && (usage_ + charge) > capacity_) {
-    ClockHandle* old = &table_.array_[clock_pointer_];
-    clock_pointer_ = table_.ModTableSize(clock_pointer_ + 1);
-    // Clock list contains only elements which can be evicted.
-    if (!old->IsInClockList()) {
-      continue;
-    }
-    if (old->GetClockPriority() == ClockHandle::ClockPriority::LOW) {
-      ClockRemove(old);
-      table_.Remove(old);
-      assert(usage_ >= old->total_charge);
-      usage_ -= old->total_charge;
-      deleted->push_back(*old);
-      return;
-    }
-    old->DecreaseClockPriority();
-  }
+ClockHandle* ClockCacheShard::DetachedInsert(ClockHandle* h) {
+  ClockHandle* e = new ClockHandle();
+  *e = *h;
+  e->SetDetached();
+  e->TryExternalRef();
+  detached_usage_ += h->total_charge;
+  return e;
 }
 
 size_t ClockCacheShard::CalcEstimatedHandleCharge(
@@ -301,22 +443,14 @@ int ClockCacheShard::CalcHashBits(
 }
 
 void ClockCacheShard::SetCapacity(size_t capacity) {
-  assert(false);  // Not supported. TODO(Guido) Support it?
-  autovector<ClockHandle> last_reference_list;
-  {
-    DMutexLock l(mutex_);
-    capacity_ = capacity;
-    EvictFromClock(0, &last_reference_list);
-  }
-
-  // Free the entries here outside of mutex for performance reasons.
-  for (auto& h : last_reference_list) {
-    h.FreeData();
+  if (capacity > table_.GetCapacity()) {
+    assert(false);  // Not supported.
   }
+  table_.SetCapacity(capacity);
+  table_.ClockRun(detached_usage_);
 }
 
 void ClockCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
-  DMutexLock l(mutex_);
   strict_capacity_limit_ = strict_capacity_limit;
 }
 
@@ -340,189 +474,166 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
   }
 
   Status s = Status::OK();
-  autovector<ClockHandle> last_reference_list;
-  {
-    DMutexLock l(mutex_);
-    assert(table_.GetOccupancy() <= table_.GetOccupancyLimit());
-    // Free the space following strict clock policy until enough space
-    // is freed or the clock list is empty.
-    EvictFromClock(tmp.total_charge, &last_reference_list);
-    if ((usage_ + tmp.total_charge > capacity_ &&
-         (strict_capacity_limit_ || handle == nullptr)) ||
-        table_.GetOccupancy() == table_.GetOccupancyLimit()) {
-      if (handle == nullptr) {
-        // Don't insert the entry but still return ok, as if the entry inserted
-        // into cache and get evicted immediately.
-        last_reference_list.push_back(tmp);
+
+  // Use a local copy to minimize cache synchronization.
+  size_t detached_usage = detached_usage_;
+
+  // Free space with the clock policy until enough space is freed or there are
+  // no evictable elements.
+  table_.ClockRun(tmp.total_charge + detached_usage);
+
+  // Use local copies to minimize cache synchronization
+  // (occupancy_ and usage_ are read and written by all insertions).
+  uint32_t occupancy_local = table_.GetOccupancy();
+  size_t total_usage = table_.GetUsage() + detached_usage;
+
+  // TODO: Currently we support strict_capacity_limit == false as long as the
+  // number of pinned elements is below table_.GetOccupancyLimit(). We can
+  // always support it as follows: whenever we exceed this limit, we dynamically
+  // allocate a handle and return it (when the user provides a handle pointer,
+  // of course). Then, Release checks whether the handle was dynamically
+  // allocated, or is stored in the table.
+  if (total_usage + tmp.total_charge > table_.GetCapacity() &&
+      (strict_capacity_limit_ || handle == nullptr)) {
+    if (handle == nullptr) {
+      // Don't insert the entry but still return ok, as if the entry inserted
+      // into cache and get evicted immediately.
+      tmp.FreeData();
+    } else {
+      if (occupancy_local + 1 > table_.GetOccupancyLimit()) {
+        // TODO: Consider using a distinct status for this case, but usually
+        // it will be handled the same way as reaching charge capacity limit
+        s = Status::MemoryLimit(
+            "Insert failed because all slots in the hash table are full.");
       } else {
-        if (table_.GetOccupancy() == table_.GetOccupancyLimit()) {
-          // TODO: Consider using a distinct status for this case, but usually
-          // it will be handled the same way as reaching charge capacity limit
-          s = Status::MemoryLimit(
-              "Insert failed because all slots in the hash table are full.");
-        } else {
-          s = Status::MemoryLimit(
-              "Insert failed because the total charge has exceeded the "
-              "capacity.");
-        }
+        s = Status::MemoryLimit(
+            "Insert failed because the total charge has exceeded the "
+            "capacity.");
       }
+    }
+  } else {
+    ClockHandle* h = nullptr;
+    if (handle != nullptr && occupancy_local + 1 > table_.GetOccupancyLimit()) {
+      // Even if the user wishes to overload the cache, we can't insert into
+      // the hash table. Instead, we dynamically allocate a new handle.
+      h = DetachedInsert(&tmp);
+      // TODO: Return special status?
     } else {
       // Insert into the cache. Note that the cache might get larger than its
       // capacity if not enough space was freed up.
-      ClockHandle* old;
-      ClockHandle* h = table_.Insert(&tmp, &old);
-      assert(h != nullptr);  // We're below occupancy, so this insertion should
-                             // never fail.
-      usage_ += h->total_charge;
-      if (old != nullptr) {
+      autovector<ClockHandle> deleted;
+      h = table_.Insert(&tmp, &deleted, handle != nullptr);
+      if (h == nullptr && handle != nullptr) {
+        // The table is full. This can happen when many threads simultaneously
+        // attempt an insert, and the table is operating close to full capacity.
+        h = DetachedInsert(&tmp);
+      }
+      // Notice that if handle == nullptr, we don't insert the entry but still
+      // return ok.
+      if (deleted.size() > 0) {
         s = Status::OkOverwritten();
-        assert(old->IsVisible());
-        table_.Exclude(old);
-        if (!old->HasRefs()) {
-          // old is in clock because it's in cache and its reference count is 0.
-          ClockRemove(old);
-          table_.Remove(old);
-          assert(usage_ >= old->total_charge);
-          usage_ -= old->total_charge;
-          last_reference_list.push_back(*old);
-        }
-      }
-      if (handle == nullptr) {
-        ClockInsert(h);
-      } else {
-        // If caller already holds a ref, no need to take one here.
-        if (!h->HasRefs()) {
-          h->Ref();
-        }
-        *handle = reinterpret_cast<Cache::Handle*>(h);
       }
+      table_.Free(&deleted);
+    }
+    if (handle != nullptr) {
+      *handle = reinterpret_cast<Cache::Handle*>(h);
     }
-  }
-
-  // Free the entries here outside of mutex for performance reasons.
-  for (auto& h : last_reference_list) {
-    h.FreeData();
   }
 
   return s;
 }
 
-Cache::Handle* ClockCacheShard::Lookup(const Slice& key, uint32_t /* hash */) {
-  ClockHandle* h = nullptr;
-  {
-    DMutexLock l(mutex_);
-    h = table_.Lookup(key);
-    if (h != nullptr) {
-      assert(h->IsVisible());
-      if (!h->HasRefs()) {
-        // The entry is in clock since it's in the hash table and has no
-        // external references.
-        ClockRemove(h);
-      }
-      h->Ref();
-      h->SetHit();
-    }
-  }
-  return reinterpret_cast<Cache::Handle*>(h);
+Cache::Handle* ClockCacheShard::Lookup(const Slice& key, uint32_t hash) {
+  return reinterpret_cast<Cache::Handle*>(table_.Lookup(key, hash));
 }
 
 bool ClockCacheShard::Ref(Cache::Handle* h) {
   ClockHandle* e = reinterpret_cast<ClockHandle*>(h);
-  DMutexLock l(mutex_);
-  // To create another reference - entry must be already externally referenced.
-  assert(e->HasRefs());
-  e->Ref();
-  return true;
+  assert(e->ExternalRefs() > 0);
+  return e->TryExternalRef();
 }
 
 bool ClockCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
+  // In contrast with LRUCache's Release, this function won't delete the handle
+  // when the cache is above capacity and the reference is the last one. Space
+  // is only freed up by EvictFromClock (called by Insert when space is needed)
+  // and Erase. We do this to avoid an extra atomic read of the variable usage_.
   if (handle == nullptr) {
     return false;
   }
+
   ClockHandle* h = reinterpret_cast<ClockHandle*>(handle);
-  ClockHandle copy;
-  bool last_reference = false;
-  assert(!h->IsInClockList());
-  {
-    DMutexLock l(mutex_);
-    last_reference = h->Unref();
-    if (last_reference && h->IsVisible()) {
-      // The item is still in cache, and nobody else holds a reference to it.
-      if (usage_ > capacity_ || erase_if_last_ref) {
-        // The clock list must be empty since the cache is full.
-        assert(clock_usage_ == 0 || erase_if_last_ref);
-        // Take this opportunity and remove the item.
-        table_.Remove(h);
-      } else {
-        // Put the item back on the clock list, and don't free it.
-        ClockInsert(h);
-        last_reference = false;
-      }
-    }
-    // If it was the last reference, then decrement the cache usage.
-    if (last_reference) {
-      assert(usage_ >= h->total_charge);
-      usage_ -= h->total_charge;
-      copy = *h;
+
+  if (UNLIKELY(h->IsDetached())) {
+    h->ReleaseExternalRef();
+    if (h->TryExclusiveRef()) {
+      // Only the last reference will succeed.
+      // Don't bother releasing the exclusive ref.
+      h->FreeData();
+      detached_usage_ -= h->total_charge;
+      delete h;
+      return true;
     }
+    return false;
   }
 
-  // Free the entry here outside of mutex for performance reasons.
-  if (last_reference) {
-    copy.FreeData();
-  }
-  return last_reference;
-}
+  uint32_t refs = h->refs;
+  bool last_reference = ((refs & ClockHandle::EXTERNAL_REFS) == 1);
+  bool will_be_deleted = refs & ClockHandle::WILL_BE_DELETED;
 
-void ClockCacheShard::Erase(const Slice& key, uint32_t /* hash */) {
-  ClockHandle copy;
-  bool last_reference = false;
-  {
-    DMutexLock l(mutex_);
-    ClockHandle* h = table_.Lookup(key);
-    if (h != nullptr) {
-      table_.Exclude(h);
-      if (!h->HasRefs()) {
-        // The entry is in Clock since it's in cache and has no external
-        // references.
-        ClockRemove(h);
-        table_.Remove(h);
-        assert(usage_ >= h->total_charge);
-        usage_ -= h->total_charge;
-        last_reference = true;
-        copy = *h;
-      }
+  if (last_reference && (will_be_deleted || erase_if_last_ref)) {
+    autovector<ClockHandle> deleted;
+    h->SetWillBeDeleted(true);
+    h->ReleaseExternalRef();
+    if (table_.SpinTryRemove(h, &deleted)) {
+      h->ReleaseExclusiveRef();
+      table_.Free(&deleted);
+      return true;
     }
+  } else {
+    h->ReleaseExternalRef();
   }
-  // Free the entry here outside of mutex for performance reasons.
-  // last_reference will only be true if e != nullptr.
-  if (last_reference) {
-    copy.FreeData();
-  }
+
+  return false;
 }
 
-size_t ClockCacheShard::GetUsage() const {
-  DMutexLock l(mutex_);
-  return usage_;
+void ClockCacheShard::Erase(const Slice& key, uint32_t hash) {
+  autovector<ClockHandle> deleted;
+  uint32_t probe = 0;
+  table_.RemoveAll(key, hash, probe, &deleted);
+  table_.Free(&deleted);
 }
 
+size_t ClockCacheShard::GetUsage() const { return table_.GetUsage(); }
+
 size_t ClockCacheShard::GetPinnedUsage() const {
-  DMutexLock l(mutex_);
-  assert(usage_ >= clock_usage_);
-  return usage_ - clock_usage_;
-}
+  // Computes the pinned usage by scanning the whole hash table. This
+  // is slow, but avoids keeping an exact counter on the clock usage,
+  // i.e., the number of not externally referenced elements.
+  // Why avoid this counter? Because Lookup removes elements from the clock
+  // list, so it would need to update the pinned usage every time,
+  // which creates additional synchronization costs.
+  size_t clock_usage = 0;
 
-std::string ClockCacheShard::GetPrintableOptions() const {
-  return std::string{};
+  table_.ConstApplyToEntriesRange(
+      [&clock_usage](const ClockHandle* h) {
+        if (h->ExternalRefs() > 1) {
+          // We check > 1 because we are holding an external ref.
+          clock_usage += h->total_charge;
+        }
+      },
+      0, table_.GetTableSize(), true);
+
+  return clock_usage + detached_usage_;
 }
 
 ClockCache::ClockCache(size_t capacity, size_t estimated_value_size,
                        int num_shard_bits, bool strict_capacity_limit,
                        CacheMetadataChargePolicy metadata_charge_policy)
-    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
+    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit),
+      num_shards_(1 << num_shard_bits) {
   assert(estimated_value_size > 0 ||
          metadata_charge_policy != kDontChargeCacheMetadata);
-  num_shards_ = 1 << num_shard_bits;
   shards_ = reinterpret_cast<ClockCacheShard*>(
       port::cacheline_aligned_alloc(sizeof(ClockCacheShard) * num_shards_));
   size_t per_shard = (capacity + (num_shards_ - 1)) / num_shards_;
@@ -586,8 +697,10 @@ void ClockCache::DisownData() {
 std::shared_ptr<Cache> NewClockCache(
     size_t capacity, int num_shard_bits, bool strict_capacity_limit,
     CacheMetadataChargePolicy metadata_charge_policy) {
-  return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit, 0.5,
-                     nullptr, kDefaultToAdaptiveMutex, metadata_charge_policy);
+  return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit,
+                     /* high_pri_pool_ratio */ 0.5, nullptr,
+                     kDefaultToAdaptiveMutex, metadata_charge_policy,
+                     /* low_pri_pool_ratio */ 0.0);
 }
 
 std::shared_ptr<Cache> ExperimentalNewClockCache(

cache/clock_cache.h

@@ -9,7 +9,11 @@
 
 #pragma once
 
+#include <sys/types.h>
+
 #include <array>
+#include <atomic>
+#include <cstdint>
 #include <memory>
 #include <string>
 
@@ -21,175 +25,296 @@
 #include "rocksdb/cache.h"
 #include "rocksdb/secondary_cache.h"
 #include "util/autovector.h"
-#include "util/distributed_mutex.h"
 
 namespace ROCKSDB_NAMESPACE {
 
 namespace clock_cache {
 
-// Clock cache implementation. This is based on FastLRUCache's open-addressed
-// hash table. Importantly, it stores elements in an array, and resolves
-// collision using a probing strategy. Visibility and referenceability of
-// elements works as usual. See fast_lru_cache.h for a detailed description.
+// Forward declaration of friend class.
+class ClockCacheTest;
+
+// An experimental alternative to LRUCache, using a lock-free, open-addressed
+// hash table and clock eviction.
+
+// ----------------------------------------------------------------------------
+// 1. INTRODUCTION
 //
-// The main difference with FastLRUCache is, not surprisingly, the eviction
-// algorithm
-// ---instead of an LRU list, we maintain a circular list with the elements
-// available for eviction, which the clock algorithm traverses to pick the next
-// victim. The clock list is represented using the array of handles, and we
-// simply mark those elements that are present in the list. This is done using
-// different clock flags, namely NONE, LOW, MEDIUM, HIGH, that represent
-// priorities: NONE means that the element is not part of the clock list, and
-// LOW to HIGH represent how close an element is from being evictable (LOW being
-// immediately evictable). When the clock pointer steps on an element that is
-// not immediately evictable, it decreases its priority.
+// In RocksDB, a Cache is a concurrent unordered dictionary that supports
+// external references (a.k.a. user references). A ClockCache is a type of Cache
+// that uses the clock algorithm as its eviction policy. Internally, a
+// ClockCache is an open-addressed hash table that stores all KV pairs in a
+// large array. Every slot in the hash table is a ClockHandle, which holds a KV
+// pair plus some additional metadata that controls the different aspects of the
+// cache: external references, the hashing mechanism, concurrent access and the
+// clock algorithm.
+//
+//
+// 2. EXTERNAL REFERENCES
+//
+// An externally referenced handle can't be deleted (either evicted by the clock
+// algorithm, or explicitly deleted) or replaced by a new version (via an insert
+// of the same key) until all external references to it have been released by
+// the users. ClockHandles have two members to support external references:
+// - EXTERNAL_REFS counter: The number of external refs. When EXTERNAL_REFS > 0,
+//    the handle is externally referenced. Updates that intend to modify the
+//    handle will refrain from doing so. Eventually, when all references are
+//    released, we have EXTERNAL_REFS == 0, and updates can operate normally on
+//    the handle.
+// - WILL_BE_DELETED flag: An handle is marked for deletion when an operation
+//    decides the handle should be deleted. This happens either when the last
+//    reference to a handle is released (and the release operation is instructed
+//    to delete on last reference) or on when a delete operation is called on
+//    the item. This flag is needed because an externally referenced handle
+//    can't be immediately deleted. In these cases, the flag will be later read
+//    and acted upon by the eviction algorithm. Importantly, WILL_BE_DELETED is
+//    used not only to defer deletions, but also as a barrier for external
+//    references: once WILL_BE_DELETED is set, lookups (which are the most
+//    common way to acquire new external references) will ignore the handle.
+//    For this reason, when WILL_BE_DELETED is set, we say the handle is
+//    invisible (and, otherwise, that it's visible).
+//
+//
+// 3. HASHING AND COLLISION RESOLUTION
+//
+// ClockCache uses an open-addressed hash table to store the handles.
+// We use a variant of tombstones to manage collisions: every slot keeps a
+// count of how many KV pairs that are currently in the cache have probed the
+// slot in an attempt to insert. Probes are generated with double-hashing
+// (although the code can be easily modified to use other probing schemes, like
+// linear probing).
+//
+// A slot in the hash table can be in a few different states:
+// - Element: The slot contains an element. This is indicated with the
+//    IS_ELEMENT flag. Element can be sub-classified depending on the
+//    value of WILL_BE_DELETED:
+//    * Visible element.
+//    * Invisible element.
+// - Tombstone: The slot doesn't contain an element, but there is some other
+//    element that probed this slot during its insertion.
+// - Empty: The slot is unused---it's neither an element nor a tombstone.
+//
+// A slot cycles through the following sequence of states:
+// empty or tombstone --> visible element --> invisible element -->
+// empty or tombstone. Initially a slot is available---it's either
+// empty or a tombstone. As soon as a KV pair is written into the slot, it
+// becomes a visible element. At some point, the handle will be deleted
+// by an explicit delete operation, the eviction algorithm, or an overwriting
+// insert. In either case, the handle is marked for deletion. When the an
+// attempt to delete the element finally succeeds, the slot is freed up
+// and becomes available again.
+//
+//
+// 4. CONCURRENCY
+//
+// ClockCache is lock-free. At a high level, we synchronize the operations
+// using a read-prioritized, non-blocking variant of RW locks on every slot of
+// the hash table. To do this we generalize the concept of reference:
+// - Internal reference: Taken by a thread that is attempting to read a slot
+//    or do a very precise type of update.
+// - Exclusive reference: Taken by a thread that is attempting to write a
+//    a slot extensively.
+//
+// We defer the precise definitions to the comments in the code below.
+// A crucial feature of our references is that attempting to take one never
+// blocks the thread. Another important feature is that readers are
+// prioritized, as they use extremely fast synchronization primitives---they
+// use atomic arithmetic/bit operations, but no compare-and-swaps (which are
+// much slower).
+//
+// Internal references are used by threads to read slots during a probing
+// sequence, making them the most common references (probing is performed
+// in almost every operation, not just lookups). During a lookup, once
+// the target element is found, and just before the handle is handed over
+// to the user, an internal reference is converted into an external reference.
+// During an update operation, once the target slot is found, an internal
+// reference is converted into an exclusive reference. Interestingly, we
+// can't atomically upgrade from internal to exclusive, or we may run into a
+// deadlock. Releasing the internal reference and then taking an exclusive
+// reference avoids the deadlock, but then the handle may change inbetween.
+// One of the key observations we use in our implementation is that we can
+// make up for this lack of atomicity using IS_ELEMENT and WILL_BE_DELETED.
+//
+// Distinguishing internal from external references is useful for two reasons:
+// - Internal references are short lived, but external references are typically
+//    not. This is helpful when acquiring an exclusive ref: if there are any
+//    external references to the item, it's probably not worth waiting until
+//    they go away.
+// - We can precisely determine when there are no more external references to a
+//    handle, and proceed to mark it for deletion. This is useful when users
+//    release external references.
+//
+//
+// 5. CLOCK ALGORITHM
+//
+// The clock algorithm circularly sweeps through the hash table to find the next
+// victim. Recall that handles that are referenced are not evictable; the clock
+// algorithm never picks those. We use different clock priorities: NONE, LOW,
+// MEDIUM and HIGH. Priorities LOW, MEDIUM and HIGH represent how close an
+// element is from being evicted, LOW being the closest to evicted. NONE means
+// the slot is not evictable. NONE priority is used in one of the following
+// cases:
+// (a) the slot doesn't contain an element, or
+// (b) the slot contains an externally referenced element, or
+// (c) the slot contains an element that used to be externally referenced,
+//      and the clock pointer has not swept through the slot since the element
+//      stopped being externally referenced.
+// ----------------------------------------------------------------------------
 
-constexpr double kLoadFactor = 0.35;  // See fast_lru_cache.h.
+// The load factor p is a real number in (0, 1) such that at all
+// times at most a fraction p of all slots, without counting tombstones,
+// are occupied by elements. This means that the probability that a
+// random probe hits an empty slot is at most p, and thus at most 1/p probes
+// are required on average. For example, p = 70% implies that between 1 and 2
+// probes are needed on average (bear in mind that this reasoning doesn't
+// consider the effects of clustering over time).
+// Because the size of the hash table is always rounded up to the next
+// power of 2, p is really an upper bound on the actual load factor---the
+// actual load factor is anywhere between p/2 and p. This is a bit wasteful,
+// but bear in mind that slots only hold metadata, not actual values.
+// Since space cost is dominated by the values (the LSM blocks),
+// overprovisioning the table with metadata only increases the total cache space
+// usage by a tiny fraction.
+constexpr double kLoadFactor = 0.35;
 
-constexpr double kStrictLoadFactor = 0.7;  // See fast_lru_cache.h.
+// The user can exceed kLoadFactor if the sizes of the inserted values don't
+// match estimated_value_size, or if strict_capacity_limit == false. To
+// avoid a performance drop, we set a strict upper bound on the load factor.
+constexpr double kStrictLoadFactor = 0.7;
+
+// Maximum number of spins when trying to acquire a ref.
+// TODO(Guido) This value was set arbitrarily. Is it appropriate?
+// What's the best way to bound the spinning?
+constexpr uint32_t kSpinsPerTry = 100000;
 
 // Arbitrary seeds.
 constexpr uint32_t kProbingSeed1 = 0xbc9f1d34;
 constexpr uint32_t kProbingSeed2 = 0x7a2bb9d5;
 
-// An experimental (under development!) alternative to LRUCache
-
 struct ClockHandle {
   void* value;
   Cache::DeleterFn deleter;
   uint32_t hash;
-  size_t total_charge;  // TODO(opt): Only allow uint32_t?
-  // The number of external refs to this entry.
-  uint32_t refs;
-
-  static constexpr int kIsVisibleOffset = 0;
-  static constexpr int kIsElementOffset = 1;
-  static constexpr int kClockPriorityOffset = 2;
-  static constexpr int kIsHitOffset = 4;
-  static constexpr int kCachePriorityOffset = 5;
-
-  enum Flags : uint8_t {
-    // Whether the handle is visible to Lookups.
-    IS_VISIBLE = (1 << kIsVisibleOffset),
-    // Whether the slot is in use by an element.
-    IS_ELEMENT = (1 << kIsElementOffset),
-    // Clock priorities. Represents how close a handle is from
-    // being evictable.
-    CLOCK_PRIORITY = (3 << kClockPriorityOffset),
-    // Whether the handle has been looked up after its insertion.
-    HAS_HIT = (1 << kIsHitOffset),
-    CACHE_PRIORITY = (1 << kCachePriorityOffset),
-  };
-  uint8_t flags;
-
-  enum ClockPriority : uint8_t {
-    NONE = (0 << kClockPriorityOffset),  // Not an element in the eyes of clock.
-    LOW = (1 << kClockPriorityOffset),   // Immediately evictable.
-    MEDIUM = (2 << kClockPriorityOffset),
-    HIGH = (3 << kClockPriorityOffset)
-    // Priority is NONE if and only if
-    // (i) the handle is not an element, or
-    // (ii) the handle is an element but it is being referenced.
-  };
-
-  // The number of elements that hash to this slot or a lower one,
-  // but wind up in a higher slot.
-  uint32_t displacements;
-
+  size_t total_charge;
   std::array<char, kCacheKeySize> key_data;
 
-  ClockHandle() {
-    value = nullptr;
-    deleter = nullptr;
-    hash = 0;
-    total_charge = 0;
-    refs = 0;
-    flags = 0;
-    SetIsVisible(false);
+  static constexpr uint8_t kIsElementOffset = 0;
+  static constexpr uint8_t kClockPriorityOffset = 1;
+  static constexpr uint8_t kIsHitOffset = 3;
+  static constexpr uint8_t kCachePriorityOffset = 4;
+
+  enum Flags : uint8_t {
+    // Whether the slot is in use by an element.
+    IS_ELEMENT = 1 << kIsElementOffset,
+    // Clock priorities. Represents how close a handle is from being evictable.
+    CLOCK_PRIORITY = 3 << kClockPriorityOffset,
+    // Whether the handle has been looked up after its insertion.
+    HAS_HIT = 1 << kIsHitOffset,
+    // The value of Cache::Priority of the handle.
+    CACHE_PRIORITY = 1 << kCachePriorityOffset,
+  };
+
+  std::atomic<uint8_t> flags;
+
+  enum ClockPriority : uint8_t {
+    NONE = (0 << kClockPriorityOffset),
+    LOW = (1 << kClockPriorityOffset),
+    MEDIUM = (2 << kClockPriorityOffset),
+    HIGH = (3 << kClockPriorityOffset)
+  };
+
+  // The number of elements that hash to this slot or a lower one, but wind
+  // up in this slot or a higher one.
+  std::atomic<uint32_t> displacements;
+
+  static constexpr uint8_t kExternalRefsOffset = 0;
+  static constexpr uint8_t kSharedRefsOffset = 15;
+  static constexpr uint8_t kExclusiveRefOffset = 30;
+  static constexpr uint8_t kWillBeDeletedOffset = 31;
+
+  enum Refs : uint32_t {
+    // Synchronization model:
+    // - An external reference guarantees that hash, value, key_data
+    //    and the IS_ELEMENT flag are not modified. Doesn't allow
+    //    any writes.
+    // - An internal reference has the same guarantees as an
+    //    external reference, and additionally allows the following
+    //    idempotent updates on the handle:
+    //      * set CLOCK_PRIORITY to NONE;
+    //      * set the HAS_HIT bit;
+    //      * set the WILL_BE_DELETED bit.
+    // - A shared reference is either an external reference or an
+    //    internal reference.
+    // - An exclusive reference guarantees that no other thread has a shared
+    //    or exclusive reference to the handle, and allows writes
+    //    on the handle.
+
+    // Number of external references to the slot.
+    EXTERNAL_REFS = ((uint32_t{1} << 15) - 1)
+                    << kExternalRefsOffset,  // Bits 0, ..., 14
+    // Number of internal references plus external references to the slot.
+    SHARED_REFS = ((uint32_t{1} << 15) - 1)
+                  << kSharedRefsOffset,  // Bits 15, ..., 29
+    // Whether a thread has an exclusive reference to the slot.
+    EXCLUSIVE_REF = uint32_t{1} << kExclusiveRefOffset,  // Bit 30
+    // Whether the handle will be deleted soon. When this bit is set, new
+    // internal references to this handle stop being accepted.
+    // External references may still be granted---they can be created from
+    // existing external references, or converting from existing internal
+    // references.
+    WILL_BE_DELETED = uint32_t{1} << kWillBeDeletedOffset  // Bit 31
+
+    // Having these 4 fields in a single variable allows us to support the
+    // following operations efficiently:
+    // - Convert an internal reference into an external reference in a single
+    //    atomic arithmetic operation.
+    // - Attempt to take a shared reference using a single atomic arithmetic
+    //    operation. This is because we can increment the internal ref count
+    //    as well as checking whether the entry is marked for deletion using a
+    //    single atomic arithmetic operation (and one non-atomic comparison).
+  };
+
+  static constexpr uint32_t kOneInternalRef = 0x8000;
+  static constexpr uint32_t kOneExternalRef = 0x8001;
+
+  std::atomic<uint32_t> refs;
+
+  // True iff the handle is allocated separately from hash table.
+  bool detached;
+
+  ClockHandle()
+      : value(nullptr),
+        deleter(nullptr),
+        hash(0),
+        total_charge(0),
+        flags(0),
+        displacements(0),
+        refs(0),
+        detached(false) {
+    SetWillBeDeleted(false);
     SetIsElement(false);
     SetClockPriority(ClockPriority::NONE);
     SetCachePriority(Cache::Priority::LOW);
-    displacements = 0;
     key_data.fill(0);
   }
 
+  // The copy ctor and assignment operator are only used to copy a handle
+  // for immediate deletion. (We need to copy because the slot may become
+  // re-used before the deletion is completed.) We only copy the necessary
+  // members to carry out the deletion. In particular, we don't need
+  // the atomic members.
+  ClockHandle(const ClockHandle& other) { *this = other; }
+
+  void operator=(const ClockHandle& other) {
+    value = other.value;
+    deleter = other.deleter;
+    key_data = other.key_data;
+    hash = other.hash;
+    total_charge = other.total_charge;
+  }
+
   Slice key() const { return Slice(key_data.data(), kCacheKeySize); }
 
-  // Increase the reference count by 1.
-  void Ref() { refs++; }
-
-  // Just reduce the reference count by 1. Return true if it was last reference.
-  bool Unref() {
-    assert(refs > 0);
-    refs--;
-    return refs == 0;
-  }
-
-  // Return true if there are external refs, false otherwise.
-  bool HasRefs() const { return refs > 0; }
-
-  bool IsVisible() const { return flags & IS_VISIBLE; }
-
-  void SetIsVisible(bool is_visible) {
-    if (is_visible) {
-      flags |= IS_VISIBLE;
-    } else {
-      flags &= ~IS_VISIBLE;
-    }
-  }
-
-  bool IsElement() const { return flags & IS_ELEMENT; }
-
-  void SetIsElement(bool is_element) {
-    if (is_element) {
-      flags |= IS_ELEMENT;
-    } else {
-      flags &= ~IS_ELEMENT;
-    }
-  }
-
-  bool HasHit() const { return flags & HAS_HIT; }
-
-  void SetHit() { flags |= HAS_HIT; }
-
-  bool IsInClockList() const {
-    return GetClockPriority() != ClockHandle::ClockPriority::NONE;
-  }
-
-  Cache::Priority GetCachePriority() const {
-    return static_cast<Cache::Priority>(flags & CACHE_PRIORITY);
-  }
-
-  void SetCachePriority(Cache::Priority priority) {
-    if (priority == Cache::Priority::HIGH) {
-      flags |= Flags::CACHE_PRIORITY;
-    } else {
-      flags &= ~Flags::CACHE_PRIORITY;
-    }
-  }
-
-  ClockPriority GetClockPriority() const {
-    return static_cast<ClockPriority>(flags & Flags::CLOCK_PRIORITY);
-  }
-
-  void SetClockPriority(ClockPriority priority) {
-    flags &= ~Flags::CLOCK_PRIORITY;
-    flags |= priority;
-  }
-
-  void DecreaseClockPriority() {
-    uint8_t p = static_cast<uint8_t>(flags & Flags::CLOCK_PRIORITY) >>
-                kClockPriorityOffset;
-    assert(p > 0);
-    p--;
-    flags &= ~Flags::CLOCK_PRIORITY;
-    ClockPriority new_priority =
-        static_cast<ClockPriority>(p << kClockPriorityOffset);
-    flags |= new_priority;
-  }
-
   void FreeData() {
-    assert(refs == 0);
     if (deleter) {
       (*deleter)(key(), value);
     }
@@ -232,51 +357,254 @@ struct ClockHandle {
     return total_charge - meta_charge;
   }
 
-  inline bool IsEmpty() {
+  // flags functions.
+
+  bool IsElement() const { return flags & Flags::IS_ELEMENT; }
+
+  void SetIsElement(bool is_element) {
+    if (is_element) {
+      flags |= Flags::IS_ELEMENT;
+    } else {
+      flags &= static_cast<uint8_t>(~Flags::IS_ELEMENT);
+    }
+  }
+
+  bool HasHit() const { return flags & HAS_HIT; }
+
+  void SetHit() { flags |= HAS_HIT; }
+
+  Cache::Priority GetCachePriority() const {
+    return static_cast<Cache::Priority>(flags & CACHE_PRIORITY);
+  }
+
+  void SetCachePriority(Cache::Priority priority) {
+    if (priority == Cache::Priority::HIGH) {
+      flags |= Flags::CACHE_PRIORITY;
+    } else {
+      flags &= static_cast<uint8_t>(~Flags::CACHE_PRIORITY);
+    }
+  }
+
+  bool IsInClock() const {
+    return GetClockPriority() != ClockHandle::ClockPriority::NONE;
+  }
+
+  ClockPriority GetClockPriority() const {
+    return static_cast<ClockPriority>(flags & Flags::CLOCK_PRIORITY);
+  }
+
+  void SetClockPriority(ClockPriority priority) {
+    flags &= static_cast<uint8_t>(~Flags::CLOCK_PRIORITY);
+    flags |= priority;
+  }
+
+  void DecreaseClockPriority() {
+    uint8_t p = static_cast<uint8_t>(flags & Flags::CLOCK_PRIORITY) >>
+                kClockPriorityOffset;
+    assert(p > 0);
+    p--;
+    flags &= static_cast<uint8_t>(~Flags::CLOCK_PRIORITY);
+    ClockPriority new_priority =
+        static_cast<ClockPriority>(p << kClockPriorityOffset);
+    flags |= new_priority;
+  }
+
+  bool IsDetached() { return detached; }
+
+  void SetDetached() { detached = true; }
+
+  inline bool IsEmpty() const {
     return !this->IsElement() && this->displacements == 0;
   }
 
-  inline bool IsTombstone() {
+  inline bool IsTombstone() const {
     return !this->IsElement() && this->displacements > 0;
   }
 
-  inline bool Matches(const Slice& some_key) {
-    return this->IsElement() && this->key() == some_key;
+  inline bool Matches(const Slice& some_key, uint32_t some_hash) const {
+    return this->hash == some_hash && this->key() == some_key;
   }
+
+  // refs functions.
+
+  inline bool WillBeDeleted() const { return refs & WILL_BE_DELETED; }
+
+  void SetWillBeDeleted(bool will_be_deleted) {
+    if (will_be_deleted) {
+      refs |= WILL_BE_DELETED;
+    } else {
+      refs &= ~WILL_BE_DELETED;
+    }
+  }
+
+  uint32_t ExternalRefs() const {
+    return (refs & EXTERNAL_REFS) >> kExternalRefsOffset;
+  }
+
+  // Tries to take an internal ref. Returns true iff it succeeds.
+  inline bool TryInternalRef() {
+    if (!((refs += kOneInternalRef) & (EXCLUSIVE_REF | WILL_BE_DELETED))) {
+      return true;
+    }
+    refs -= kOneInternalRef;
+    return false;
+  }
+
+  // Tries to take an external ref. Returns true iff it succeeds.
+  inline bool TryExternalRef() {
+    if (!((refs += kOneExternalRef) & EXCLUSIVE_REF)) {
+      return true;
+    }
+    refs -= kOneExternalRef;
+    return false;
+  }
+
+  // Tries to take an exclusive ref. Returns true iff it succeeds.
+  // TODO(Guido) After every TryExclusiveRef call, we always call
+  // WillBeDeleted(). We could save an atomic read by having an output parameter
+  // with the last value of refs.
+  inline bool TryExclusiveRef() {
+    uint32_t will_be_deleted = refs & WILL_BE_DELETED;
+    uint32_t expected = will_be_deleted;
+    return refs.compare_exchange_strong(expected,
+                                        EXCLUSIVE_REF | will_be_deleted);
+  }
+
+  // Repeatedly tries to take an exclusive reference, but aborts as soon
+  // as an external or exclusive reference is detected (since the wait
+  // would presumably be too long).
+  inline bool SpinTryExclusiveRef() {
+    uint32_t expected = 0;
+    uint32_t will_be_deleted = 0;
+    uint32_t spins = kSpinsPerTry;
+    while (!refs.compare_exchange_strong(expected,
+                                         EXCLUSIVE_REF | will_be_deleted) &&
+           spins--) {
+      std::this_thread::yield();
+      if (expected & (EXTERNAL_REFS | EXCLUSIVE_REF)) {
+        return false;
+      }
+      will_be_deleted = expected & WILL_BE_DELETED;
+      expected = will_be_deleted;
+    }
+    return true;
+  }
+
+  // Take an external ref, assuming there is already one external ref
+  // to the handle.
+  void Ref() {
+    // TODO(Guido) Is it okay to assume that the existing external reference
+    // survives until this function returns?
+    refs += kOneExternalRef;
+  }
+
+  inline void ReleaseExternalRef() { refs -= kOneExternalRef; }
+
+  inline void ReleaseInternalRef() { refs -= kOneInternalRef; }
+
+  inline void ReleaseExclusiveRef() { refs.fetch_and(~EXCLUSIVE_REF); }
+
+  // Downgrade an exclusive ref to external.
+  inline void ExclusiveToExternalRef() {
+    refs += kOneExternalRef;
+    ReleaseExclusiveRef();
+  }
+
+  // Convert an internal ref into external.
+  inline void InternalToExternalRef() {
+    refs += kOneExternalRef - kOneInternalRef;
+  }
+
 };  // struct ClockHandle
 
 class ClockHandleTable {
  public:
-  explicit ClockHandleTable(int hash_bits);
+  explicit ClockHandleTable(size_t capacity, int hash_bits);
   ~ClockHandleTable();
 
-  // Returns a pointer to a visible element matching the key/hash, or
-  // nullptr if not present.
-  ClockHandle* Lookup(const Slice& key);
+  // Returns a pointer to a visible handle matching the key/hash, or
+  // nullptr if not present. When an actual handle is produced, an
+  // internal reference is handed over.
+  ClockHandle* Lookup(const Slice& key, uint32_t hash);
 
-  // Inserts a copy of h into the hash table.
-  // Returns a pointer to the inserted handle, or nullptr if no slot
-  // available was found. If an existing visible element matching the
-  // key/hash is already present in the hash table, the argument old
-  // is set to pointe to it; otherwise, it's set to nullptr.
-  ClockHandle* Insert(ClockHandle* h, ClockHandle** old);
+  // Inserts a copy of h into the hash table. Returns a pointer to the
+  // inserted handle, or nullptr if no available slot was found. Every
+  // existing visible handle matching the key is already present in the
+  // hash table is marked as WILL_BE_DELETED. The deletion is also attempted,
+  // and, if the attempt is successful, the handle is inserted into the
+  // autovector deleted. When take_reference is true, the function hands
+  // over an external reference on the handle, and otherwise no reference is
+  // produced.
+  ClockHandle* Insert(ClockHandle* h, autovector<ClockHandle>* deleted,
+                      bool take_reference);
 
-  // Removes h from the hash table. The handle must already be off
-  // the clock list.
-  void Remove(ClockHandle* h);
+  // Assigns h the appropriate clock priority, making it evictable.
+  void ClockOn(ClockHandle* h);
 
-  // Turns a visible element h into a ghost (i.e., not visible).
-  void Exclude(ClockHandle* h);
+  // Makes h non-evictable.
+  void ClockOff(ClockHandle* h);
 
-  // Assigns a copy of h to the given slot.
-  void Assign(int slot, ClockHandle* h);
+  // Runs the clock eviction algorithm until usage_ + charge is at most
+  // capacity_.
+  void ClockRun(size_t charge);
 
-  template <typename T>
-  void ApplyToEntriesRange(T func, uint32_t index_begin, uint32_t index_end) {
+  // Remove h from the hash table. Requires an exclusive ref to h.
+  void Remove(ClockHandle* h, autovector<ClockHandle>* deleted);
+
+  // Remove from the hash table all handles with matching key/hash along a
+  // probe sequence, starting from the given probe number. Doesn't
+  // require any references.
+  void RemoveAll(const Slice& key, uint32_t hash, uint32_t& probe,
+                 autovector<ClockHandle>* deleted);
+
+  void RemoveAll(const Slice& key, uint32_t hash,
+                 autovector<ClockHandle>* deleted) {
+    uint32_t probe = 0;
+    RemoveAll(key, hash, probe, deleted);
+  }
+
+  // Tries to remove h from the hash table. If the attempt is successful,
+  // the function hands over an exclusive ref to h.
+  bool TryRemove(ClockHandle* h, autovector<ClockHandle>* deleted);
+
+  // Similar to TryRemove, except that it spins, increasing the chances of
+  // success. Requires that the caller thread has no shared ref to h.
+  bool SpinTryRemove(ClockHandle* h, autovector<ClockHandle>* deleted);
+
+  // Call this function after an Insert, Remove, RemoveAll, TryRemove
+  // or SpinTryRemove. It frees the deleted values and updates the hash table
+  // metadata.
+  void Free(autovector<ClockHandle>* deleted);
+
+  void ApplyToEntriesRange(std::function<void(ClockHandle*)> func,
+                           uint32_t index_begin, uint32_t index_end,
+                           bool apply_if_will_be_deleted) {
     for (uint32_t i = index_begin; i < index_end; i++) {
       ClockHandle* h = &array_[i];
-      if (h->IsVisible()) {
-        func(h);
+      if (h->TryExclusiveRef()) {
+        if (h->IsElement() &&
+            (apply_if_will_be_deleted || !h->WillBeDeleted())) {
+          func(h);
+        }
+        h->ReleaseExclusiveRef();
+      }
+    }
+  }
+
+  void ConstApplyToEntriesRange(std::function<void(const ClockHandle*)> func,
+                                uint32_t index_begin, uint32_t index_end,
+                                bool apply_if_will_be_deleted) const {
+    for (uint32_t i = index_begin; i < index_end; i++) {
+      ClockHandle* h = &array_[i];
+      // We take an external ref because we are handing over control
+      // to a user-defined function, and because the handle will not be
+      // modified.
+      if (h->TryExternalRef()) {
+        if (h->IsElement() &&
+            (apply_if_will_be_deleted || !h->WillBeDeleted())) {
+          func(h);
+        }
+        h->ReleaseExternalRef();
       }
     }
   }
@@ -289,43 +617,83 @@ class ClockHandleTable {
 
   uint32_t GetOccupancy() const { return occupancy_; }
 
+  size_t GetUsage() const { return usage_; }
+
+  size_t GetCapacity() const { return capacity_; }
+
+  void SetCapacity(size_t capacity) { capacity_ = capacity; }
+
   // Returns x mod 2^{length_bits_}.
   uint32_t ModTableSize(uint32_t x) { return x & length_bits_mask_; }
 
  private:
-  friend class ClockCacheShard;
+  // Extracts the element information from a handle (src), and assigns it
+  // to a hash table slot (dst). Doesn't touch displacements and refs,
+  // which are maintained by the hash table algorithm.
+  void Assign(ClockHandle* dst, ClockHandle* src);
 
-  int FindVisibleElement(const Slice& key, int& probe, int displacement);
+  // Returns the first slot in the probe sequence, starting from the given
+  // probe number, with a handle e such that match(e) is true. At every
+  // step, the function first tests whether match(e) holds. If this is false,
+  // it evaluates abort(e) to decide whether the search should be aborted,
+  // and in the affirmative returns -1. For every handle e probed except
+  // the last one, the function runs update(e).
+  // The probe parameter is modified as follows. We say a probe to a handle
+  // e is aborting if match(e) is false and abort(e) is true. Then the final
+  // value of probe is one more than the last non-aborting probe during the
+  // call. This is so that that the variable can be used to keep track of
+  // progress across consecutive calls to FindSlot.
+  inline ClockHandle* FindSlot(const Slice& key,
+                               std::function<bool(ClockHandle*)> match,
+                               std::function<bool(ClockHandle*)> stop,
+                               std::function<void(ClockHandle*)> update,
+                               uint32_t& probe);
 
-  int FindAvailableSlot(const Slice& key, int& probe, int displacement);
+  // Returns an available slot for the given key. All copies of the
+  // key found along the probing sequence until an available slot is
+  // found are marked for deletion. On each of them, a deletion is
+  // attempted, and when the attempt succeeds the slot is assigned to
+  // the new copy of the element.
+  ClockHandle* FindAvailableSlot(const Slice& key, uint32_t hash,
+                                 uint32_t& probe,
+                                 autovector<ClockHandle>* deleted);
 
-  int FindVisibleElementOrAvailableSlot(const Slice& key, int& probe,
-                                        int displacement);
-
-  // Returns the index of the first slot probed (hashing with
-  // the given key) with a handle e such that cond(e) is true.
-  // Otherwise, if no match is found, returns -1.
-  // For every handle e probed except the final slot, updates
-  // e->displacements += displacement.
-  // The argument probe is modified such that consecutive calls
-  // to FindSlot continue probing right after where the previous
-  // call left.
-  int FindSlot(const Slice& key, std::function<bool(ClockHandle*)> cond,
-               int& probe, int displacement);
+  // After a failed FindSlot call (i.e., with answer -1) in
+  // FindAvailableSlot, this function fixes all displacements's
+  // starting from the 0-th probe, until the given probe.
+  void Rollback(const Slice& key, uint32_t probe);
 
   // Number of hash bits used for table index.
   // The size of the table is 1 << length_bits_.
-  int length_bits_;
+  const int length_bits_;
 
+  // For faster computation of ModTableSize.
   const uint32_t length_bits_mask_;
 
-  // Number of elements in the table.
-  uint32_t occupancy_;
-
   // Maximum number of elements the user can store in the table.
-  uint32_t occupancy_limit_;
+  const uint32_t occupancy_limit_;
 
+  // Maximum total charge of all elements stored in the table.
+  size_t capacity_;
+
+  // We partition the following members into different cache lines
+  // to avoid false sharing among Lookup, Release, Erase and Insert
+  // operations in ClockCacheShard.
+
+  ALIGN_AS(CACHE_LINE_SIZE)
+  // Array of slots comprising the hash table.
   std::unique_ptr<ClockHandle[]> array_;
+
+  ALIGN_AS(CACHE_LINE_SIZE)
+  // Clock algorithm sweep pointer.
+  std::atomic<uint32_t> clock_pointer_;
+
+  ALIGN_AS(CACHE_LINE_SIZE)
+  // Number of elements in the table.
+  std::atomic<uint32_t> occupancy_;
+
+  // Memory size for entries residing in the cache.
+  std::atomic<size_t> usage_;
 };  // class ClockHandleTable
 
 // A single shard of sharded cache.
@@ -345,10 +713,10 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShard {
   void SetStrictCapacityLimit(bool strict_capacity_limit) override;
 
   // Like Cache methods, but with an extra "hash" parameter.
-  // Insert an item into the hash table and, if handle is null, insert into
-  // the clock list. Older items are evicted as necessary. If the cache is full
-  // and free_handle_on_fail is true, the item is deleted and handle is set to
-  // nullptr.
+  // Insert an item into the hash table and, if handle is null, make it
+  // evictable by the clock algorithm. Older items are evicted as necessary.
+  // If the cache is full and free_handle_on_fail is true, the item is deleted
+  // and handle is set to nullptr.
   Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
                 Cache::DeleterFn deleter, Cache::Handle** handle,
                 Cache::Priority priority) override;
@@ -366,20 +734,26 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShard {
                         Statistics* /*stats*/) override {
     return Lookup(key, hash);
   }
+
   Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
 
   bool Release(Cache::Handle* handle, bool /*useful*/,
                bool erase_if_last_ref) override {
     return Release(handle, erase_if_last_ref);
   }
+
   bool IsReady(Cache::Handle* /*handle*/) override { return true; }
+
   void Wait(Cache::Handle* /*handle*/) override {}
 
   bool Ref(Cache::Handle* handle) override;
+
   bool Release(Cache::Handle* handle, bool erase_if_last_ref = false) override;
+
   void Erase(const Slice& key, uint32_t hash) override;
 
   size_t GetUsage() const override;
+
   size_t GetPinnedUsage() const override;
 
   void ApplyToSomeEntries(
@@ -389,18 +763,13 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShard {
 
   void EraseUnRefEntries() override;
 
-  std::string GetPrintableOptions() const override;
+  std::string GetPrintableOptions() const override { return std::string{}; }
 
  private:
   friend class ClockCache;
-  void ClockRemove(ClockHandle* e);
-  void ClockInsert(ClockHandle* e);
+  friend class ClockCacheTest;
 
-  // Free some space following strict clock policy until enough space
-  // to hold (usage_ + charge) is freed or the clock list is empty
-  // This function is not thread safe - it needs to be executed while
-  // holding the mutex_.
-  void EvictFromClock(size_t charge, autovector<ClockHandle>* deleted);
+  ClockHandle* DetachedInsert(ClockHandle* h);
 
   // Returns the charge of a single handle.
   static size_t CalcEstimatedHandleCharge(
@@ -412,37 +781,13 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShard {
   static int CalcHashBits(size_t capacity, size_t estimated_value_size,
                           CacheMetadataChargePolicy metadata_charge_policy);
 
-  // Initialized before use.
-  size_t capacity_;
-
   // Whether to reject insertion if cache reaches its full capacity.
-  bool strict_capacity_limit_;
+  std::atomic<bool> strict_capacity_limit_;
 
-  uint32_t clock_pointer_;
+  // Handles allocated separately from the table.
+  std::atomic<size_t> detached_usage_;
 
-  // ------------^^^^^^^^^^^^^-----------
-  // Not frequently modified data members
-  // ------------------------------------
-  //
-  // We separate data members that are updated frequently from the ones that
-  // are not frequently updated so that they don't share the same cache line
-  // which will lead into false cache sharing
-  //
-  // ------------------------------------
-  // Frequently modified data members
-  // ------------vvvvvvvvvvvvv-----------
   ClockHandleTable table_;
-
-  // Memory size for entries residing in the cache.
-  size_t usage_;
-
-  // Memory size for unpinned entries in the clock list.
-  size_t clock_usage_;
-
-  // mutex_ protects the following state.
-  // We don't count mutex_ as the cache's internal state so semantically we
-  // don't mind mutex_ invoking the non-const actions.
-  mutable DMutex mutex_;
 };  // class ClockCacheShard
 
 class ClockCache
@@ -455,19 +800,29 @@ class ClockCache
              bool strict_capacity_limit,
              CacheMetadataChargePolicy metadata_charge_policy =
                  kDontChargeCacheMetadata);
+
   ~ClockCache() override;
+
   const char* Name() const override { return "ClockCache"; }
+
   CacheShard* GetShard(uint32_t shard) override;
+
   const CacheShard* GetShard(uint32_t shard) const override;
+
   void* Value(Handle* handle) override;
+
   size_t GetCharge(Handle* handle) const override;
+
   uint32_t GetHash(Handle* handle) const override;
+
   DeleterFn GetDeleter(Handle* handle) const override;
+
   void DisownData() override;
 
  private:
   ClockCacheShard* shards_ = nullptr;
-  int num_shards_ = 0;
+
+  int num_shards_;
 };  // class ClockCache
 
 }  // namespace clock_cache

cache/compressed_secondary_cache.cc

@@ -5,6 +5,8 @@
 
 #include "cache/compressed_secondary_cache.h"
 
+#include <algorithm>
+#include <cstdint>
 #include <memory>
 
 #include "memory/memory_allocator.h"
@@ -13,28 +15,20 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-namespace {
-
-void DeletionCallback(const Slice& /*key*/, void* obj) {
-  delete reinterpret_cast<CacheAllocationPtr*>(obj);
-  obj = nullptr;
-}
-
-}  // namespace
-
 CompressedSecondaryCache::CompressedSecondaryCache(
     size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-    double high_pri_pool_ratio,
+    double high_pri_pool_ratio, double low_pri_pool_ratio,
     std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
     CacheMetadataChargePolicy metadata_charge_policy,
     CompressionType compression_type, uint32_t compress_format_version)
     : cache_options_(capacity, num_shard_bits, strict_capacity_limit,
                      high_pri_pool_ratio, memory_allocator, use_adaptive_mutex,
                      metadata_charge_policy, compression_type,
-                     compress_format_version) {
-  cache_ = NewLRUCache(capacity, num_shard_bits, strict_capacity_limit,
-                       high_pri_pool_ratio, memory_allocator,
-                       use_adaptive_mutex, metadata_charge_policy);
+                     compress_format_version, low_pri_pool_ratio) {
+  cache_ =
+      NewLRUCache(capacity, num_shard_bits, strict_capacity_limit,
+                  high_pri_pool_ratio, memory_allocator, use_adaptive_mutex,
+                  metadata_charge_policy, low_pri_pool_ratio);
 }
 
 CompressedSecondaryCache::~CompressedSecondaryCache() { cache_.reset(); }
@@ -49,26 +43,29 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
     return handle;
   }
 
-  CacheAllocationPtr* ptr =
-      reinterpret_cast<CacheAllocationPtr*>(cache_->Value(lru_handle));
-  void* value = nullptr;
-  size_t charge = 0;
-  Status s;
+  CacheValueChunk* handle_value =
+      reinterpret_cast<CacheValueChunk*>(cache_->Value(lru_handle));
+  size_t handle_value_charge{0};
+  CacheAllocationPtr merged_value =
+      MergeChunksIntoValue(handle_value, handle_value_charge);
 
+  Status s;
+  void* value{nullptr};
+  size_t charge{0};
   if (cache_options_.compression_type == kNoCompression) {
-    s = create_cb(ptr->get(), cache_->GetCharge(lru_handle), &value, &charge);
+    s = create_cb(merged_value.get(), handle_value_charge, &value, &charge);
   } else {
     UncompressionContext uncompression_context(cache_options_.compression_type);
     UncompressionInfo uncompression_info(uncompression_context,
                                          UncompressionDict::GetEmptyDict(),
                                          cache_options_.compression_type);
 
-    size_t uncompressed_size = 0;
+    size_t uncompressed_size{0};
     CacheAllocationPtr uncompressed;
-    uncompressed = UncompressData(
-        uncompression_info, (char*)ptr->get(), cache_->GetCharge(lru_handle),
-        &uncompressed_size, cache_options_.compress_format_version,
-        cache_options_.memory_allocator.get());
+    uncompressed = UncompressData(uncompression_info, (char*)merged_value.get(),
+                                  handle_value_charge, &uncompressed_size,
+                                  cache_options_.compress_format_version,
+                                  cache_options_.memory_allocator.get());
 
     if (!uncompressed) {
       cache_->Release(lru_handle, /* erase_if_last_ref */ true);
@@ -104,7 +101,7 @@ Status CompressedSecondaryCache::Insert(const Slice& key, void* value,
   if (cache_options_.compression_type != kNoCompression) {
     CompressionOptions compression_opts;
     CompressionContext compression_context(cache_options_.compression_type);
-    uint64_t sample_for_compression = 0;
+    uint64_t sample_for_compression{0};
     CompressionInfo compression_info(
         compression_opts, compression_context, CompressionDict::GetEmptyDict(),
         cache_options_.compression_type, sample_for_compression);
@@ -118,14 +115,12 @@ Status CompressedSecondaryCache::Insert(const Slice& key, void* value,
     }
 
     val = Slice(compressed_val);
-    size = compressed_val.size();
-    ptr = AllocateBlock(size, cache_options_.memory_allocator.get());
-    memcpy(ptr.get(), compressed_val.data(), size);
   }
 
-  CacheAllocationPtr* buf = new CacheAllocationPtr(std::move(ptr));
-
-  return cache_->Insert(key, buf, size, DeletionCallback);
+  size_t charge{0};
+  CacheValueChunk* value_chunks_head =
+      SplitValueIntoChunks(val, cache_options_.compression_type, charge);
+  return cache_->Insert(key, value_chunks_head, charge, DeletionCallback);
 }
 
 void CompressedSecondaryCache::Erase(const Slice& key) { cache_->Erase(key); }
@@ -133,7 +128,7 @@ void CompressedSecondaryCache::Erase(const Slice& key) { cache_->Erase(key); }
 std::string CompressedSecondaryCache::GetPrintableOptions() const {
   std::string ret;
   ret.reserve(20000);
-  const int kBufferSize = 200;
+  const int kBufferSize{200};
   char buffer[kBufferSize];
   ret.append(cache_->GetPrintableOptions());
   snprintf(buffer, kBufferSize, "    compression_type : %s\n",
@@ -145,16 +140,98 @@ std::string CompressedSecondaryCache::GetPrintableOptions() const {
   return ret;
 }
 
+CompressedSecondaryCache::CacheValueChunk*
+CompressedSecondaryCache::SplitValueIntoChunks(
+    const Slice& value, const CompressionType compression_type,
+    size_t& charge) {
+  assert(!value.empty());
+  const char* src_ptr = value.data();
+  size_t src_size{value.size()};
+
+  CacheValueChunk dummy_head = CacheValueChunk();
+  CacheValueChunk* current_chunk = &dummy_head;
+  // Do not split when value size is large or there is no compression.
+  size_t predicted_chunk_size{0};
+  size_t actual_chunk_size{0};
+  size_t tmp_size{0};
+  while (src_size > 0) {
+    predicted_chunk_size = sizeof(CacheValueChunk) - 1 + src_size;
+    auto upper =
+        std::upper_bound(malloc_bin_sizes_.begin(), malloc_bin_sizes_.end(),
+                         predicted_chunk_size);
+    // Do not split when value size is too small, too large, close to a bin
+    // size, or there is no compression.
+    if (upper == malloc_bin_sizes_.begin() ||
+        upper == malloc_bin_sizes_.end() ||
+        *upper - predicted_chunk_size < malloc_bin_sizes_.front() ||
+        compression_type == kNoCompression) {
+      tmp_size = predicted_chunk_size;
+    } else {
+      tmp_size = *(--upper);
+    }
+
+    CacheValueChunk* new_chunk =
+        reinterpret_cast<CacheValueChunk*>(new char[tmp_size]);
+    current_chunk->next = new_chunk;
+    current_chunk = current_chunk->next;
+    actual_chunk_size = tmp_size - sizeof(CacheValueChunk) + 1;
+    memcpy(current_chunk->data, src_ptr, actual_chunk_size);
+    current_chunk->size = actual_chunk_size;
+    src_ptr += actual_chunk_size;
+    src_size -= actual_chunk_size;
+    charge += tmp_size;
+  }
+  current_chunk->next = nullptr;
+
+  return dummy_head.next;
+}
+
+CacheAllocationPtr CompressedSecondaryCache::MergeChunksIntoValue(
+    const void* chunks_head, size_t& charge) {
+  const CacheValueChunk* head =
+      reinterpret_cast<const CacheValueChunk*>(chunks_head);
+  const CacheValueChunk* current_chunk = head;
+  charge = 0;
+  while (current_chunk != nullptr) {
+    charge += current_chunk->size;
+    current_chunk = current_chunk->next;
+  }
+
+  CacheAllocationPtr ptr =
+      AllocateBlock(charge, cache_options_.memory_allocator.get());
+  current_chunk = head;
+  size_t pos{0};
+  while (current_chunk != nullptr) {
+    memcpy(ptr.get() + pos, current_chunk->data, current_chunk->size);
+    pos += current_chunk->size;
+    current_chunk = current_chunk->next;
+  }
+
+  return ptr;
+}
+
+void CompressedSecondaryCache::DeletionCallback(const Slice& /*key*/,
+                                                void* obj) {
+  CacheValueChunk* chunks_head = reinterpret_cast<CacheValueChunk*>(obj);
+  while (chunks_head != nullptr) {
+    CacheValueChunk* tmp_chunk = chunks_head;
+    chunks_head = chunks_head->next;
+    tmp_chunk->Free();
+  }
+  obj = nullptr;
+}
+
 std::shared_ptr<SecondaryCache> NewCompressedSecondaryCache(
     size_t capacity, int num_shard_bits, bool strict_capacity_limit,
     double high_pri_pool_ratio,
     std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
     CacheMetadataChargePolicy metadata_charge_policy,
-    CompressionType compression_type, uint32_t compress_format_version) {
+    CompressionType compression_type, uint32_t compress_format_version,
+    double low_pri_pool_ratio) {
   return std::make_shared<CompressedSecondaryCache>(
       capacity, num_shard_bits, strict_capacity_limit, high_pri_pool_ratio,
-      memory_allocator, use_adaptive_mutex, metadata_charge_policy,
-      compression_type, compress_format_version);
+      low_pri_pool_ratio, memory_allocator, use_adaptive_mutex,
+      metadata_charge_policy, compression_type, compress_format_version);
 }
 
 std::shared_ptr<SecondaryCache> NewCompressedSecondaryCache(
@@ -165,7 +242,7 @@ std::shared_ptr<SecondaryCache> NewCompressedSecondaryCache(
       opts.capacity, opts.num_shard_bits, opts.strict_capacity_limit,
       opts.high_pri_pool_ratio, opts.memory_allocator, opts.use_adaptive_mutex,
       opts.metadata_charge_policy, opts.compression_type,
-      opts.compress_format_version);
+      opts.compress_format_version, opts.low_pri_pool_ratio);
 }
 
 }  // namespace ROCKSDB_NAMESPACE

cache/compressed_secondary_cache.h

@@ -5,6 +5,8 @@
 
 #pragma once
 
+#include <array>
+#include <cstddef>
 #include <memory>
 
 #include "cache/lru_cache.h"
@@ -54,11 +56,11 @@ class CompressedSecondaryCache : public SecondaryCache {
  public:
   CompressedSecondaryCache(
       size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-      double high_pri_pool_ratio,
+      double high_pri_pool_ratio, double low_pri_pool_ratio,
       std::shared_ptr<MemoryAllocator> memory_allocator = nullptr,
       bool use_adaptive_mutex = kDefaultToAdaptiveMutex,
       CacheMetadataChargePolicy metadata_charge_policy =
-          kDontChargeCacheMetadata,
+          kDefaultCacheMetadataChargePolicy,
       CompressionType compression_type = CompressionType::kLZ4Compression,
       uint32_t compress_format_version = 2);
   virtual ~CompressedSecondaryCache() override;
@@ -79,6 +81,36 @@ class CompressedSecondaryCache : public SecondaryCache {
   std::string GetPrintableOptions() const override;
 
  private:
+  friend class CompressedSecondaryCacheTest;
+  static constexpr std::array<uint16_t, 33> malloc_bin_sizes_{
+      32,   64,   96,   128,  160,  192,  224,   256,   320,   384,   448,
+      512,  640,  768,  896,  1024, 1280, 1536,  1792,  2048,  2560,  3072,
+      3584, 4096, 5120, 6144, 7168, 8192, 10240, 12288, 14336, 16384, 32768};
+
+  struct CacheValueChunk {
+    // TODO try "CacheAllocationPtr next;".
+    CacheValueChunk* next;
+    size_t size;
+    // Beginning of the chunk data (MUST BE THE LAST FIELD IN THIS STRUCT!)
+    char data[1];
+
+    void Free() { delete[] reinterpret_cast<char*>(this); }
+  };
+
+  // Split value into chunks to better fit into jemalloc bins. The chunks
+  // are stored in CacheValueChunk and extra charge is needed for each chunk,
+  // so the cache charge is recalculated here.
+  CacheValueChunk* SplitValueIntoChunks(const Slice& value,
+                                        const CompressionType compression_type,
+                                        size_t& charge);
+
+  // After merging chunks, the extra charge for each chunk is removed, so
+  // the charge is recalculated.
+  CacheAllocationPtr MergeChunksIntoValue(const void* chunks_head,
+                                          size_t& charge);
+
+  // An implementation of Cache::DeleterFn.
+  static void DeletionCallback(const Slice& /*key*/, void* obj);
   std::shared_ptr<Cache> cache_;
   CompressedSecondaryCacheOptions cache_options_;
 };

cache/compressed_secondary_cache_test.cc

@@ -7,9 +7,12 @@
 
 #include <algorithm>
 #include <cstdint>
+#include <iterator>
 
+#include "cache/lru_cache.h"
 #include "memory/jemalloc_nodump_allocator.h"
 #include "memory/memory_allocator.h"
+#include "rocksdb/compression_type.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/secondary_cache.h"
 #include "test_util/testharness.h"
@@ -136,7 +139,6 @@ class CompressedSecondaryCacheTest : public testing::Test {
     CompressedSecondaryCacheOptions opts;
     opts.capacity = 2048;
     opts.num_shard_bits = 0;
-    opts.metadata_charge_policy = kDontChargeCacheMetadata;
 
     if (sec_cache_is_compressed) {
       if (!LZ4_Supported()) {
@@ -162,6 +164,8 @@ class CompressedSecondaryCacheTest : public testing::Test {
     }
     std::shared_ptr<SecondaryCache> sec_cache =
         NewCompressedSecondaryCache(opts);
+
+    BasicTestHelper(sec_cache);
   }
 
   void FailsTest(bool sec_cache_is_compressed) {
@@ -177,7 +181,6 @@ class CompressedSecondaryCacheTest : public testing::Test {
 
     secondary_cache_opts.capacity = 1100;
     secondary_cache_opts.num_shard_bits = 0;
-    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
     std::shared_ptr<SecondaryCache> sec_cache =
         NewCompressedSecondaryCache(secondary_cache_opts);
 
@@ -235,34 +238,37 @@ class CompressedSecondaryCacheTest : public testing::Test {
 
     secondary_cache_opts.capacity = 2300;
     secondary_cache_opts.num_shard_bits = 0;
-    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
     std::shared_ptr<SecondaryCache> secondary_cache =
         NewCompressedSecondaryCache(secondary_cache_opts);
-    LRUCacheOptions lru_cache_opts(1024, 0, false, 0.5, nullptr,
-                                   kDefaultToAdaptiveMutex,
-                                   kDontChargeCacheMetadata);
+    LRUCacheOptions lru_cache_opts(
+        1300 /* capacity */, 0 /* num_shard_bits */,
+        false /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
+        nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
+        kDefaultCacheMetadataChargePolicy);
     lru_cache_opts.secondary_cache = secondary_cache;
     std::shared_ptr<Cache> cache = NewLRUCache(lru_cache_opts);
     std::shared_ptr<Statistics> stats = CreateDBStatistics();
 
     Random rnd(301);
 
-    std::string str1 = rnd.RandomString(1010);
+    std::string str1;
+    test::CompressibleString(&rnd, 0.5, 1001, &str1);
     std::string str1_clone{str1};
     TestItem* item1 = new TestItem(str1.data(), str1.length());
     ASSERT_OK(cache->Insert("k1", item1, &CompressedSecondaryCacheTest::helper_,
                             str1.length()));
 
-    std::string str2 = rnd.RandomString(1020);
+    std::string str2;
+    test::CompressibleString(&rnd, 0.5, 1012, &str2);
     TestItem* item2 = new TestItem(str2.data(), str2.length());
-    // After Insert, lru cache contains k2 and secondary cache contains k1.
+    // After Insert, cache contains k2 and secondary cache contains k1.
     ASSERT_OK(cache->Insert("k2", item2, &CompressedSecondaryCacheTest::helper_,
                             str2.length()));
 
-    std::string str3 = rnd.RandomString(1020);
+    std::string str3;
+    test::CompressibleString(&rnd, 0.5, 1024, &str3);
     TestItem* item3 = new TestItem(str3.data(), str3.length());
-    // After Insert, lru cache contains k3 and secondary cache contains k1 and
-    // k2
+    // After Insert, cache contains k3 and secondary cache contains k1 and k2.
     ASSERT_OK(cache->Insert("k3", item3, &CompressedSecondaryCacheTest::helper_,
                             str3.length()));
 
@@ -287,7 +293,6 @@ class CompressedSecondaryCacheTest : public testing::Test {
     handle = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_,
                            test_item_creator, Cache::Priority::LOW, true,
                            stats.get());
-
     ASSERT_NE(handle, nullptr);
     TestItem* val1_1 = static_cast<TestItem*>(cache->Value(handle));
     ASSERT_NE(val1_1, nullptr);
@@ -316,19 +321,22 @@ class CompressedSecondaryCacheTest : public testing::Test {
       secondary_cache_opts.compression_type = CompressionType::kNoCompression;
     }
 
-    secondary_cache_opts.capacity = 2048;
+    secondary_cache_opts.capacity = 2300;
     secondary_cache_opts.num_shard_bits = 0;
-    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
     std::shared_ptr<SecondaryCache> secondary_cache =
         NewCompressedSecondaryCache(secondary_cache_opts);
 
-    LRUCacheOptions opts(1024, 0, false, 0.5, nullptr, kDefaultToAdaptiveMutex,
-                         kDontChargeCacheMetadata);
+    LRUCacheOptions opts(
+        1024 /* capacity */, 0 /* num_shard_bits */,
+        false /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
+        nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
+        kDefaultCacheMetadataChargePolicy);
     opts.secondary_cache = secondary_cache;
     std::shared_ptr<Cache> cache = NewLRUCache(opts);
 
     Random rnd(301);
-    std::string str1 = rnd.RandomString(1020);
+    std::string str1;
+    test::CompressibleString(&rnd, 0.5, 1001, &str1);
     auto item1 =
         std::unique_ptr<TestItem>(new TestItem(str1.data(), str1.length()));
     ASSERT_NOK(cache->Insert("k1", item1.get(), nullptr, str1.length()));
@@ -361,25 +369,30 @@ class CompressedSecondaryCacheTest : public testing::Test {
       secondary_cache_opts.compression_type = CompressionType::kNoCompression;
     }
 
-    secondary_cache_opts.capacity = 2048;
+    secondary_cache_opts.capacity = 2300;
     secondary_cache_opts.num_shard_bits = 0;
-    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
 
     std::shared_ptr<SecondaryCache> secondary_cache =
         NewCompressedSecondaryCache(secondary_cache_opts);
 
-    LRUCacheOptions opts(1024, 0, false, 0.5, nullptr, kDefaultToAdaptiveMutex,
-                         kDontChargeCacheMetadata);
+    LRUCacheOptions opts(
+        1200 /* capacity */, 0 /* num_shard_bits */,
+        false /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
+        nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
+        kDefaultCacheMetadataChargePolicy);
     opts.secondary_cache = secondary_cache;
     std::shared_ptr<Cache> cache = NewLRUCache(opts);
 
     Random rnd(301);
-    std::string str1 = rnd.RandomString(1020);
+    std::string str1;
+    test::CompressibleString(&rnd, 0.5, 1001, &str1);
     TestItem* item1 = new TestItem(str1.data(), str1.length());
     ASSERT_OK(cache->Insert("k1", item1,
                             &CompressedSecondaryCacheTest::helper_fail_,
                             str1.length()));
-    std::string str2 = rnd.RandomString(1020);
+
+    std::string str2;
+    test::CompressibleString(&rnd, 0.5, 1002, &str2);
     TestItem* item2 = new TestItem(str2.data(), str2.length());
     // k1 should be demoted to the secondary cache.
     ASSERT_OK(cache->Insert("k2", item2,
@@ -417,25 +430,29 @@ class CompressedSecondaryCacheTest : public testing::Test {
       secondary_cache_opts.compression_type = CompressionType::kNoCompression;
     }
 
-    secondary_cache_opts.capacity = 2048;
+    secondary_cache_opts.capacity = 2300;
     secondary_cache_opts.num_shard_bits = 0;
-    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
 
     std::shared_ptr<SecondaryCache> secondary_cache =
         NewCompressedSecondaryCache(secondary_cache_opts);
 
-    LRUCacheOptions opts(1024, 0, false, 0.5, nullptr, kDefaultToAdaptiveMutex,
-                         kDontChargeCacheMetadata);
+    LRUCacheOptions opts(
+        1200 /* capacity */, 0 /* num_shard_bits */,
+        false /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
+        nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
+        kDefaultCacheMetadataChargePolicy);
     opts.secondary_cache = secondary_cache;
     std::shared_ptr<Cache> cache = NewLRUCache(opts);
 
     Random rnd(301);
-    std::string str1 = rnd.RandomString(1020);
+    std::string str1;
+    test::CompressibleString(&rnd, 0.5, 1001, &str1);
     TestItem* item1 = new TestItem(str1.data(), str1.length());
     ASSERT_OK(cache->Insert("k1", item1, &CompressedSecondaryCacheTest::helper_,
                             str1.length()));
 
-    std::string str2 = rnd.RandomString(1020);
+    std::string str2;
+    test::CompressibleString(&rnd, 0.5, 1002, &str2);
     TestItem* item2 = new TestItem(str2.data(), str2.length());
     // k1 should be demoted to the secondary cache.
     ASSERT_OK(cache->Insert("k2", item2, &CompressedSecondaryCacheTest::helper_,
@@ -473,24 +490,30 @@ class CompressedSecondaryCacheTest : public testing::Test {
       secondary_cache_opts.compression_type = CompressionType::kNoCompression;
     }
 
-    secondary_cache_opts.capacity = 2048;
+    secondary_cache_opts.capacity = 2300;
     secondary_cache_opts.num_shard_bits = 0;
-    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
 
     std::shared_ptr<SecondaryCache> secondary_cache =
         NewCompressedSecondaryCache(secondary_cache_opts);
 
-    LRUCacheOptions opts(1024, 0, /*_strict_capacity_limit=*/true, 0.5, nullptr,
-                         kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
+    LRUCacheOptions opts(
+        1200 /* capacity */, 0 /* num_shard_bits */,
+        true /* strict_capacity_limit */, 0.5 /* high_pri_pool_ratio */,
+        nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
+        kDefaultCacheMetadataChargePolicy);
     opts.secondary_cache = secondary_cache;
     std::shared_ptr<Cache> cache = NewLRUCache(opts);
 
     Random rnd(301);
-    std::string str1 = rnd.RandomString(1020);
+    std::string str1;
+    test::CompressibleString(&rnd, 0.5, 1001, &str1);
     TestItem* item1 = new TestItem(str1.data(), str1.length());
     ASSERT_OK(cache->Insert("k1", item1, &CompressedSecondaryCacheTest::helper_,
                             str1.length()));
-    std::string str2 = rnd.RandomString(1020);
+
+    std::string str2;
+    test::CompressibleString(&rnd, 0.5, 1002, &str2);
+    std::string str2_clone{str2};
     TestItem* item2 = new TestItem(str2.data(), str2.length());
     // k1 should be demoted to the secondary cache.
     ASSERT_OK(cache->Insert("k2", item2, &CompressedSecondaryCacheTest::helper_,
@@ -501,8 +524,9 @@ class CompressedSecondaryCacheTest : public testing::Test {
                             test_item_creator, Cache::Priority::LOW, true);
     ASSERT_NE(handle2, nullptr);
     cache->Release(handle2);
-    // k1 promotion should fail due to the block cache being at capacity,
-    // but the lookup should still succeed
+
+    // k1 promotion should fail because cache is at capacity and
+    // strict_capacity_limit is true, but the lookup should still succeed.
     Cache::Handle* handle1;
     handle1 = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_,
                             test_item_creator, Cache::Priority::LOW, true);
@@ -519,6 +543,119 @@ class CompressedSecondaryCacheTest : public testing::Test {
     secondary_cache.reset();
   }
 
+  void SplitValueIntoChunksTest() {
+    JemallocAllocatorOptions jopts;
+    std::shared_ptr<MemoryAllocator> allocator;
+    std::string msg;
+    if (JemallocNodumpAllocator::IsSupported(&msg)) {
+      Status s = NewJemallocNodumpAllocator(jopts, &allocator);
+      if (!s.ok()) {
+        ROCKSDB_GTEST_BYPASS("JEMALLOC not supported");
+      }
+    } else {
+      ROCKSDB_GTEST_BYPASS("JEMALLOC not supported");
+    }
+
+    using CacheValueChunk = CompressedSecondaryCache::CacheValueChunk;
+    std::unique_ptr<CompressedSecondaryCache> sec_cache =
+        std::make_unique<CompressedSecondaryCache>(1000, 0, true, 0.5, 0.0,
+                                                   allocator);
+    Random rnd(301);
+    // 10000 = 8169 + 1769 + 62 , so there should be 3 chunks after split.
+    size_t str_size{10000};
+    std::string str = rnd.RandomString(static_cast<int>(str_size));
+    size_t charge{0};
+    CacheValueChunk* chunks_head =
+        sec_cache->SplitValueIntoChunks(str, kLZ4Compression, charge);
+    ASSERT_EQ(charge, str_size + 3 * (sizeof(CacheValueChunk) - 1));
+
+    CacheValueChunk* current_chunk = chunks_head;
+    ASSERT_EQ(current_chunk->size, 8192 - sizeof(CacheValueChunk) + 1);
+    current_chunk = current_chunk->next;
+    ASSERT_EQ(current_chunk->size, 1792 - sizeof(CacheValueChunk) + 1);
+    current_chunk = current_chunk->next;
+    ASSERT_EQ(current_chunk->size, 62);
+
+    sec_cache->DeletionCallback("dummy", chunks_head);
+  }
+
+  void MergeChunksIntoValueTest() {
+    using CacheValueChunk = CompressedSecondaryCache::CacheValueChunk;
+    Random rnd(301);
+    size_t size1{2048};
+    std::string str1 = rnd.RandomString(static_cast<int>(size1));
+    CacheValueChunk* current_chunk = reinterpret_cast<CacheValueChunk*>(
+        new char[sizeof(CacheValueChunk) - 1 + size1]);
+    CacheValueChunk* chunks_head = current_chunk;
+    memcpy(current_chunk->data, str1.data(), size1);
+    current_chunk->size = size1;
+
+    size_t size2{256};
+    std::string str2 = rnd.RandomString(static_cast<int>(size2));
+    current_chunk->next = reinterpret_cast<CacheValueChunk*>(
+        new char[sizeof(CacheValueChunk) - 1 + size2]);
+    current_chunk = current_chunk->next;
+    memcpy(current_chunk->data, str2.data(), size2);
+    current_chunk->size = size2;
+
+    size_t size3{31};
+    std::string str3 = rnd.RandomString(static_cast<int>(size3));
+    current_chunk->next = reinterpret_cast<CacheValueChunk*>(
+        new char[sizeof(CacheValueChunk) - 1 + size3]);
+    current_chunk = current_chunk->next;
+    memcpy(current_chunk->data, str3.data(), size3);
+    current_chunk->size = size3;
+    current_chunk->next = nullptr;
+
+    std::string str = str1 + str2 + str3;
+
+    std::unique_ptr<CompressedSecondaryCache> sec_cache =
+        std::make_unique<CompressedSecondaryCache>(1000, 0, true, 0.5, 0.0);
+    size_t charge{0};
+    CacheAllocationPtr value =
+        sec_cache->MergeChunksIntoValue(chunks_head, charge);
+    ASSERT_EQ(charge, size1 + size2 + size3);
+    std::string value_str{value.get(), charge};
+    ASSERT_EQ(strcmp(value_str.data(), str.data()), 0);
+
+    sec_cache->DeletionCallback("dummy", chunks_head);
+  }
+
+  void SplictValueAndMergeChunksTest() {
+    JemallocAllocatorOptions jopts;
+    std::shared_ptr<MemoryAllocator> allocator;
+    std::string msg;
+    if (JemallocNodumpAllocator::IsSupported(&msg)) {
+      Status s = NewJemallocNodumpAllocator(jopts, &allocator);
+      if (!s.ok()) {
+        ROCKSDB_GTEST_BYPASS("JEMALLOC not supported");
+      }
+    } else {
+      ROCKSDB_GTEST_BYPASS("JEMALLOC not supported");
+    }
+
+    using CacheValueChunk = CompressedSecondaryCache::CacheValueChunk;
+    std::unique_ptr<CompressedSecondaryCache> sec_cache =
+        std::make_unique<CompressedSecondaryCache>(1000, 0, true, 0.5, 0.0,
+                                                   allocator);
+    Random rnd(301);
+    // 10000 = 8169 + 1769 + 62 , so there should be 3 chunks after split.
+    size_t str_size{10000};
+    std::string str = rnd.RandomString(static_cast<int>(str_size));
+    size_t charge{0};
+    CacheValueChunk* chunks_head =
+        sec_cache->SplitValueIntoChunks(str, kLZ4Compression, charge);
+    ASSERT_EQ(charge, str_size + 3 * (sizeof(CacheValueChunk) - 1));
+
+    CacheAllocationPtr value =
+        sec_cache->MergeChunksIntoValue(chunks_head, charge);
+    ASSERT_EQ(charge, str_size);
+    std::string value_str{value.get(), charge};
+    ASSERT_EQ(strcmp(value_str.data(), str.data()), 0);
+
+    sec_cache->DeletionCallback("dummy", chunks_head);
+  }
+
  private:
   bool fail_create_;
 };
@@ -639,6 +776,18 @@ TEST_F(CompressedSecondaryCacheTest,
   IntegrationFullCapacityTest(true);
 }
 
+TEST_F(CompressedSecondaryCacheTest, SplitValueIntoChunksTest) {
+  SplitValueIntoChunksTest();
+}
+
+TEST_F(CompressedSecondaryCacheTest, MergeChunksIntoValueTest) {
+  MergeChunksIntoValueTest();
+}
+
+TEST_F(CompressedSecondaryCacheTest, SplictValueAndMergeChunksTest) {
+  SplictValueAndMergeChunksTest();
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

cache/fast_lru_cache.cc

@@ -52,6 +52,7 @@ LRUHandle* LRUHandleTable::Insert(LRUHandle* h, LRUHandle** old) {
                                                1 /*displacement*/);
   *old = nullptr;
   if (slot == -1) {
+    // TODO(Guido) Don't we need to roll back displacements here?
     return nullptr;
   }
 
@@ -298,10 +299,12 @@ int LRUCacheShard::CalcHashBits(
 }
 
 void LRUCacheShard::SetCapacity(size_t capacity) {
-  assert(false);  // Not supported. TODO(Guido) Support it?
   autovector<LRUHandle> last_reference_list;
   {
     DMutexLock l(mutex_);
+    if (capacity > capacity_) {
+      assert(false);  // Not supported.
+    }
     capacity_ = capacity;
     EvictFromLRU(0, &last_reference_list);
   }

cache/lru_cache.cc

@@ -111,14 +111,17 @@ void LRUHandleTable::Resize() {
 
 LRUCacheShard::LRUCacheShard(
     size_t capacity, bool strict_capacity_limit, double high_pri_pool_ratio,
-    bool use_adaptive_mutex, CacheMetadataChargePolicy metadata_charge_policy,
-    int max_upper_hash_bits,
+    double low_pri_pool_ratio, bool use_adaptive_mutex,
+    CacheMetadataChargePolicy metadata_charge_policy, int max_upper_hash_bits,
     const std::shared_ptr<SecondaryCache>& secondary_cache)
     : capacity_(0),
       high_pri_pool_usage_(0),
+      low_pri_pool_usage_(0),
       strict_capacity_limit_(strict_capacity_limit),
       high_pri_pool_ratio_(high_pri_pool_ratio),
       high_pri_pool_capacity_(0),
+      low_pri_pool_ratio_(low_pri_pool_ratio),
+      low_pri_pool_capacity_(0),
       table_(max_upper_hash_bits),
       usage_(0),
       lru_usage_(0),
@@ -129,6 +132,7 @@ LRUCacheShard::LRUCacheShard(
   lru_.next = &lru_;
   lru_.prev = &lru_;
   lru_low_pri_ = &lru_;
+  lru_bottom_pri_ = &lru_;
   SetCapacity(capacity);
 }
 
@@ -192,10 +196,12 @@ void LRUCacheShard::ApplyToSomeEntries(
       index_begin, index_end);
 }
 
-void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
+void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri,
+                                    LRUHandle** lru_bottom_pri) {
   DMutexLock l(mutex_);
   *lru = &lru_;
   *lru_low_pri = lru_low_pri_;
+  *lru_bottom_pri = lru_bottom_pri_;
 }
 
 size_t LRUCacheShard::TEST_GetLRUSize() {
@@ -214,20 +220,32 @@ double LRUCacheShard::GetHighPriPoolRatio() {
   return high_pri_pool_ratio_;
 }
 
+double LRUCacheShard::GetLowPriPoolRatio() {
+  DMutexLock l(mutex_);
+  return low_pri_pool_ratio_;
+}
+
 void LRUCacheShard::LRU_Remove(LRUHandle* e) {
   assert(e->next != nullptr);
   assert(e->prev != nullptr);
   if (lru_low_pri_ == e) {
     lru_low_pri_ = e->prev;
   }
+  if (lru_bottom_pri_ == e) {
+    lru_bottom_pri_ = e->prev;
+  }
   e->next->prev = e->prev;
   e->prev->next = e->next;
   e->prev = e->next = nullptr;
   assert(lru_usage_ >= e->total_charge);
   lru_usage_ -= e->total_charge;
+  assert(!e->InHighPriPool() || !e->InLowPriPool());
   if (e->InHighPriPool()) {
     assert(high_pri_pool_usage_ >= e->total_charge);
     high_pri_pool_usage_ -= e->total_charge;
+  } else if (e->InLowPriPool()) {
+    assert(low_pri_pool_usage_ >= e->total_charge);
+    low_pri_pool_usage_ -= e->total_charge;
   }
 }
 
@@ -241,17 +259,34 @@ void LRUCacheShard::LRU_Insert(LRUHandle* e) {
     e->prev->next = e;
     e->next->prev = e;
     e->SetInHighPriPool(true);
+    e->SetInLowPriPool(false);
     high_pri_pool_usage_ += e->total_charge;
     MaintainPoolSize();
-  } else {
-    // Insert "e" to the head of low-pri pool. Note that when
-    // high_pri_pool_ratio is 0, head of low-pri pool is also head of LRU list.
+  } else if (low_pri_pool_ratio_ > 0 &&
+             (e->IsHighPri() || e->IsLowPri() || e->HasHit())) {
+    // Insert "e" to the head of low-pri pool.
     e->next = lru_low_pri_->next;
     e->prev = lru_low_pri_;
     e->prev->next = e;
     e->next->prev = e;
     e->SetInHighPriPool(false);
+    e->SetInLowPriPool(true);
+    low_pri_pool_usage_ += e->total_charge;
+    MaintainPoolSize();
     lru_low_pri_ = e;
+  } else {
+    // Insert "e" to the head of bottom-pri pool.
+    e->next = lru_bottom_pri_->next;
+    e->prev = lru_bottom_pri_;
+    e->prev->next = e;
+    e->next->prev = e;
+    e->SetInHighPriPool(false);
+    e->SetInLowPriPool(false);
+    // if the low-pri pool is empty, lru_low_pri_ also needs to be updated.
+    if (lru_bottom_pri_ == lru_low_pri_) {
+      lru_low_pri_ = e;
+    }
+    lru_bottom_pri_ = e;
   }
   lru_usage_ += e->total_charge;
 }
@@ -262,8 +297,20 @@ void LRUCacheShard::MaintainPoolSize() {
     lru_low_pri_ = lru_low_pri_->next;
     assert(lru_low_pri_ != &lru_);
     lru_low_pri_->SetInHighPriPool(false);
+    lru_low_pri_->SetInLowPriPool(true);
     assert(high_pri_pool_usage_ >= lru_low_pri_->total_charge);
     high_pri_pool_usage_ -= lru_low_pri_->total_charge;
+    low_pri_pool_usage_ += lru_low_pri_->total_charge;
+  }
+
+  while (low_pri_pool_usage_ > low_pri_pool_capacity_) {
+    // Overflow last entry in low-pri pool to bottom-pri pool.
+    lru_bottom_pri_ = lru_bottom_pri_->next;
+    assert(lru_bottom_pri_ != &lru_);
+    lru_bottom_pri_->SetInHighPriPool(false);
+    lru_bottom_pri_->SetInLowPriPool(false);
+    assert(low_pri_pool_usage_ >= lru_bottom_pri_->total_charge);
+    low_pri_pool_usage_ -= lru_bottom_pri_->total_charge;
   }
 }
 
@@ -288,6 +335,7 @@ void LRUCacheShard::SetCapacity(size_t capacity) {
     DMutexLock l(mutex_);
     capacity_ = capacity;
     high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
+    low_pri_pool_capacity_ = capacity_ * low_pri_pool_ratio_;
     EvictFromLRU(0, &last_reference_list);
   }
 
@@ -458,11 +506,12 @@ Cache::Handle* LRUCacheShard::Lookup(
       memcpy(e->key_data, key.data(), key.size());
       e->value = nullptr;
       e->sec_handle = secondary_handle.release();
+      e->total_charge = 0;
       e->Ref();
+      e->SetIsInSecondaryCache(is_in_sec_cache);
 
       if (wait) {
         Promote(e);
-        e->SetIsInSecondaryCache(is_in_sec_cache);
         if (!e->value) {
           // The secondary cache returned a handle, but the lookup failed.
           e->Unref();
@@ -476,7 +525,6 @@ Cache::Handle* LRUCacheShard::Lookup(
         // If wait is false, we always return a handle and let the caller
         // release the handle after checking for success or failure.
         e->SetIncomplete(true);
-        e->SetIsInSecondaryCache(is_in_sec_cache);
         // This may be slightly inaccurate, if the lookup eventually fails.
         // But the probability is very low.
         PERF_COUNTER_ADD(secondary_cache_hit_count, 1);
@@ -503,6 +551,13 @@ void LRUCacheShard::SetHighPriorityPoolRatio(double high_pri_pool_ratio) {
   MaintainPoolSize();
 }
 
+void LRUCacheShard::SetLowPriorityPoolRatio(double low_pri_pool_ratio) {
+  DMutexLock l(mutex_);
+  low_pri_pool_ratio_ = low_pri_pool_ratio;
+  low_pri_pool_capacity_ = capacity_ * low_pri_pool_ratio_;
+  MaintainPoolSize();
+}
+
 bool LRUCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
   if (handle == nullptr) {
     return false;
@@ -529,7 +584,7 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
     // If it was the last reference, and the entry is either not secondary
     // cache compatible (i.e a dummy entry for accounting), or is secondary
     // cache compatible and has a non-null value, then decrement the cache
-    // usage. If value is null in the latter case, taht means the lookup
+    // usage. If value is null in the latter case, that means the lookup
     // failed and we didn't charge the cache.
     if (last_reference && (!e->IsSecondaryCacheCompatible() || e->value)) {
       assert(usage_ >= e->total_charge);
@@ -634,12 +689,15 @@ std::string LRUCacheShard::GetPrintableOptions() const {
     DMutexLock l(mutex_);
     snprintf(buffer, kBufferSize, "    high_pri_pool_ratio: %.3lf\n",
              high_pri_pool_ratio_);
+    snprintf(buffer + strlen(buffer), kBufferSize - strlen(buffer),
+             "    low_pri_pool_ratio: %.3lf\n", low_pri_pool_ratio_);
   }
   return std::string(buffer);
 }
 
 LRUCache::LRUCache(size_t capacity, int num_shard_bits,
                    bool strict_capacity_limit, double high_pri_pool_ratio,
+                   double low_pri_pool_ratio,
                    std::shared_ptr<MemoryAllocator> allocator,
                    bool use_adaptive_mutex,
                    CacheMetadataChargePolicy metadata_charge_policy,
@@ -653,7 +711,7 @@ LRUCache::LRUCache(size_t capacity, int num_shard_bits,
   for (int i = 0; i < num_shards_; i++) {
     new (&shards_[i]) LRUCacheShard(
         per_shard, strict_capacity_limit, high_pri_pool_ratio,
-        use_adaptive_mutex, metadata_charge_policy,
+        low_pri_pool_ratio, use_adaptive_mutex, metadata_charge_policy,
         /* max_upper_hash_bits */ 32 - num_shard_bits, secondary_cache);
   }
   secondary_cache_ = secondary_cache;
@@ -775,7 +833,8 @@ std::shared_ptr<Cache> NewLRUCache(
     double high_pri_pool_ratio,
     std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
     CacheMetadataChargePolicy metadata_charge_policy,
-    const std::shared_ptr<SecondaryCache>& secondary_cache) {
+    const std::shared_ptr<SecondaryCache>& secondary_cache,
+    double low_pri_pool_ratio) {
   if (num_shard_bits >= 20) {
     return nullptr;  // The cache cannot be sharded into too many fine pieces.
   }
@@ -783,30 +842,40 @@ std::shared_ptr<Cache> NewLRUCache(
     // Invalid high_pri_pool_ratio
     return nullptr;
   }
+  if (low_pri_pool_ratio < 0.0 || low_pri_pool_ratio > 1.0) {
+    // Invalid high_pri_pool_ratio
+    return nullptr;
+  }
+  if (low_pri_pool_ratio + high_pri_pool_ratio > 1.0) {
+    // Invalid high_pri_pool_ratio and low_pri_pool_ratio combination
+    return nullptr;
+  }
   if (num_shard_bits < 0) {
     num_shard_bits = GetDefaultCacheShardBits(capacity);
   }
   return std::make_shared<LRUCache>(
       capacity, num_shard_bits, strict_capacity_limit, high_pri_pool_ratio,
-      std::move(memory_allocator), use_adaptive_mutex, metadata_charge_policy,
-      secondary_cache);
+      low_pri_pool_ratio, std::move(memory_allocator), use_adaptive_mutex,
+      metadata_charge_policy, secondary_cache);
 }
 
 std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts) {
-  return NewLRUCache(
-      cache_opts.capacity, cache_opts.num_shard_bits,
-      cache_opts.strict_capacity_limit, cache_opts.high_pri_pool_ratio,
-      cache_opts.memory_allocator, cache_opts.use_adaptive_mutex,
-      cache_opts.metadata_charge_policy, cache_opts.secondary_cache);
+  return NewLRUCache(cache_opts.capacity, cache_opts.num_shard_bits,
+                     cache_opts.strict_capacity_limit,
+                     cache_opts.high_pri_pool_ratio,
+                     cache_opts.memory_allocator, cache_opts.use_adaptive_mutex,
+                     cache_opts.metadata_charge_policy,
+                     cache_opts.secondary_cache, cache_opts.low_pri_pool_ratio);
 }
 
 std::shared_ptr<Cache> NewLRUCache(
     size_t capacity, int num_shard_bits, bool strict_capacity_limit,
     double high_pri_pool_ratio,
     std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
-    CacheMetadataChargePolicy metadata_charge_policy) {
+    CacheMetadataChargePolicy metadata_charge_policy,
+    double low_pri_pool_ratio) {
   return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit,
                      high_pri_pool_ratio, memory_allocator, use_adaptive_mutex,
-                     metadata_charge_policy, nullptr);
+                     metadata_charge_policy, nullptr, low_pri_pool_ratio);
 }
 }  // namespace ROCKSDB_NAMESPACE

cache/lru_cache.h

@@ -74,7 +74,7 @@ struct LRUHandle {
   // The number of external refs to this entry. The cache itself is not counted.
   uint32_t refs;
 
-  enum Flags : uint8_t {
+  enum Flags : uint16_t {
     // Whether this entry is referenced by the hash table.
     IN_CACHE = (1 << 0),
     // Whether this entry is high priority entry.
@@ -89,9 +89,13 @@ struct LRUHandle {
     IS_PENDING = (1 << 5),
     // Whether this handle is still in a lower tier
     IS_IN_SECONDARY_CACHE = (1 << 6),
+    // Whether this entry is low priority entry.
+    IS_LOW_PRI = (1 << 7),
+    // Whether this entry is in low-pri pool.
+    IN_LOW_PRI_POOL = (1 << 8),
   };
 
-  uint8_t flags;
+  uint16_t flags;
 
 #ifdef __SANITIZE_THREAD__
   // TSAN can report a false data race on flags, where one thread is writing
@@ -122,6 +126,8 @@ struct LRUHandle {
   bool InCache() const { return flags & IN_CACHE; }
   bool IsHighPri() const { return flags & IS_HIGH_PRI; }
   bool InHighPriPool() const { return flags & IN_HIGH_PRI_POOL; }
+  bool IsLowPri() const { return flags & IS_LOW_PRI; }
+  bool InLowPriPool() const { return flags & IN_LOW_PRI_POOL; }
   bool HasHit() const { return flags & HAS_HIT; }
   bool IsSecondaryCacheCompatible() const {
 #ifdef __SANITIZE_THREAD__
@@ -144,8 +150,13 @@ struct LRUHandle {
   void SetPriority(Cache::Priority priority) {
     if (priority == Cache::Priority::HIGH) {
       flags |= IS_HIGH_PRI;
+      flags &= ~IS_LOW_PRI;
+    } else if (priority == Cache::Priority::LOW) {
+      flags &= ~IS_HIGH_PRI;
+      flags |= IS_LOW_PRI;
     } else {
       flags &= ~IS_HIGH_PRI;
+      flags &= ~IS_LOW_PRI;
     }
   }
 
@@ -157,6 +168,14 @@ struct LRUHandle {
     }
   }
 
+  void SetInLowPriPool(bool in_low_pri_pool) {
+    if (in_low_pri_pool) {
+      flags |= IN_LOW_PRI_POOL;
+    } else {
+      flags &= ~IN_LOW_PRI_POOL;
+    }
+  }
+
   void SetHit() { flags |= HAS_HIT; }
 
   void SetSecondaryCacheCompatible(bool compat) {
@@ -298,7 +317,8 @@ class LRUHandleTable {
 class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
  public:
   LRUCacheShard(size_t capacity, bool strict_capacity_limit,
-                double high_pri_pool_ratio, bool use_adaptive_mutex,
+                double high_pri_pool_ratio, double low_pri_pool_ratio,
+                bool use_adaptive_mutex,
                 CacheMetadataChargePolicy metadata_charge_policy,
                 int max_upper_hash_bits,
                 const std::shared_ptr<SecondaryCache>& secondary_cache);
@@ -315,6 +335,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // Set percentage of capacity reserved for high-pri cache entries.
   void SetHighPriorityPoolRatio(double high_pri_pool_ratio);
 
+  // Set percentage of capacity reserved for low-pri cache entries.
+  void SetLowPriorityPoolRatio(double low_pri_pool_ratio);
+
   // Like Cache methods, but with an extra "hash" parameter.
   virtual Status Insert(const Slice& key, uint32_t hash, void* value,
                         size_t charge, Cache::DeleterFn deleter,
@@ -366,15 +389,19 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
 
   virtual std::string GetPrintableOptions() const override;
 
-  void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri);
+  void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri,
+                       LRUHandle** lru_bottom_pri);
 
-  //  Retrieves number of elements in LRU, for unit test purpose only.
-  //  Not threadsafe.
+  // Retrieves number of elements in LRU, for unit test purpose only.
+  // Not threadsafe.
   size_t TEST_GetLRUSize();
 
-  //  Retrieves high pri pool ratio
+  // Retrieves high pri pool ratio
   double GetHighPriPoolRatio();
 
+  // Retrieves low pri pool ratio
+  double GetLowPriPoolRatio();
+
  private:
   friend class LRUCache;
   // Insert an item into the hash table and, if handle is null, insert into
@@ -414,6 +441,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // Memory size for entries in high-pri pool.
   size_t high_pri_pool_usage_;
 
+  // Memory size for entries in low-pri pool.
+  size_t low_pri_pool_usage_;
+
   // Whether to reject insertion if cache reaches its full capacity.
   bool strict_capacity_limit_;
 
@@ -424,6 +454,13 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // Remember the value to avoid recomputing each time.
   double high_pri_pool_capacity_;
 
+  // Ratio of capacity reserved for low priority cache entries.
+  double low_pri_pool_ratio_;
+
+  // Low-pri pool size, equals to capacity * low_pri_pool_ratio.
+  // Remember the value to avoid recomputing each time.
+  double low_pri_pool_capacity_;
+
   // Dummy head of LRU list.
   // lru.prev is newest entry, lru.next is oldest entry.
   // LRU contains items which can be evicted, ie reference only by cache
@@ -432,6 +469,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // Pointer to head of low-pri pool in LRU list.
   LRUHandle* lru_low_pri_;
 
+  // Pointer to head of bottom-pri pool in LRU list.
+  LRUHandle* lru_bottom_pri_;
+
   // ------------^^^^^^^^^^^^^-----------
   // Not frequently modified data members
   // ------------------------------------
@@ -466,7 +506,7 @@ class LRUCache
     : public ShardedCache {
  public:
   LRUCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-           double high_pri_pool_ratio,
+           double high_pri_pool_ratio, double low_pri_pool_ratio,
            std::shared_ptr<MemoryAllocator> memory_allocator = nullptr,
            bool use_adaptive_mutex = kDefaultToAdaptiveMutex,
            CacheMetadataChargePolicy metadata_charge_policy =
@@ -484,9 +524,9 @@ class LRUCache
   virtual void WaitAll(std::vector<Handle*>& handles) override;
   std::string GetPrintableOptions() const override;
 
-  //  Retrieves number of elements in LRU, for unit test purpose only.
+  // Retrieves number of elements in LRU, for unit test purpose only.
   size_t TEST_GetLRUSize();
-  //  Retrieves high pri pool ratio.
+  // Retrieves high pri pool ratio.
   double GetHighPriPoolRatio();
 
  private:

cache/sharded_cache.h

@@ -86,7 +86,7 @@ class ShardedCache : public Cache {
                         DeleterFn deleter, Handle** handle,
                         Priority priority) override;
   virtual Status Insert(const Slice& key, void* value,
-                        const CacheItemHelper* helper, size_t chargge,
+                        const CacheItemHelper* helper, size_t charge,
                         Handle** handle = nullptr,
                         Priority priority = Priority::LOW) override;
   virtual Handle* Lookup(const Slice& key, Statistics* stats) override;

crash_test.mk

@@ -18,7 +18,9 @@ CRASHTEST_PY=$(PYTHON) -u tools/db_crashtest.py --stress_cmd=$(DB_STRESS_CMD)
 	whitebox_crash_test whitebox_crash_test_with_atomic_flush \
 	whitebox_crash_test_with_txn whitebox_crash_test_with_ts \
 	blackbox_crash_test_with_multiops_wc_txn \
-	blackbox_crash_test_with_multiops_wp_txn
+	blackbox_crash_test_with_multiops_wp_txn \
+	crash_test_with_tiered_storage blackbox_crash_test_with_tiered_storage \
+	whitebox_crash_test_with_tiered_storage \
 
 crash_test: $(DB_STRESS_CMD)
 # Do not parallelize
@@ -42,6 +44,11 @@ crash_test_with_ts: $(DB_STRESS_CMD)
 	$(CRASHTEST_MAKE) whitebox_crash_test_with_ts
 	$(CRASHTEST_MAKE) blackbox_crash_test_with_ts
 
+crash_test_with_tiered_storage: $(DB_STRESS_CMD)
+# Do not parallelize
+	$(CRASHTEST_MAKE) whitebox_crash_test_with_tiered_storage
+	$(CRASHTEST_MAKE) blackbox_crash_test_with_tiered_storage
+
 crash_test_with_multiops_wc_txn: $(DB_STRESS_CMD)
 	$(CRASHTEST_MAKE) blackbox_crash_test_with_multiops_wc_txn
 
@@ -70,6 +77,9 @@ blackbox_crash_test_with_multiops_wc_txn: $(DB_STRESS_CMD)
 blackbox_crash_test_with_multiops_wp_txn: $(DB_STRESS_CMD)
 	$(CRASHTEST_PY) --test_multiops_txn --write_policy write_prepared blackbox $(CRASH_TEST_EXT_ARGS)
 
+blackbox_crash_test_with_tiered_storage: $(DB_STRESS_CMD)
+	$(CRASHTEST_PY) --enable_tiered_storage blackbox $(CRASH_TEST_EXT_ARGS)
+
 ifeq ($(CRASH_TEST_KILL_ODD),)
   CRASH_TEST_KILL_ODD=888887
 endif
@@ -91,3 +101,7 @@ whitebox_crash_test_with_txn: $(DB_STRESS_CMD)
 whitebox_crash_test_with_ts: $(DB_STRESS_CMD)
 	$(CRASHTEST_PY) --enable_ts whitebox --random_kill_odd \
       $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
+
+whitebox_crash_test_with_tiered_storage: $(DB_STRESS_CMD)
+	$(CRASHTEST_PY) --enable_tiered_storage whitebox --random_kill_odd \
+      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)

db/arena_wrapped_db_iter.cc

@@ -89,8 +89,8 @@ Status ArenaWrappedDBIter::Refresh() {
         ReadRangeDelAggregator* range_del_agg =
             db_iter_->GetRangeDelAggregator();
         std::unique_ptr<FragmentedRangeTombstoneIterator> range_del_iter;
-        range_del_iter.reset(
-            sv->mem->NewRangeTombstoneIterator(read_options_, latest_seq));
+        range_del_iter.reset(sv->mem->NewRangeTombstoneIterator(
+            read_options_, latest_seq, false /* immutable_memtable */));
         range_del_agg->AddTombstones(std::move(range_del_iter));
         cfd_->ReturnThreadLocalSuperVersion(sv);
       }

db/blob/blob_file_builder.cc

@@ -32,9 +32,9 @@ BlobFileBuilder::BlobFileBuilder(
     VersionSet* versions, FileSystem* fs,
     const ImmutableOptions* immutable_options,
     const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
-    int job_id, uint32_t column_family_id,
-    const std::string& column_family_name, Env::IOPriority io_priority,
-    Env::WriteLifeTimeHint write_hint,
+    std::string db_id, std::string db_session_id, int job_id,
+    uint32_t column_family_id, const std::string& column_family_name,
+    Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint,
     const std::shared_ptr<IOTracer>& io_tracer,
     BlobFileCompletionCallback* blob_callback,
     BlobFileCreationReason creation_reason,
@@ -42,17 +42,18 @@ BlobFileBuilder::BlobFileBuilder(
     std::vector<BlobFileAddition>* blob_file_additions)
     : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, fs,
                       immutable_options, mutable_cf_options, file_options,
-                      job_id, column_family_id, column_family_name, io_priority,
-                      write_hint, io_tracer, blob_callback, creation_reason,
-                      blob_file_paths, blob_file_additions) {}
+                      db_id, db_session_id, job_id, column_family_id,
+                      column_family_name, io_priority, write_hint, io_tracer,
+                      blob_callback, creation_reason, blob_file_paths,
+                      blob_file_additions) {}
 
 BlobFileBuilder::BlobFileBuilder(
     std::function<uint64_t()> file_number_generator, FileSystem* fs,
     const ImmutableOptions* immutable_options,
     const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
-    int job_id, uint32_t column_family_id,
-    const std::string& column_family_name, Env::IOPriority io_priority,
-    Env::WriteLifeTimeHint write_hint,
+    std::string db_id, std::string db_session_id, int job_id,
+    uint32_t column_family_id, const std::string& column_family_name,
+    Env::IOPriority io_priority, Env::WriteLifeTimeHint write_hint,
     const std::shared_ptr<IOTracer>& io_tracer,
     BlobFileCompletionCallback* blob_callback,
     BlobFileCreationReason creation_reason,
@@ -64,7 +65,10 @@ BlobFileBuilder::BlobFileBuilder(
       min_blob_size_(mutable_cf_options->min_blob_size),
       blob_file_size_(mutable_cf_options->blob_file_size),
       blob_compression_type_(mutable_cf_options->blob_compression_type),
+      prepopulate_blob_cache_(mutable_cf_options->prepopulate_blob_cache),
       file_options_(file_options),
+      db_id_(std::move(db_id)),
+      db_session_id_(std::move(db_session_id)),
       job_id_(job_id),
       column_family_id_(column_family_id),
       column_family_name_(column_family_name),
@@ -133,6 +137,16 @@ Status BlobFileBuilder::Add(const Slice& key, const Slice& value,
     }
   }
 
+  {
+    const Status s =
+        PutBlobIntoCacheIfNeeded(value, blob_file_number, blob_offset);
+    if (!s.ok()) {
+      ROCKS_LOG_WARN(immutable_options_->info_log,
+                     "Failed to pre-populate the blob into blob cache: %s",
+                     s.ToString().c_str());
+    }
+  }
+
   BlobIndex::EncodeBlob(blob_index, blob_file_number, blob_offset, blob.size(),
                         blob_compression_type_);
 
@@ -372,4 +386,48 @@ void BlobFileBuilder::Abandon(const Status& s) {
   blob_count_ = 0;
   blob_bytes_ = 0;
 }
+
+Status BlobFileBuilder::PutBlobIntoCacheIfNeeded(const Slice& blob,
+                                                 uint64_t blob_file_number,
+                                                 uint64_t blob_offset) const {
+  Status s = Status::OK();
+
+  auto blob_cache = immutable_options_->blob_cache;
+  auto statistics = immutable_options_->statistics.get();
+  bool warm_cache =
+      prepopulate_blob_cache_ == PrepopulateBlobCache::kFlushOnly &&
+      creation_reason_ == BlobFileCreationReason::kFlush;
+
+  if (blob_cache && warm_cache) {
+    const OffsetableCacheKey base_cache_key(db_id_, db_session_id_,
+                                            blob_file_number);
+    const CacheKey cache_key = base_cache_key.WithOffset(blob_offset);
+    const Slice key = cache_key.AsSlice();
+
+    const Cache::Priority priority = Cache::Priority::BOTTOM;
+
+    // Objects to be put into the cache have to be heap-allocated and
+    // self-contained, i.e. own their contents. The Cache has to be able to
+    // take unique ownership of them. Therefore, we copy the blob into a
+    // string directly, and insert that into the cache.
+    std::unique_ptr<std::string> buf = std::make_unique<std::string>();
+    buf->assign(blob.data(), blob.size());
+
+    // TODO: support custom allocators and provide a better estimated memory
+    // usage using malloc_usable_size.
+    s = blob_cache->Insert(key, buf.get(), buf->size(),
+                           &DeleteCacheEntry<std::string>,
+                           nullptr /* cache_handle */, priority);
+    if (s.ok()) {
+      RecordTick(statistics, BLOB_DB_CACHE_ADD);
+      RecordTick(statistics, BLOB_DB_CACHE_BYTES_WRITE, buf->size());
+      buf.release();
+    } else {
+      RecordTick(statistics, BLOB_DB_CACHE_ADD_FAILURES);
+    }
+  }
+
+  return s;
+}
+
 }  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_builder.h

@@ -10,6 +10,7 @@
 #include <string>
 #include <vector>
 
+#include "rocksdb/advanced_options.h"
 #include "rocksdb/compression_type.h"
 #include "rocksdb/env.h"
 #include "rocksdb/rocksdb_namespace.h"
@@ -35,7 +36,8 @@ class BlobFileBuilder {
   BlobFileBuilder(VersionSet* versions, FileSystem* fs,
                   const ImmutableOptions* immutable_options,
                   const MutableCFOptions* mutable_cf_options,
-                  const FileOptions* file_options, int job_id,
+                  const FileOptions* file_options, std::string db_id,
+                  std::string db_session_id, int job_id,
                   uint32_t column_family_id,
                   const std::string& column_family_name,
                   Env::IOPriority io_priority,
@@ -49,7 +51,8 @@ class BlobFileBuilder {
   BlobFileBuilder(std::function<uint64_t()> file_number_generator,
                   FileSystem* fs, const ImmutableOptions* immutable_options,
                   const MutableCFOptions* mutable_cf_options,
-                  const FileOptions* file_options, int job_id,
+                  const FileOptions* file_options, std::string db_id,
+                  std::string db_session_id, int job_id,
                   uint32_t column_family_id,
                   const std::string& column_family_name,
                   Env::IOPriority io_priority,
@@ -78,13 +81,19 @@ class BlobFileBuilder {
   Status CloseBlobFile();
   Status CloseBlobFileIfNeeded();
 
+  Status PutBlobIntoCacheIfNeeded(const Slice& blob, uint64_t blob_file_number,
+                                  uint64_t blob_offset) const;
+
   std::function<uint64_t()> file_number_generator_;
   FileSystem* fs_;
   const ImmutableOptions* immutable_options_;
   uint64_t min_blob_size_;
   uint64_t blob_file_size_;
   CompressionType blob_compression_type_;
+  PrepopulateBlobCache prepopulate_blob_cache_;
   const FileOptions* file_options_;
+  const std::string db_id_;
+  const std::string db_session_id_;
   int job_id_;
   uint32_t column_family_id_;
   std::string column_family_name_;

db/blob/blob_file_builder_test.cc

@@ -144,8 +144,9 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
 
   BlobFileBuilder builder(
       TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
+      column_family_id, column_family_name, io_priority, write_hint,
+      nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
       BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
 
   std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
@@ -228,8 +229,9 @@ TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
 
   BlobFileBuilder builder(
       TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
+      column_family_id, column_family_name, io_priority, write_hint,
+      nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
       BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
 
   std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
@@ -315,8 +317,9 @@ TEST_F(BlobFileBuilderTest, InlinedValues) {
 
   BlobFileBuilder builder(
       TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
+      column_family_id, column_family_name, io_priority, write_hint,
+      nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
       BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
 
   for (size_t i = 0; i < number_of_blobs; ++i) {
@@ -369,8 +372,9 @@ TEST_F(BlobFileBuilderTest, Compression) {
 
   BlobFileBuilder builder(
       TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
+      column_family_id, column_family_name, io_priority, write_hint,
+      nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
       BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
 
   const std::string key("1");
@@ -452,8 +456,9 @@ TEST_F(BlobFileBuilderTest, CompressionError) {
 
   BlobFileBuilder builder(
       TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
+      column_family_id, column_family_name, io_priority, write_hint,
+      nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
       BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
 
   SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue",
@@ -531,8 +536,9 @@ TEST_F(BlobFileBuilderTest, Checksum) {
 
   BlobFileBuilder builder(
       TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
+      column_family_id, column_family_name, io_priority, write_hint,
+      nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
       BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
 
   const std::string key("1");
@@ -628,8 +634,9 @@ TEST_P(BlobFileBuilderIOErrorTest, IOError) {
 
   BlobFileBuilder builder(
       TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
+      &file_options_, "" /*db_id*/, "" /*db_session_id*/, job_id,
+      column_family_id, column_family_name, io_priority, write_hint,
+      nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
       BlobFileCreationReason::kFlush, &blob_file_paths, &blob_file_additions);
 
   SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) {

db/blob/blob_file_reader.cc

@@ -414,7 +414,7 @@ void BlobFileReader::MultiGetBlob(const ReadOptions& read_options,
     assert(blob_reqs[i]->offset >= adjustment);
     adjustments.push_back(adjustment);
 
-    FSReadRequest read_req;
+    FSReadRequest read_req = {};
     read_req.offset = blob_reqs[i]->offset - adjustment;
     read_req.len = blob_reqs[i]->len + adjustment;
     read_reqs.emplace_back(read_req);

db/blob/blob_index.h

@@ -92,7 +92,7 @@ class BlobIndex {
   }
 
   Status DecodeFrom(Slice slice) {
-    static const std::string kErrorMessage = "Error while decoding blob index";
+    const char* kErrorMessage = "Error while decoding blob index";
     assert(slice.size() > 0);
     type_ = static_cast<Type>(*slice.data());
     if (type_ >= Type::kUnknown) {

db/blob/blob_log_format.cc

@@ -26,8 +26,7 @@ void BlobLogHeader::EncodeTo(std::string* dst) {
 }
 
 Status BlobLogHeader::DecodeFrom(Slice src) {
-  static const std::string kErrorMessage =
-      "Error while decoding blob log header";
+  const char* kErrorMessage = "Error while decoding blob log header";
   if (src.size() != BlobLogHeader::kSize) {
     return Status::Corruption(kErrorMessage,
                               "Unexpected blob file header size");
@@ -71,8 +70,7 @@ void BlobLogFooter::EncodeTo(std::string* dst) {
 }
 
 Status BlobLogFooter::DecodeFrom(Slice src) {
-  static const std::string kErrorMessage =
-      "Error while decoding blob log footer";
+  const char* kErrorMessage = "Error while decoding blob log footer";
   if (src.size() != BlobLogFooter::kSize) {
     return Status::Corruption(kErrorMessage,
                               "Unexpected blob file footer size");
@@ -112,7 +110,7 @@ void BlobLogRecord::EncodeHeaderTo(std::string* dst) {
 }
 
 Status BlobLogRecord::DecodeHeaderFrom(Slice src) {
-  static const std::string kErrorMessage = "Error while decoding blob record";
+  const char* kErrorMessage = "Error while decoding blob record";
   if (src.size() != BlobLogRecord::kHeaderSize) {
     return Status::Corruption(kErrorMessage,
                               "Unexpected blob record header size");

db/blob/blob_log_writer.cc

@@ -70,32 +70,38 @@ Status BlobLogWriter::AppendFooter(BlobLogFooter& footer,
   std::string str;
   footer.EncodeTo(&str);
 
-  Status s = dest_->Append(Slice(str));
-  if (s.ok()) {
-    block_offset_ += str.size();
-
-    s = Sync();
-
+  Status s;
+  if (dest_->seen_error()) {
+    s.PermitUncheckedError();
+    return Status::IOError("Seen Error. Skip closing.");
+  } else {
+    s = dest_->Append(Slice(str));
     if (s.ok()) {
-      s = dest_->Close();
+      block_offset_ += str.size();
+
+      s = Sync();
 
       if (s.ok()) {
-        assert(!!checksum_method == !!checksum_value);
+        s = dest_->Close();
 
-        if (checksum_method) {
-          assert(checksum_method->empty());
+        if (s.ok()) {
+          assert(!!checksum_method == !!checksum_value);
 
-          std::string method = dest_->GetFileChecksumFuncName();
-          if (method != kUnknownFileChecksumFuncName) {
-            *checksum_method = std::move(method);
+          if (checksum_method) {
+            assert(checksum_method->empty());
+
+            std::string method = dest_->GetFileChecksumFuncName();
+            if (method != kUnknownFileChecksumFuncName) {
+              *checksum_method = std::move(method);
+            }
           }
-        }
-        if (checksum_value) {
-          assert(checksum_value->empty());
+          if (checksum_value) {
+            assert(checksum_value->empty());
 
-          std::string value = dest_->GetFileChecksum();
-          if (value != kUnknownFileChecksum) {
-            *checksum_value = std::move(value);
+            std::string value = dest_->GetFileChecksum();
+            if (value != kUnknownFileChecksum) {
+              *checksum_value = std::move(value);
+            }
           }
         }
       }

db/blob/blob_source.cc

@@ -8,6 +8,8 @@
 #include <cassert>
 #include <string>
 
+#include "cache/cache_reservation_manager.h"
+#include "cache/charged_cache.h"
 #include "db/blob/blob_file_reader.h"
 #include "db/blob/blob_log_format.h"
 #include "monitoring/statistics.h"
@@ -25,7 +27,19 @@ BlobSource::BlobSource(const ImmutableOptions* immutable_options,
       db_session_id_(db_session_id),
       statistics_(immutable_options->statistics.get()),
       blob_file_cache_(blob_file_cache),
-      blob_cache_(immutable_options->blob_cache) {}
+      blob_cache_(immutable_options->blob_cache),
+      lowest_used_cache_tier_(immutable_options->lowest_used_cache_tier) {
+#ifndef ROCKSDB_LITE
+  auto bbto =
+      immutable_options->table_factory->GetOptions<BlockBasedTableOptions>();
+  if (bbto &&
+      bbto->cache_usage_options.options_overrides.at(CacheEntryRole::kBlobCache)
+              .charged == CacheEntryRoleOptions::Decision::kEnabled) {
+    blob_cache_ = std::make_shared<ChargedCache>(immutable_options->blob_cache,
+                                                 bbto->block_cache);
+  }
+#endif  // ROCKSDB_LITE
+}
 
 BlobSource::~BlobSource() = default;
 
@@ -56,21 +70,22 @@ Status BlobSource::PutBlobIntoCache(const Slice& cache_key,
   assert(blob_cache_);
 
   Status s;
-  const Cache::Priority priority = Cache::Priority::LOW;
+  const Cache::Priority priority = Cache::Priority::BOTTOM;
 
   // Objects to be put into the cache have to be heap-allocated and
   // self-contained, i.e. own their contents. The Cache has to be able to take
   // unique ownership of them. Therefore, we copy the blob into a string
   // directly, and insert that into the cache.
-  std::string* buf = new std::string();
+  std::unique_ptr<std::string> buf = std::make_unique<std::string>();
   buf->assign(blob->data(), blob->size());
 
   // TODO: support custom allocators and provide a better estimated memory
   // usage using malloc_usable_size.
   Cache::Handle* cache_handle = nullptr;
-  s = InsertEntryIntoCache(cache_key, buf, buf->size(), &cache_handle,
+  s = InsertEntryIntoCache(cache_key, buf.get(), buf->size(), &cache_handle,
                            priority);
   if (s.ok()) {
+    buf.release();
     assert(cache_handle != nullptr);
     *cached_blob =
         CacheHandleGuard<std::string>(blob_cache_.get(), cache_handle);
@@ -81,7 +96,24 @@ Status BlobSource::PutBlobIntoCache(const Slice& cache_key,
 
 Cache::Handle* BlobSource::GetEntryFromCache(const Slice& key) const {
   Cache::Handle* cache_handle = nullptr;
-  cache_handle = blob_cache_->Lookup(key, statistics_);
+
+  if (lowest_used_cache_tier_ == CacheTier::kNonVolatileBlockTier) {
+    Cache::CreateCallback create_cb = [&](const void* buf, size_t size,
+                                          void** out_obj,
+                                          size_t* charge) -> Status {
+      std::string* blob = new std::string();
+      blob->assign(static_cast<const char*>(buf), size);
+      *out_obj = blob;
+      *charge = size;
+      return Status::OK();
+    };
+    cache_handle = blob_cache_->Lookup(key, GetCacheItemHelper(), create_cb,
+                                       Cache::Priority::BOTTOM,
+                                       true /* wait_for_cache */, statistics_);
+  } else {
+    cache_handle = blob_cache_->Lookup(key, statistics_);
+  }
+
   if (cache_handle != nullptr) {
     PERF_COUNTER_ADD(blob_cache_hit_count, 1);
     RecordTick(statistics_, BLOB_DB_CACHE_HIT);
@@ -97,9 +129,16 @@ Status BlobSource::InsertEntryIntoCache(const Slice& key, std::string* value,
                                         size_t charge,
                                         Cache::Handle** cache_handle,
                                         Cache::Priority priority) const {
-  const Status s =
-      blob_cache_->Insert(key, value, charge, &DeleteCacheEntry<std::string>,
-                          cache_handle, priority);
+  Status s;
+
+  if (lowest_used_cache_tier_ == CacheTier::kNonVolatileBlockTier) {
+    s = blob_cache_->Insert(key, value, GetCacheItemHelper(), charge,
+                            cache_handle, priority);
+  } else {
+    s = blob_cache_->Insert(key, value, charge, &DeleteCacheEntry<std::string>,
+                            cache_handle, priority);
+  }
+
   if (s.ok()) {
     assert(*cache_handle != nullptr);
     RecordTick(statistics_, BLOB_DB_CACHE_ADD);
@@ -108,6 +147,7 @@ Status BlobSource::InsertEntryIntoCache(const Slice& key, std::string* value,
   } else {
     RecordTick(statistics_, BLOB_DB_CACHE_ADD_FAILURES);
   }
+
   return s;
 }
 
@@ -246,7 +286,7 @@ void BlobSource::MultiGetBlob(const ReadOptions& read_options,
 
 void BlobSource::MultiGetBlobFromOneFile(const ReadOptions& read_options,
                                          uint64_t file_number,
-                                         uint64_t file_size,
+                                         uint64_t /*file_size*/,
                                          autovector<BlobReadRequest>& blob_reqs,
                                          uint64_t* bytes_read) {
   const size_t num_blobs = blob_reqs.size();
@@ -263,8 +303,7 @@ void BlobSource::MultiGetBlobFromOneFile(const ReadOptions& read_options,
   Mask cache_hit_mask = 0;
 
   uint64_t total_bytes = 0;
-  const OffsetableCacheKey base_cache_key(db_id_, db_session_id_, file_number,
-                                          file_size);
+  const OffsetableCacheKey base_cache_key(db_id_, db_session_id_, file_number);
 
   if (blob_cache_) {
     size_t cached_blob_count = 0;
@@ -402,4 +441,25 @@ bool BlobSource::TEST_BlobInCache(uint64_t file_number, uint64_t file_size,
   return false;
 }
 
+// Callbacks for secondary blob cache
+size_t BlobSource::SizeCallback(void* obj) {
+  assert(obj != nullptr);
+  return static_cast<const std::string*>(obj)->size();
+}
+
+Status BlobSource::SaveToCallback(void* from_obj, size_t from_offset,
+                                  size_t length, void* out) {
+  assert(from_obj != nullptr);
+  const std::string* buf = static_cast<const std::string*>(from_obj);
+  assert(buf->size() >= from_offset + length);
+  memcpy(out, buf->data() + from_offset, length);
+  return Status::OK();
+}
+
+Cache::CacheItemHelper* BlobSource::GetCacheItemHelper() {
+  static Cache::CacheItemHelper cache_helper(SizeCallback, SaveToCallback,
+                                             &DeleteCacheEntry<std::string>);
+  return &cache_helper;
+}
+
 }  // namespace ROCKSDB_NAMESPACE

db/blob/blob_source.h

@@ -99,6 +99,8 @@ class BlobSource {
                                                blob_file_reader);
   }
 
+  inline Cache* GetBlobCache() const { return blob_cache_.get(); }
+
   bool TEST_BlobInCache(uint64_t file_number, uint64_t file_size,
                         uint64_t offset) const;
 
@@ -116,13 +118,20 @@ class BlobSource {
                               size_t charge, Cache::Handle** cache_handle,
                               Cache::Priority priority) const;
 
-  inline CacheKey GetCacheKey(uint64_t file_number, uint64_t file_size,
+  inline CacheKey GetCacheKey(uint64_t file_number, uint64_t /*file_size*/,
                               uint64_t offset) const {
-    OffsetableCacheKey base_cache_key(db_id_, db_session_id_, file_number,
-                                      file_size);
+    OffsetableCacheKey base_cache_key(db_id_, db_session_id_, file_number);
     return base_cache_key.WithOffset(offset);
   }
 
+  // Callbacks for secondary blob cache
+  static size_t SizeCallback(void* obj);
+
+  static Status SaveToCallback(void* from_obj, size_t from_offset,
+                               size_t length, void* out);
+
+  static Cache::CacheItemHelper* GetCacheItemHelper();
+
   const std::string& db_id_;
   const std::string& db_session_id_;
 
@@ -133,6 +142,12 @@ class BlobSource {
 
   // A cache to store uncompressed blobs.
   std::shared_ptr<Cache> blob_cache_;
+
+  // The control option of how the cache tiers will be used. Currently rocksdb
+  // support block/blob cache (volatile tier) and secondary cache (this tier
+  // isn't strictly speaking a non-volatile tier since the compressed cache in
+  // this tier is in volatile memory).
+  const CacheTier lowest_used_cache_tier_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

db/blob/blob_source_test.cc

@@ -11,6 +11,8 @@
 #include <memory>
 #include <string>
 
+#include "cache/charged_cache.h"
+#include "cache/compressed_secondary_cache.h"
 #include "db/blob/blob_file_cache.h"
 #include "db/blob/blob_file_reader.h"
 #include "db/blob/blob_log_format.h"
@@ -21,6 +23,7 @@
 #include "options/cf_options.h"
 #include "rocksdb/options.h"
 #include "util/compression.h"
+#include "util/random.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -118,6 +121,8 @@ class BlobSourceTest : public DBTestBase {
     co.capacity = 8 << 20;
     co.num_shard_bits = 2;
     co.metadata_charge_policy = kDontChargeCacheMetadata;
+    co.high_pri_pool_ratio = 0.2;
+    co.low_pri_pool_ratio = 0.2;
     options_.blob_cache = NewLRUCache(co);
     options_.lowest_used_cache_tier = CacheTier::kVolatileTier;
 
@@ -181,9 +186,10 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
   FileOptions file_options;
   constexpr HistogramImpl* blob_file_read_hist = nullptr;
 
-  std::unique_ptr<BlobFileCache> blob_file_cache(new BlobFileCache(
-      backing_cache.get(), &immutable_options, &file_options, column_family_id,
-      blob_file_read_hist, nullptr /*IOTracer*/));
+  std::unique_ptr<BlobFileCache> blob_file_cache =
+      std::make_unique<BlobFileCache>(
+          backing_cache.get(), &immutable_options, &file_options,
+          column_family_id, blob_file_read_hist, nullptr /*IOTracer*/);
 
   BlobSource blob_source(&immutable_options, db_id_, db_session_id_,
                          blob_file_cache.get());
@@ -479,9 +485,10 @@ TEST_F(BlobSourceTest, GetCompressedBlobs) {
   auto backing_cache = NewLRUCache(capacity);  // Blob file cache
 
   FileOptions file_options;
-  std::unique_ptr<BlobFileCache> blob_file_cache(new BlobFileCache(
-      backing_cache.get(), &immutable_options, &file_options, column_family_id,
-      nullptr /*HistogramImpl*/, nullptr /*IOTracer*/));
+  std::unique_ptr<BlobFileCache> blob_file_cache =
+      std::make_unique<BlobFileCache>(
+          backing_cache.get(), &immutable_options, &file_options,
+          column_family_id, nullptr /*HistogramImpl*/, nullptr /*IOTracer*/);
 
   BlobSource blob_source(&immutable_options, db_id_, db_session_id_,
                          blob_file_cache.get());
@@ -623,9 +630,10 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromMultiFiles) {
   FileOptions file_options;
   constexpr HistogramImpl* blob_file_read_hist = nullptr;
 
-  std::unique_ptr<BlobFileCache> blob_file_cache(new BlobFileCache(
-      backing_cache.get(), &immutable_options, &file_options, column_family_id,
-      blob_file_read_hist, nullptr /*IOTracer*/));
+  std::unique_ptr<BlobFileCache> blob_file_cache =
+      std::make_unique<BlobFileCache>(
+          backing_cache.get(), &immutable_options, &file_options,
+          column_family_id, blob_file_read_hist, nullptr /*IOTracer*/);
 
   BlobSource blob_source(&immutable_options, db_id_, db_session_id_,
                          blob_file_cache.get());
@@ -805,9 +813,10 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
   FileOptions file_options;
   constexpr HistogramImpl* blob_file_read_hist = nullptr;
 
-  std::unique_ptr<BlobFileCache> blob_file_cache(new BlobFileCache(
-      backing_cache.get(), &immutable_options, &file_options, column_family_id,
-      blob_file_read_hist, nullptr /*IOTracer*/));
+  std::unique_ptr<BlobFileCache> blob_file_cache =
+      std::make_unique<BlobFileCache>(
+          backing_cache.get(), &immutable_options, &file_options,
+          column_family_id, blob_file_read_hist, nullptr /*IOTracer*/);
 
   BlobSource blob_source(&immutable_options, db_id_, db_session_id_,
                          blob_file_cache.get());
@@ -1020,6 +1029,512 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
   }
 }
 
+class BlobSecondaryCacheTest : public DBTestBase {
+ protected:
+ public:
+  explicit BlobSecondaryCacheTest()
+      : DBTestBase("blob_secondary_cache_test", /*env_do_fsync=*/true) {
+    options_.env = env_;
+    options_.enable_blob_files = true;
+    options_.create_if_missing = true;
+
+    // Set a small cache capacity to evict entries from the cache, and to test
+    // that secondary cache is used properly.
+    lru_cache_ops_.capacity = 1024;
+    lru_cache_ops_.num_shard_bits = 0;
+    lru_cache_ops_.strict_capacity_limit = true;
+    lru_cache_ops_.metadata_charge_policy = kDontChargeCacheMetadata;
+    lru_cache_ops_.high_pri_pool_ratio = 0.2;
+    lru_cache_ops_.low_pri_pool_ratio = 0.2;
+
+    secondary_cache_opts_.capacity = 8 << 20;  // 8 MB
+    secondary_cache_opts_.num_shard_bits = 0;
+    secondary_cache_opts_.metadata_charge_policy = kDontChargeCacheMetadata;
+
+    // Read blobs from the secondary cache if they are not in the primary cache
+    options_.lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier;
+
+    assert(db_->GetDbIdentity(db_id_).ok());
+    assert(db_->GetDbSessionId(db_session_id_).ok());
+  }
+
+  Options options_;
+
+  LRUCacheOptions lru_cache_ops_;
+  CompressedSecondaryCacheOptions secondary_cache_opts_;
+
+  std::string db_id_;
+  std::string db_session_id_;
+};
+
+TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
+  if (!Snappy_Supported()) {
+    return;
+  }
+
+  secondary_cache_opts_.compression_type = kSnappyCompression;
+  lru_cache_ops_.secondary_cache =
+      NewCompressedSecondaryCache(secondary_cache_opts_);
+  options_.blob_cache = NewLRUCache(lru_cache_ops_);
+
+  options_.cf_paths.emplace_back(
+      test::PerThreadDBPath(
+          env_, "BlobSecondaryCacheTest_GetBlobsFromSecondaryCache"),
+      0);
+
+  options_.statistics = CreateDBStatistics();
+  Statistics* statistics = options_.statistics.get();
+  assert(statistics);
+
+  DestroyAndReopen(options_);
+
+  ImmutableOptions immutable_options(options_);
+
+  constexpr uint32_t column_family_id = 1;
+  constexpr bool has_ttl = false;
+  constexpr ExpirationRange expiration_range;
+  constexpr uint64_t file_number = 1;
+
+  Random rnd(301);
+
+  std::vector<std::string> key_strs{"key0", "key1"};
+  std::vector<std::string> blob_strs{rnd.RandomString(1010),
+                                     rnd.RandomString(1020)};
+
+  std::vector<Slice> keys{key_strs[0], key_strs[1]};
+  std::vector<Slice> blobs{blob_strs[0], blob_strs[1]};
+
+  std::vector<uint64_t> blob_offsets(keys.size());
+  std::vector<uint64_t> blob_sizes(keys.size());
+
+  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
+                expiration_range, file_number, keys, blobs, kNoCompression,
+                blob_offsets, blob_sizes);
+
+  constexpr size_t capacity = 1024;
+  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);
+
+  FileOptions file_options;
+  constexpr HistogramImpl* blob_file_read_hist = nullptr;
+
+  std::unique_ptr<BlobFileCache> blob_file_cache(new BlobFileCache(
+      backing_cache.get(), &immutable_options, &file_options, column_family_id,
+      blob_file_read_hist, nullptr /*IOTracer*/));
+
+  BlobSource blob_source(&immutable_options, db_id_, db_session_id_,
+                         blob_file_cache.get());
+
+  CacheHandleGuard<BlobFileReader> file_reader;
+  ASSERT_OK(blob_source.GetBlobFileReader(file_number, &file_reader));
+  ASSERT_NE(file_reader.GetValue(), nullptr);
+  const uint64_t file_size = file_reader.GetValue()->GetFileSize();
+  ASSERT_EQ(file_reader.GetValue()->GetCompressionType(), kNoCompression);
+
+  ReadOptions read_options;
+  read_options.verify_checksums = true;
+
+  auto blob_cache = options_.blob_cache;
+  auto secondary_cache = lru_cache_ops_.secondary_cache;
+
+  Cache::CreateCallback create_cb = [&](const void* buf, size_t size,
+                                        void** out_obj,
+                                        size_t* charge) -> Status {
+    std::string* blob = new std::string();
+    blob->assign(static_cast<const char*>(buf), size);
+    *out_obj = blob;
+    *charge = size;
+    return Status::OK();
+  };
+
+  {
+    // GetBlob
+    std::vector<PinnableSlice> values(keys.size());
+
+    read_options.fill_cache = true;
+    get_perf_context()->Reset();
+
+    // key0 should be filled to the primary cache from the blob file.
+    ASSERT_OK(blob_source.GetBlob(read_options, keys[0], file_number,
+                                  blob_offsets[0], file_size, blob_sizes[0],
+                                  kNoCompression, nullptr /* prefetch_buffer */,
+                                  &values[0], nullptr /* bytes_read */));
+    ASSERT_EQ(values[0], blobs[0]);
+    ASSERT_TRUE(
+        blob_source.TEST_BlobInCache(file_number, file_size, blob_offsets[0]));
+
+    // key0 should be demoted to the secondary cache, and key1 should be filled
+    // to the primary cache from the blob file.
+    ASSERT_OK(blob_source.GetBlob(read_options, keys[1], file_number,
+                                  blob_offsets[1], file_size, blob_sizes[1],
+                                  kNoCompression, nullptr /* prefetch_buffer */,
+                                  &values[1], nullptr /* bytes_read */));
+    ASSERT_EQ(values[1], blobs[1]);
+    ASSERT_TRUE(
+        blob_source.TEST_BlobInCache(file_number, file_size, blob_offsets[1]));
+
+    OffsetableCacheKey base_cache_key(db_id_, db_session_id_, file_number);
+
+    // blob_cache here only looks at the primary cache since we didn't provide
+    // the cache item helper for the secondary cache. However, since key0 is
+    // demoted to the secondary cache, we shouldn't be able to find it in the
+    // primary cache.
+    {
+      CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[0]);
+      const Slice key0 = cache_key.AsSlice();
+      auto handle0 = blob_cache->Lookup(key0, statistics);
+      ASSERT_EQ(handle0, nullptr);
+
+      // key0 should be in the secondary cache. After looking up key0 in the
+      // secondary cache, it will be erased from the secondary cache.
+      bool is_in_sec_cache = false;
+      auto sec_handle0 =
+          secondary_cache->Lookup(key0, create_cb, true, is_in_sec_cache);
+      ASSERT_FALSE(is_in_sec_cache);
+      ASSERT_NE(sec_handle0, nullptr);
+      ASSERT_TRUE(sec_handle0->IsReady());
+      auto value = static_cast<std::string*>(sec_handle0->Value());
+      ASSERT_EQ(*value, blobs[0]);
+      delete value;
+
+      // key0 doesn't exist in the blob cache
+      ASSERT_FALSE(blob_source.TEST_BlobInCache(file_number, file_size,
+                                                blob_offsets[0]));
+    }
+
+    // key1 should exist in the primary cache.
+    {
+      CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[1]);
+      const Slice key1 = cache_key.AsSlice();
+      auto handle1 = blob_cache->Lookup(key1, statistics);
+      ASSERT_NE(handle1, nullptr);
+      blob_cache->Release(handle1);
+
+      bool is_in_sec_cache = false;
+      auto sec_handle1 =
+          secondary_cache->Lookup(key1, create_cb, true, is_in_sec_cache);
+      ASSERT_FALSE(is_in_sec_cache);
+      ASSERT_EQ(sec_handle1, nullptr);
+
+      ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size,
+                                               blob_offsets[1]));
+    }
+
+    {
+      // fetch key0 from the blob file to the primary cache.
+      ASSERT_OK(blob_source.GetBlob(
+          read_options, keys[0], file_number, blob_offsets[0], file_size,
+          blob_sizes[0], kNoCompression, nullptr /* prefetch_buffer */,
+          &values[0], nullptr /* bytes_read */));
+      ASSERT_EQ(values[0], blobs[0]);
+
+      // key0 should be in the primary cache.
+      CacheKey cache_key0 = base_cache_key.WithOffset(blob_offsets[0]);
+      const Slice key0 = cache_key0.AsSlice();
+      auto handle0 = blob_cache->Lookup(key0, statistics);
+      ASSERT_NE(handle0, nullptr);
+      auto value = static_cast<std::string*>(blob_cache->Value(handle0));
+      ASSERT_EQ(*value, blobs[0]);
+      blob_cache->Release(handle0);
+
+      // key1 is not in the primary cache, and it should be demoted to the
+      // secondary cache.
+      CacheKey cache_key1 = base_cache_key.WithOffset(blob_offsets[1]);
+      const Slice key1 = cache_key1.AsSlice();
+      auto handle1 = blob_cache->Lookup(key1, statistics);
+      ASSERT_EQ(handle1, nullptr);
+
+      // erase key0 from the primary cache.
+      blob_cache->Erase(key0);
+      handle0 = blob_cache->Lookup(key0, statistics);
+      ASSERT_EQ(handle0, nullptr);
+
+      // key1 promotion should succeed due to the primary cache being empty. we
+      // did't call secondary cache's Lookup() here, because it will remove the
+      // key but it won't be able to promote the key to the primary cache.
+      // Instead we use the end-to-end blob source API to promote the key to
+      // the primary cache.
+      ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size,
+                                               blob_offsets[1]));
+
+      // key1 should be in the primary cache.
+      handle1 = blob_cache->Lookup(key1, statistics);
+      ASSERT_NE(handle1, nullptr);
+      value = static_cast<std::string*>(blob_cache->Value(handle1));
+      ASSERT_EQ(*value, blobs[1]);
+      blob_cache->Release(handle1);
+    }
+  }
+}
+
+class BlobSourceCacheReservationTest : public DBTestBase {
+ public:
+  explicit BlobSourceCacheReservationTest()
+      : DBTestBase("blob_source_cache_reservation_test",
+                   /*env_do_fsync=*/true) {
+    options_.env = env_;
+    options_.enable_blob_files = true;
+    options_.create_if_missing = true;
+
+    LRUCacheOptions co;
+    co.capacity = kCacheCapacity;
+    co.num_shard_bits = kNumShardBits;
+    co.metadata_charge_policy = kDontChargeCacheMetadata;
+
+    co.high_pri_pool_ratio = 0.0;
+    co.low_pri_pool_ratio = 0.0;
+    std::shared_ptr<Cache> blob_cache = NewLRUCache(co);
+
+    co.high_pri_pool_ratio = 0.5;
+    co.low_pri_pool_ratio = 0.5;
+    std::shared_ptr<Cache> block_cache = NewLRUCache(co);
+
+    options_.blob_cache = blob_cache;
+    options_.lowest_used_cache_tier = CacheTier::kVolatileTier;
+
+    BlockBasedTableOptions block_based_options;
+    block_based_options.no_block_cache = false;
+    block_based_options.block_cache = block_cache;
+    block_based_options.cache_usage_options.options_overrides.insert(
+        {CacheEntryRole::kBlobCache,
+         {/* charged = */ CacheEntryRoleOptions::Decision::kEnabled}});
+    options_.table_factory.reset(
+        NewBlockBasedTableFactory(block_based_options));
+
+    assert(db_->GetDbIdentity(db_id_).ok());
+    assert(db_->GetDbSessionId(db_session_id_).ok());
+  }
+
+  void GenerateKeysAndBlobs() {
+    for (size_t i = 0; i < kNumBlobs; ++i) {
+      key_strs_.push_back("key" + std::to_string(i));
+      blob_strs_.push_back("blob" + std::to_string(i));
+    }
+
+    blob_file_size_ = BlobLogHeader::kSize;
+    for (size_t i = 0; i < kNumBlobs; ++i) {
+      keys_.push_back({key_strs_[i]});
+      blobs_.push_back({blob_strs_[i]});
+      blob_file_size_ +=
+          BlobLogRecord::kHeaderSize + keys_[i].size() + blobs_[i].size();
+    }
+    blob_file_size_ += BlobLogFooter::kSize;
+  }
+
+  static constexpr std::size_t kSizeDummyEntry = CacheReservationManagerImpl<
+      CacheEntryRole::kBlobCache>::GetDummyEntrySize();
+  static constexpr std::size_t kCacheCapacity = 1 * kSizeDummyEntry;
+  static constexpr int kNumShardBits = 0;  // 2^0 shard
+
+  static constexpr uint32_t kColumnFamilyId = 1;
+  static constexpr bool kHasTTL = false;
+  static constexpr uint64_t kBlobFileNumber = 1;
+  static constexpr size_t kNumBlobs = 16;
+
+  std::vector<Slice> keys_;
+  std::vector<Slice> blobs_;
+  std::vector<std::string> key_strs_;
+  std::vector<std::string> blob_strs_;
+  uint64_t blob_file_size_;
+
+  Options options_;
+  std::string db_id_;
+  std::string db_session_id_;
+};
+
+#ifndef ROCKSDB_LITE
+TEST_F(BlobSourceCacheReservationTest, SimpleCacheReservation) {
+  options_.cf_paths.emplace_back(
+      test::PerThreadDBPath(
+          env_, "BlobSourceCacheReservationTest_SimpleCacheReservation"),
+      0);
+
+  GenerateKeysAndBlobs();
+
+  DestroyAndReopen(options_);
+
+  ImmutableOptions immutable_options(options_);
+
+  constexpr ExpirationRange expiration_range;
+
+  std::vector<uint64_t> blob_offsets(keys_.size());
+  std::vector<uint64_t> blob_sizes(keys_.size());
+
+  WriteBlobFile(immutable_options, kColumnFamilyId, kHasTTL, expiration_range,
+                expiration_range, kBlobFileNumber, keys_, blobs_,
+                kNoCompression, blob_offsets, blob_sizes);
+
+  constexpr size_t capacity = 10;
+  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);
+
+  FileOptions file_options;
+  constexpr HistogramImpl* blob_file_read_hist = nullptr;
+
+  std::unique_ptr<BlobFileCache> blob_file_cache =
+      std::make_unique<BlobFileCache>(
+          backing_cache.get(), &immutable_options, &file_options,
+          kColumnFamilyId, blob_file_read_hist, nullptr /*IOTracer*/);
+
+  BlobSource blob_source(&immutable_options, db_id_, db_session_id_,
+                         blob_file_cache.get());
+
+  ConcurrentCacheReservationManager* cache_res_mgr =
+      static_cast<ChargedCache*>(blob_source.GetBlobCache())
+          ->TEST_GetCacheReservationManager();
+  ASSERT_NE(cache_res_mgr, nullptr);
+
+  ReadOptions read_options;
+  read_options.verify_checksums = true;
+
+  std::vector<PinnableSlice> values(keys_.size());
+
+  {
+    read_options.fill_cache = false;
+
+    for (size_t i = 0; i < kNumBlobs; ++i) {
+      ASSERT_OK(blob_source.GetBlob(
+          read_options, keys_[i], kBlobFileNumber, blob_offsets[i],
+          blob_file_size_, blob_sizes[i], kNoCompression,
+          nullptr /* prefetch_buffer */, &values[i], nullptr /* bytes_read */));
+      ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), 0);
+      ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), 0);
+    }
+  }
+
+  {
+    read_options.fill_cache = true;
+
+    // num_blobs is 16, so the total blob cache usage is less than a single
+    // dummy entry. Therefore, cache reservation manager only reserves one dummy
+    // entry here.
+    uint64_t blob_bytes = 0;
+    for (size_t i = 0; i < kNumBlobs; ++i) {
+      ASSERT_OK(blob_source.GetBlob(
+          read_options, keys_[i], kBlobFileNumber, blob_offsets[i],
+          blob_file_size_, blob_sizes[i], kNoCompression,
+          nullptr /* prefetch_buffer */, &values[i], nullptr /* bytes_read */));
+      blob_bytes += blob_sizes[i];
+      ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), kSizeDummyEntry);
+      ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), blob_bytes);
+      ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(),
+                options_.blob_cache->GetUsage());
+    }
+  }
+
+  {
+    OffsetableCacheKey base_cache_key(db_id_, db_session_id_, kBlobFileNumber);
+    size_t blob_bytes = options_.blob_cache->GetUsage();
+
+    for (size_t i = 0; i < kNumBlobs; ++i) {
+      CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[i]);
+      // We didn't call options_.blob_cache->Erase() here, this is because
+      // the cache wrapper's Erase() method must be called to update the
+      // cache usage after erasing the cache entry.
+      blob_source.GetBlobCache()->Erase(cache_key.AsSlice());
+      if (i == kNumBlobs - 1) {
+        // The last blob is not in the cache. cache_res_mgr should not reserve
+        // any space for it.
+        ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), 0);
+      } else {
+        ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), kSizeDummyEntry);
+      }
+      blob_bytes -= blob_sizes[i];
+      ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), blob_bytes);
+      ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(),
+                options_.blob_cache->GetUsage());
+    }
+  }
+}
+
+TEST_F(BlobSourceCacheReservationTest, IncreaseCacheReservationOnFullCache) {
+  options_.cf_paths.emplace_back(
+      test::PerThreadDBPath(
+          env_,
+          "BlobSourceCacheReservationTest_IncreaseCacheReservationOnFullCache"),
+      0);
+
+  GenerateKeysAndBlobs();
+
+  DestroyAndReopen(options_);
+
+  ImmutableOptions immutable_options(options_);
+  constexpr size_t blob_size = kSizeDummyEntry / (kNumBlobs / 2);
+  for (size_t i = 0; i < kNumBlobs; ++i) {
+    blob_file_size_ -= blobs_[i].size();  // old blob size
+    blob_strs_[i].resize(blob_size, '@');
+    blobs_[i] = Slice(blob_strs_[i]);
+    blob_file_size_ += blobs_[i].size();  // new blob size
+  }
+
+  std::vector<uint64_t> blob_offsets(keys_.size());
+  std::vector<uint64_t> blob_sizes(keys_.size());
+
+  constexpr ExpirationRange expiration_range;
+  WriteBlobFile(immutable_options, kColumnFamilyId, kHasTTL, expiration_range,
+                expiration_range, kBlobFileNumber, keys_, blobs_,
+                kNoCompression, blob_offsets, blob_sizes);
+
+  constexpr size_t capacity = 10;
+  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);
+
+  FileOptions file_options;
+  constexpr HistogramImpl* blob_file_read_hist = nullptr;
+
+  std::unique_ptr<BlobFileCache> blob_file_cache =
+      std::make_unique<BlobFileCache>(
+          backing_cache.get(), &immutable_options, &file_options,
+          kColumnFamilyId, blob_file_read_hist, nullptr /*IOTracer*/);
+
+  BlobSource blob_source(&immutable_options, db_id_, db_session_id_,
+                         blob_file_cache.get());
+
+  ConcurrentCacheReservationManager* cache_res_mgr =
+      static_cast<ChargedCache*>(blob_source.GetBlobCache())
+          ->TEST_GetCacheReservationManager();
+  ASSERT_NE(cache_res_mgr, nullptr);
+
+  ReadOptions read_options;
+  read_options.verify_checksums = true;
+
+  std::vector<PinnableSlice> values(keys_.size());
+
+  {
+    read_options.fill_cache = false;
+
+    for (size_t i = 0; i < kNumBlobs; ++i) {
+      ASSERT_OK(blob_source.GetBlob(
+          read_options, keys_[i], kBlobFileNumber, blob_offsets[i],
+          blob_file_size_, blob_sizes[i], kNoCompression,
+          nullptr /* prefetch_buffer */, &values[i], nullptr /* bytes_read */));
+      ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), 0);
+      ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), 0);
+    }
+  }
+
+  {
+    read_options.fill_cache = true;
+
+    // Since we resized each blob to be kSizeDummyEntry / (num_blobs/ 2), we
+    // should observe cache eviction for the second half blobs.
+    uint64_t blob_bytes = 0;
+    for (size_t i = 0; i < kNumBlobs; ++i) {
+      ASSERT_OK(blob_source.GetBlob(
+          read_options, keys_[i], kBlobFileNumber, blob_offsets[i],
+          blob_file_size_, blob_sizes[i], kNoCompression,
+          nullptr /* prefetch_buffer */, &values[i], nullptr /* bytes_read */));
+      blob_bytes += blob_sizes[i];
+      ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), kSizeDummyEntry);
+      if (i >= kNumBlobs / 2) {
+        ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), kSizeDummyEntry);
+      } else {
+        ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), blob_bytes);
+      }
+      ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(),
+                options_.blob_cache->GetUsage());
+    }
+  }
+}
+#endif  // ROCKSDB_LITE
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/blob/db_blob_basic_test.cc

@@ -135,6 +135,8 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
   block_based_options.cache_index_and_filter_blocks = true;
   options.table_factory.reset(NewBlockBasedTableFactory(block_based_options));
 
+  options.statistics = CreateDBStatistics();
+
   Reopen(options);
 
   int num_blobs = 5;
@@ -165,6 +167,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
       ++i;
     }
     ASSERT_EQ(i, num_blobs);
+    ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0);
   }
 
   {
@@ -180,6 +183,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
     iter->SeekToFirst();
     ASSERT_NOK(iter->status());
     ASSERT_FALSE(iter->Valid());
+    ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0);
   }
 
   {
@@ -198,6 +202,8 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
       ++i;
     }
     ASSERT_EQ(i, num_blobs);
+    ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD),
+              num_blobs);
   }
 
   {
@@ -217,6 +223,149 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
       ++i;
     }
     ASSERT_EQ(i, num_blobs);
+    ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 0);
+  }
+}
+
+TEST_F(DBBlobBasicTest, IterateBlobsFromCachePinning) {
+  constexpr size_t min_blob_size = 6;
+
+  Options options = GetDefaultOptions();
+
+  LRUCacheOptions cache_options;
+  cache_options.capacity = 2048;
+  cache_options.num_shard_bits = 0;
+  cache_options.metadata_charge_policy = kDontChargeCacheMetadata;
+
+  options.blob_cache = NewLRUCache(cache_options);
+  options.enable_blob_files = true;
+  options.min_blob_size = min_blob_size;
+
+  Reopen(options);
+
+  // Put then iterate over three key-values. The second value is below the size
+  // limit and is thus stored inline; the other two are stored separately as
+  // blobs. We expect to have something pinned in the cache iff we are
+  // positioned on a blob.
+
+  constexpr char first_key[] = "first_key";
+  constexpr char first_value[] = "long_value";
+  static_assert(sizeof(first_value) - 1 >= min_blob_size,
+                "first_value too short to be stored as blob");
+
+  ASSERT_OK(Put(first_key, first_value));
+
+  constexpr char second_key[] = "second_key";
+  constexpr char second_value[] = "short";
+  static_assert(sizeof(second_value) - 1 < min_blob_size,
+                "second_value too long to be inlined");
+
+  ASSERT_OK(Put(second_key, second_value));
+
+  constexpr char third_key[] = "third_key";
+  constexpr char third_value[] = "other_long_value";
+  static_assert(sizeof(third_value) - 1 >= min_blob_size,
+                "third_value too short to be stored as blob");
+
+  ASSERT_OK(Put(third_key, third_value));
+
+  ASSERT_OK(Flush());
+
+  {
+    ReadOptions read_options;
+    read_options.fill_cache = true;
+
+    std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
+
+    iter->SeekToFirst();
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_OK(iter->status());
+    ASSERT_EQ(iter->key(), first_key);
+    ASSERT_EQ(iter->value(), first_value);
+
+    iter->Next();
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_OK(iter->status());
+    ASSERT_EQ(iter->key(), second_key);
+    ASSERT_EQ(iter->value(), second_value);
+
+    iter->Next();
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_OK(iter->status());
+    ASSERT_EQ(iter->key(), third_key);
+    ASSERT_EQ(iter->value(), third_value);
+
+    iter->Next();
+    ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
+  }
+
+  {
+    ReadOptions read_options;
+    read_options.fill_cache = false;
+    read_options.read_tier = kBlockCacheTier;
+
+    std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
+
+    iter->SeekToFirst();
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_OK(iter->status());
+    ASSERT_EQ(iter->key(), first_key);
+    ASSERT_EQ(iter->value(), first_value);
+    ASSERT_GT(options.blob_cache->GetPinnedUsage(), 0);
+
+    iter->Next();
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_OK(iter->status());
+    ASSERT_EQ(iter->key(), second_key);
+    ASSERT_EQ(iter->value(), second_value);
+    ASSERT_EQ(options.blob_cache->GetPinnedUsage(), 0);
+
+    iter->Next();
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_OK(iter->status());
+    ASSERT_EQ(iter->key(), third_key);
+    ASSERT_EQ(iter->value(), third_value);
+    ASSERT_GT(options.blob_cache->GetPinnedUsage(), 0);
+
+    iter->Next();
+    ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
+    ASSERT_EQ(options.blob_cache->GetPinnedUsage(), 0);
+  }
+
+  {
+    ReadOptions read_options;
+    read_options.fill_cache = false;
+    read_options.read_tier = kBlockCacheTier;
+
+    std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
+
+    iter->SeekToLast();
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_OK(iter->status());
+    ASSERT_EQ(iter->key(), third_key);
+    ASSERT_EQ(iter->value(), third_value);
+    ASSERT_GT(options.blob_cache->GetPinnedUsage(), 0);
+
+    iter->Prev();
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_OK(iter->status());
+    ASSERT_EQ(iter->key(), second_key);
+    ASSERT_EQ(iter->value(), second_value);
+    ASSERT_EQ(options.blob_cache->GetPinnedUsage(), 0);
+
+    iter->Prev();
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_OK(iter->status());
+    ASSERT_EQ(iter->key(), first_key);
+    ASSERT_EQ(iter->value(), first_value);
+    ASSERT_GT(options.blob_cache->GetPinnedUsage(), 0);
+
+    iter->Prev();
+    ASSERT_FALSE(iter->Valid());
+    ASSERT_OK(iter->status());
+    ASSERT_EQ(options.blob_cache->GetPinnedUsage(), 0);
   }
 }
 
@@ -1432,6 +1581,126 @@ TEST_P(DBBlobBasicIOErrorTest, CompactionFilterReadBlob_IOError) {
   SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
+TEST_F(DBBlobBasicTest, WarmCacheWithBlobsDuringFlush) {
+  Options options = GetDefaultOptions();
+
+  LRUCacheOptions co;
+  co.capacity = 1 << 25;
+  co.num_shard_bits = 2;
+  co.metadata_charge_policy = kDontChargeCacheMetadata;
+  auto backing_cache = NewLRUCache(co);
+
+  options.blob_cache = backing_cache;
+
+  BlockBasedTableOptions block_based_options;
+  block_based_options.no_block_cache = false;
+  block_based_options.block_cache = backing_cache;
+  block_based_options.cache_index_and_filter_blocks = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(block_based_options));
+
+  options.enable_blob_files = true;
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.enable_blob_garbage_collection = true;
+  options.blob_garbage_collection_age_cutoff = 1.0;
+  options.prepopulate_blob_cache = PrepopulateBlobCache::kFlushOnly;
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+
+  DestroyAndReopen(options);
+
+  constexpr size_t kNumBlobs = 10;
+  constexpr size_t kValueSize = 100;
+
+  std::string value(kValueSize, 'a');
+
+  for (size_t i = 1; i <= kNumBlobs; i++) {
+    ASSERT_OK(Put(std::to_string(i), value));
+    ASSERT_OK(Put(std::to_string(i + kNumBlobs), value));  // Add some overlap
+    ASSERT_OK(Flush());
+    ASSERT_EQ(i * 2, options.statistics->getTickerCount(BLOB_DB_CACHE_ADD));
+    ASSERT_EQ(value, Get(std::to_string(i)));
+    ASSERT_EQ(value, Get(std::to_string(i + kNumBlobs)));
+    ASSERT_EQ(0, options.statistics->getTickerCount(BLOB_DB_CACHE_MISS));
+    ASSERT_EQ(i * 2, options.statistics->getTickerCount(BLOB_DB_CACHE_HIT));
+  }
+
+  // Verify compaction not counted
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
+                              /*end=*/nullptr));
+  EXPECT_EQ(kNumBlobs * 2,
+            options.statistics->getTickerCount(BLOB_DB_CACHE_ADD));
+}
+
+#ifndef ROCKSDB_LITE
+TEST_F(DBBlobBasicTest, DynamicallyWarmCacheDuringFlush) {
+  Options options = GetDefaultOptions();
+
+  LRUCacheOptions co;
+  co.capacity = 1 << 25;
+  co.num_shard_bits = 2;
+  co.metadata_charge_policy = kDontChargeCacheMetadata;
+  auto backing_cache = NewLRUCache(co);
+
+  options.blob_cache = backing_cache;
+
+  BlockBasedTableOptions block_based_options;
+  block_based_options.no_block_cache = false;
+  block_based_options.block_cache = backing_cache;
+  block_based_options.cache_index_and_filter_blocks = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(block_based_options));
+
+  options.enable_blob_files = true;
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.enable_blob_garbage_collection = true;
+  options.blob_garbage_collection_age_cutoff = 1.0;
+  options.prepopulate_blob_cache = PrepopulateBlobCache::kFlushOnly;
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+
+  DestroyAndReopen(options);
+
+  constexpr size_t kNumBlobs = 10;
+  constexpr size_t kValueSize = 100;
+
+  std::string value(kValueSize, 'a');
+
+  for (size_t i = 1; i <= 5; i++) {
+    ASSERT_OK(Put(std::to_string(i), value));
+    ASSERT_OK(Put(std::to_string(i + kNumBlobs), value));  // Add some overlap
+    ASSERT_OK(Flush());
+    ASSERT_EQ(2, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD));
+
+    ASSERT_EQ(value, Get(std::to_string(i)));
+    ASSERT_EQ(value, Get(std::to_string(i + kNumBlobs)));
+    ASSERT_EQ(0, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD));
+    ASSERT_EQ(0,
+              options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS));
+    ASSERT_EQ(2, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT));
+  }
+
+  ASSERT_OK(dbfull()->SetOptions({{"prepopulate_blob_cache", "kDisable"}}));
+
+  for (size_t i = 6; i <= kNumBlobs; i++) {
+    ASSERT_OK(Put(std::to_string(i), value));
+    ASSERT_OK(Put(std::to_string(i + kNumBlobs), value));  // Add some overlap
+    ASSERT_OK(Flush());
+    ASSERT_EQ(0, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD));
+
+    ASSERT_EQ(value, Get(std::to_string(i)));
+    ASSERT_EQ(value, Get(std::to_string(i + kNumBlobs)));
+    ASSERT_EQ(2, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD));
+    ASSERT_EQ(2,
+              options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS));
+    ASSERT_EQ(0, options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT));
+  }
+
+  // Verify compaction not counted
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
+                              /*end=*/nullptr));
+  EXPECT_EQ(0, options.statistics->getTickerCount(BLOB_DB_CACHE_ADD));
+}
+#endif  // !ROCKSDB_LITE
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/blob/db_blob_compaction_test.cc

@@ -54,6 +54,26 @@ class FilterByKeyLength : public CompactionFilter {
   size_t length_threshold_;
 };
 
+class FilterByValueLength : public CompactionFilter {
+ public:
+  explicit FilterByValueLength(size_t len) : length_threshold_(len) {}
+  const char* Name() const override {
+    return "rocksdb.compaction.filter.by.value.length";
+  }
+  CompactionFilter::Decision FilterV2(
+      int /*level*/, const Slice& /*key*/, ValueType /*value_type*/,
+      const Slice& existing_value, std::string* /*new_value*/,
+      std::string* /*skip_until*/) const override {
+    if (existing_value.size() < length_threshold_) {
+      return CompactionFilter::Decision::kRemove;
+    }
+    return CompactionFilter::Decision::kKeep;
+  }
+
+ private:
+  size_t length_threshold_;
+};
+
 class BadBlobCompactionFilter : public CompactionFilter {
  public:
   explicit BadBlobCompactionFilter(std::string prefix,
@@ -243,6 +263,55 @@ TEST_F(DBBlobCompactionTest, FilterByKeyLength) {
   Close();
 }
 
+TEST_F(DBBlobCompactionTest, FilterByValueLength) {
+  Options options = GetDefaultOptions();
+  options.enable_blob_files = true;
+  options.min_blob_size = 5;
+  options.create_if_missing = true;
+  constexpr size_t kValueLength = 5;
+  std::unique_ptr<CompactionFilter> compaction_filter_guard(
+      new FilterByValueLength(kValueLength));
+  options.compaction_filter = compaction_filter_guard.get();
+
+  const std::vector<std::string> short_value_keys = {"a", "e", "j"};
+  constexpr char short_value[] = "val";
+  const std::vector<std::string> long_value_keys = {"b", "f", "k"};
+  constexpr char long_value[] = "valuevalue";
+
+  DestroyAndReopen(options);
+  for (size_t i = 0; i < short_value_keys.size(); ++i) {
+    ASSERT_OK(Put(short_value_keys[i], short_value));
+  }
+  for (size_t i = 0; i < short_value_keys.size(); ++i) {
+    ASSERT_OK(Put(long_value_keys[i], long_value));
+  }
+  ASSERT_OK(Flush());
+  CompactRangeOptions cro;
+  ASSERT_OK(db_->CompactRange(cro, /*begin=*/nullptr, /*end=*/nullptr));
+  std::string value;
+  for (size_t i = 0; i < short_value_keys.size(); ++i) {
+    ASSERT_TRUE(
+        db_->Get(ReadOptions(), short_value_keys[i], &value).IsNotFound());
+    value.clear();
+  }
+  for (size_t i = 0; i < long_value_keys.size(); ++i) {
+    ASSERT_OK(db_->Get(ReadOptions(), long_value_keys[i], &value));
+    ASSERT_EQ(long_value, value);
+  }
+
+#ifndef ROCKSDB_LITE
+  const auto& compaction_stats = GetCompactionStats();
+  ASSERT_GE(compaction_stats.size(), 2);
+
+  // Filter decides between kKeep and kRemove based on value;
+  // this involves reading but not writing blobs
+  ASSERT_GT(compaction_stats[1].bytes_read_blob, 0);
+  ASSERT_EQ(compaction_stats[1].bytes_written_blob, 0);
+#endif  // ROCKSDB_LITE
+
+  Close();
+}
+
 #ifndef ROCKSDB_LITE
 TEST_F(DBBlobCompactionTest, BlobCompactWithStartingLevel) {
   Options options = GetDefaultOptions();
@@ -798,6 +867,42 @@ TEST_F(DBBlobCompactionTest, CompactionReadaheadMerge) {
   Close();
 }
 
+TEST_F(DBBlobCompactionTest, CompactionDoNotFillCache) {
+  Options options = GetDefaultOptions();
+
+  options.enable_blob_files = true;
+  options.min_blob_size = 0;
+  options.enable_blob_garbage_collection = true;
+  options.blob_garbage_collection_age_cutoff = 1.0;
+  options.disable_auto_compactions = true;
+  options.statistics = CreateDBStatistics();
+
+  LRUCacheOptions cache_options;
+  cache_options.capacity = 1 << 20;
+  cache_options.metadata_charge_policy = kDontChargeCacheMetadata;
+
+  options.blob_cache = NewLRUCache(cache_options);
+
+  Reopen(options);
+
+  ASSERT_OK(Put("key", "lime"));
+  ASSERT_OK(Put("foo", "bar"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put("key", "pie"));
+  ASSERT_OK(Put("foo", "baz"));
+  ASSERT_OK(Flush());
+
+  constexpr Slice* begin = nullptr;
+  constexpr Slice* end = nullptr;
+
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end));
+
+  ASSERT_EQ(options.statistics->getTickerCount(BLOB_DB_CACHE_ADD), 0);
+
+  Close();
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/builder.cc

@@ -66,7 +66,8 @@ Status BuildTable(
     SequenceNumber job_snapshot, SnapshotChecker* snapshot_checker,
     bool paranoid_file_checks, InternalStats* internal_stats,
     IOStatus* io_status, const std::shared_ptr<IOTracer>& io_tracer,
-    BlobFileCreationReason blob_creation_reason, EventLogger* event_logger,
+    BlobFileCreationReason blob_creation_reason,
+    const SeqnoToTimeMapping& seqno_to_time_mapping, EventLogger* event_logger,
     int job_id, const Env::IOPriority io_priority,
     TableProperties* table_properties, Env::WriteLifeTimeHint write_hint,
     const std::string* full_history_ts_low,
@@ -187,10 +188,10 @@ Status BuildTable(
          blob_file_additions)
             ? new BlobFileBuilder(
                   versions, fs, &ioptions, &mutable_cf_options, &file_options,
-                  job_id, tboptions.column_family_id,
-                  tboptions.column_family_name, io_priority, write_hint,
-                  io_tracer, blob_callback, blob_creation_reason,
-                  &blob_file_paths, blob_file_additions)
+                  tboptions.db_id, tboptions.db_session_id, job_id,
+                  tboptions.column_family_id, tboptions.column_family_name,
+                  io_priority, write_hint, io_tracer, blob_callback,
+                  blob_creation_reason, &blob_file_paths, blob_file_additions)
             : nullptr);
 
     const std::atomic<bool> kManualCompactionCanceledFalse{false};
@@ -260,6 +261,15 @@ Status BuildTable(
     if (!s.ok() || empty) {
       builder->Abandon();
     } else {
+      std::string seqno_time_mapping_str;
+      seqno_to_time_mapping.Encode(
+          seqno_time_mapping_str, meta->fd.smallest_seqno,
+          meta->fd.largest_seqno, meta->file_creation_time);
+      builder->SetSeqnoTimeTableProperties(
+          seqno_time_mapping_str,
+          ioptions.compaction_style == CompactionStyle::kCompactionStyleFIFO
+              ? meta->file_creation_time
+              : meta->oldest_ancester_time);
       s = builder->Finish();
     }
     if (io_status->ok()) {

db/builder.h

@@ -9,7 +9,9 @@
 #include <string>
 #include <utility>
 #include <vector>
+
 #include "db/range_tombstone_fragmenter.h"
+#include "db/seqno_to_time_mapping.h"
 #include "db/table_properties_collector.h"
 #include "logging/event_logger.h"
 #include "options/cf_options.h"
@@ -61,6 +63,7 @@ extern Status BuildTable(
     bool paranoid_file_checks, InternalStats* internal_stats,
     IOStatus* io_status, const std::shared_ptr<IOTracer>& io_tracer,
     BlobFileCreationReason blob_creation_reason,
+    const SeqnoToTimeMapping& seqno_to_time_mapping,
     EventLogger* event_logger = nullptr, int job_id = 0,
     const Env::IOPriority io_priority = Env::IO_HIGH,
     TableProperties* table_properties = nullptr,

db/c.cc

@@ -99,6 +99,7 @@ using ROCKSDB_NAMESPACE::Options;
 using ROCKSDB_NAMESPACE::PerfContext;
 using ROCKSDB_NAMESPACE::PerfLevel;
 using ROCKSDB_NAMESPACE::PinnableSlice;
+using ROCKSDB_NAMESPACE::PrepopulateBlobCache;
 using ROCKSDB_NAMESPACE::RandomAccessFile;
 using ROCKSDB_NAMESPACE::Range;
 using ROCKSDB_NAMESPACE::RateLimiter;
@@ -3140,6 +3141,14 @@ void rocksdb_options_set_blob_cache(rocksdb_options_t* opt,
   opt->rep.blob_cache = blob_cache->rep;
 }
 
+void rocksdb_options_set_prepopulate_blob_cache(rocksdb_options_t* opt, int t) {
+  opt->rep.prepopulate_blob_cache = static_cast<PrepopulateBlobCache>(t);
+}
+
+int rocksdb_options_get_prepopulate_blob_cache(rocksdb_options_t* opt) {
+  return static_cast<int>(opt->rep.prepopulate_blob_cache);
+}
+
 void rocksdb_options_set_num_levels(rocksdb_options_t* opt, int n) {
   opt->rep.num_levels = n;
 }
@@ -4899,6 +4908,15 @@ void rocksdb_sstfilewriter_delete_with_ts(rocksdb_sstfilewriter_t* writer,
   SaveError(errptr, writer->rep->Delete(Slice(key, keylen), Slice(ts, tslen)));
 }
 
+void rocksdb_sstfilewriter_delete_range(rocksdb_sstfilewriter_t* writer,
+                                        const char* begin_key,
+                                        size_t begin_keylen,
+                                        const char* end_key, size_t end_keylen,
+                                        char** errptr) {
+  SaveError(errptr, writer->rep->DeleteRange(Slice(begin_key, begin_keylen),
+                                             Slice(end_key, end_keylen)));
+}
+
 void rocksdb_sstfilewriter_finish(rocksdb_sstfilewriter_t* writer,
                                   char** errptr) {
   SaveError(errptr, writer->rep->Finish(nullptr));

db/c_test.c

@@ -905,6 +905,21 @@ int main(int argc, char** argv) {
     CheckGet(db, roptions, "sstk22", "v5");
     CheckGet(db, roptions, "sstk3", "v6");
 
+    rocksdb_sstfilewriter_open(writer, sstfilename, &err);
+    CheckNoError(err);
+    rocksdb_sstfilewriter_put(writer, "abc1", 4, "v7", 2, &err);
+    CheckNoError(err);
+    rocksdb_sstfilewriter_put(writer, "abc2", 4, "v8", 2, &err);
+    CheckNoError(err);
+    rocksdb_sstfilewriter_put(writer, "abc3", 4, "v9", 2, &err);
+    CheckNoError(err);
+    rocksdb_sstfilewriter_put(writer, "abc4", 4, "v10", 3, &err);
+    CheckNoError(err);
+    rocksdb_sstfilewriter_delete_range(writer, "abc1", 4, "abc4", 4, &err);
+    CheckNoError(err);
+    rocksdb_sstfilewriter_finish(writer, &err);
+    CheckNoError(err);
+
     rocksdb_ingestexternalfileoptions_destroy(ing_opt);
     rocksdb_sstfilewriter_destroy(writer);
     rocksdb_options_destroy(io_options);
@@ -2053,6 +2068,9 @@ int main(int argc, char** argv) {
     rocksdb_options_set_blob_file_starting_level(o, 5);
     CheckCondition(5 == rocksdb_options_get_blob_file_starting_level(o));
 
+    rocksdb_options_set_prepopulate_blob_cache(o, 1 /* flush only */);
+    CheckCondition(1 == rocksdb_options_get_prepopulate_blob_cache(o));
+
     // Create a copy that should be equal to the original.
     rocksdb_options_t* copy;
     copy = rocksdb_options_create_copy(o);

db/column_family.cc

@@ -699,19 +699,6 @@ ColumnFamilyData::~ColumnFamilyData() {
           id_, name_.c_str());
     }
   }
-
-  if (data_dirs_.size()) {  // Explicitly close data directories
-    Status s = Status::OK();
-    for (auto& data_dir_ptr : data_dirs_) {
-      if (data_dir_ptr) {
-        s = data_dir_ptr->Close(IOOptions(), nullptr);
-        if (!s.ok()) {
-          // TODO(zichen): add `Status Close()` and `CloseDirectories()
-          s.PermitUncheckedError();
-        }
-      }
-    }
-  }
 }
 
 bool ColumnFamilyData::UnrefAndTryDelete() {
@@ -786,13 +773,13 @@ namespace {
 std::unique_ptr<WriteControllerToken> SetupDelay(
     WriteController* write_controller, uint64_t compaction_needed_bytes,
     uint64_t prev_compaction_need_bytes, bool penalize_stop,
-    bool auto_comapctions_disabled) {
+    bool auto_compactions_disabled) {
   const uint64_t kMinWriteRate = 16 * 1024u;  // Minimum write rate 16KB/s.
 
   uint64_t max_write_rate = write_controller->max_delayed_write_rate();
   uint64_t write_rate = write_controller->delayed_write_rate();
 
-  if (auto_comapctions_disabled) {
+  if (auto_compactions_disabled) {
     // When auto compaction is disabled, always use the value user gave.
     write_rate = max_write_rate;
   } else if (write_controller->NeedsDelay() && max_write_rate > kMinWriteRate) {
@@ -1162,8 +1149,8 @@ Status ColumnFamilyData::RangesOverlapWithMemtables(
 
   auto read_seq = super_version->current->version_set()->LastSequence();
   ReadRangeDelAggregator range_del_agg(&internal_comparator_, read_seq);
-  auto* active_range_del_iter =
-      super_version->mem->NewRangeTombstoneIterator(read_opts, read_seq);
+  auto* active_range_del_iter = super_version->mem->NewRangeTombstoneIterator(
+      read_opts, read_seq, false /* immutable_memtable */);
   range_del_agg.AddTombstones(
       std::unique_ptr<FragmentedRangeTombstoneIterator>(active_range_del_iter));
   Status status;

db/column_family.h

@@ -524,6 +524,8 @@ class ColumnFamilyData {
     return file_metadata_cache_res_mgr_;
   }
 
+  SequenceNumber GetFirstMemtableSequenceNumber() const;
+
   static const uint32_t kDummyColumnFamilyDataId;
 
   // Keep track of whether the mempurge feature was ever used.

db/column_family_test.cc

@@ -1116,7 +1116,7 @@ TEST_P(ColumnFamilyTest, DifferentWriteBufferSizes) {
   CreateColumnFamilies({"one", "two", "three"});
   ColumnFamilyOptions default_cf, one, two, three;
   // setup options. all column families have max_write_buffer_number setup to 10
-  // "default" -> 100KB memtable, start flushing immediatelly
+  // "default" -> 100KB memtable, start flushing immediately
   // "one" -> 200KB memtable, start flushing with two immutable memtables
   // "two" -> 1MB memtable, start flushing with three immutable memtables
   // "three" -> 90KB memtable, start flushing with four immutable memtables
@@ -2207,9 +2207,22 @@ TEST_P(ColumnFamilyTest, FlushStaleColumnFamilies) {
   PutRandomData(0, 100, 1000);  // flush
   WaitForFlush(0);
   WaitForFlush(2);
-  // 3 files for default column families, 1 file for column family [two], zero
-  // files for column family [one], because it's empty
-  AssertCountLiveFiles(4);
+  // at least 3 files for default column families, 1 file for column family
+  // [two], zero files for column family [one], because it's empty
+  std::vector<LiveFileMetaData> metadata;
+  db_->GetLiveFilesMetaData(&metadata);
+  ASSERT_GE(metadata.size(), 4);
+  bool has_cf1_sst = false;
+  bool has_cf2_sst = false;
+  for (const auto& file : metadata) {
+    if (file.column_family_name == "one") {
+      has_cf1_sst = true;
+    } else if (file.column_family_name == "two") {
+      has_cf2_sst = true;
+    }
+  }
+  ASSERT_FALSE(has_cf1_sst);
+  ASSERT_TRUE(has_cf2_sst);
 
   ASSERT_OK(Flush(0));
   ASSERT_EQ(0, dbfull()->TEST_total_log_size());
@@ -2926,7 +2939,7 @@ TEST_P(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
   ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
 }
 
-TEST_P(ColumnFamilyTest, CreateAndDestoryOptions) {
+TEST_P(ColumnFamilyTest, CreateAndDestroyOptions) {
   std::unique_ptr<ColumnFamilyOptions> cfo(new ColumnFamilyOptions());
   ColumnFamilyHandle* cfh;
   Open();

db/compact_files_test.cc

@@ -77,7 +77,7 @@ TEST_F(CompactFilesTest, L0ConflictsFiles) {
   options.compression = kNoCompression;
 
   DB* db = nullptr;
-  DestroyDB(db_name_, options);
+  ASSERT_OK(DestroyDB(db_name_, options));
   Status s = DB::Open(options, db_name_, &db);
   assert(s.ok());
   assert(db);
@@ -128,7 +128,7 @@ TEST_F(CompactFilesTest, MultipleLevel) {
   options.listeners.emplace_back(collector);
 
   DB* db = nullptr;
-  DestroyDB(db_name_, options);
+  ASSERT_OK(DestroyDB(db_name_, options));
   Status s = DB::Open(options, db_name_, &db);
   ASSERT_OK(s);
   ASSERT_NE(db, nullptr);
@@ -211,7 +211,7 @@ TEST_F(CompactFilesTest, ObsoleteFiles) {
   options.listeners.emplace_back(collector);
 
   DB* db = nullptr;
-  DestroyDB(db_name_, options);
+  ASSERT_OK(DestroyDB(db_name_, options));
   Status s = DB::Open(options, db_name_, &db);
   ASSERT_OK(s);
   ASSERT_NE(db, nullptr);
@@ -250,7 +250,7 @@ TEST_F(CompactFilesTest, NotCutOutputOnLevel0) {
   options.listeners.emplace_back(collector);
 
   DB* db = nullptr;
-  DestroyDB(db_name_, options);
+  ASSERT_OK(DestroyDB(db_name_, options));
   Status s = DB::Open(options, db_name_, &db);
   assert(s.ok());
   assert(db);
@@ -288,7 +288,7 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
   options.listeners.emplace_back(collector);
 
   DB* db = nullptr;
-  DestroyDB(db_name_, options);
+  ASSERT_OK(DestroyDB(db_name_, options));
   Status s = DB::Open(options, db_name_, &db);
   ASSERT_OK(s);
   assert(db);
@@ -366,7 +366,7 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
   options.compaction_filter = cf.get();
 
   DB* db = nullptr;
-  DestroyDB(db_name_, options);
+  ASSERT_OK(DestroyDB(db_name_, options));
   Status s = DB::Open(options, db_name_, &db);
   ASSERT_OK(s);
 
@@ -404,7 +404,7 @@ TEST_F(CompactFilesTest, SentinelCompressionType) {
        {CompactionStyle::kCompactionStyleLevel,
         CompactionStyle::kCompactionStyleUniversal,
         CompactionStyle::kCompactionStyleNone}) {
-    DestroyDB(db_name_, Options());
+    ASSERT_OK(DestroyDB(db_name_, Options()));
     Options options;
     options.compaction_style = compaction_style;
     // L0: Snappy, L1: ZSTD, L2: Snappy
@@ -458,7 +458,7 @@ TEST_F(CompactFilesTest, GetCompactionJobInfo) {
   options.listeners.emplace_back(collector);
 
   DB* db = nullptr;
-  DestroyDB(db_name_, options);
+  ASSERT_OK(DestroyDB(db_name_, options));
   Status s = DB::Open(options, db_name_, &db);
   ASSERT_OK(s);
   assert(db);

db/compaction/compaction.cc

@@ -440,6 +440,11 @@ bool Compaction::IsTrivialMove() const {
     }
   }
 
+  // PerKeyPlacement compaction should never be trivial move.
+  if (SupportsPerKeyPlacement()) {
+    return false;
+  }
+
   return true;
 }
 
@@ -655,20 +660,23 @@ bool Compaction::IsOutputLevelEmpty() const {
 }
 
 bool Compaction::ShouldFormSubcompactions() const {
-  if (max_subcompactions_ <= 1 || cfd_ == nullptr) {
+  if (cfd_ == nullptr) {
     return false;
   }
 
-  // Note: the subcompaction boundary picking logic does not currently guarantee
-  // that all user keys that differ only by timestamp get processed by the same
-  // subcompaction.
-  if (cfd_->user_comparator()->timestamp_size() > 0) {
+  // Round-Robin pri under leveled compaction allows subcompactions by default
+  // and the number of subcompactions can be larger than max_subcompactions_
+  if (cfd_->ioptions()->compaction_pri == kRoundRobin &&
+      cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
+    return output_level_ > 0;
+  }
+
+  if (max_subcompactions_ <= 1) {
     return false;
   }
 
   if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
-    return (start_level_ == 0 || is_manual_compaction_) && output_level_ > 0 &&
-           !IsOutputLevelEmpty();
+    return (start_level_ == 0 || is_manual_compaction_) && output_level_ > 0;
   } else if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
     return number_levels_ > 1 && output_level_ > 0;
   } else {
@@ -741,10 +749,10 @@ int Compaction::EvaluatePenultimateLevel(
     return kInvalidLevel;
   }
 
-  // TODO: will add public like `options.preclude_last_level_data_seconds` for
-  //  per_key_placement feature, will check that option here. Currently, only
-  //  set by unittest
-  bool supports_per_key_placement = false;
+  bool supports_per_key_placement =
+      immutable_options.preclude_last_level_data_seconds > 0;
+
+  // it could be overridden by unittest
   TEST_SYNC_POINT_CALLBACK("Compaction::SupportsPerKeyPlacement:Enabled",
                            &supports_per_key_placement);
   if (!supports_per_key_placement) {

db/compaction/compaction.h

@@ -212,6 +212,11 @@ class Compaction {
   // Is this compaction creating a file in the bottom most level?
   bool bottommost_level() const { return bottommost_level_; }
 
+  // Is the compaction compact to the last level
+  bool is_last_level() const {
+    return output_level_ == immutable_options_.num_levels - 1;
+  }
+
   // Does this compaction include all sst files?
   bool is_full_compaction() const { return is_full_compaction_; }
 

db/compaction/compaction_iterator.cc

@@ -33,7 +33,8 @@ CompactionIterator::CompactionIterator(
     const Compaction* compaction, const CompactionFilter* compaction_filter,
     const std::atomic<bool>* shutting_down,
     const std::shared_ptr<Logger> info_log,
-    const std::string* full_history_ts_low)
+    const std::string* full_history_ts_low,
+    const SequenceNumber penultimate_level_cutoff_seqno)
     : CompactionIterator(
           input, cmp, merge_helper, last_sequence, snapshots,
           earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env,
@@ -42,7 +43,8 @@ CompactionIterator::CompactionIterator(
           manual_compaction_canceled,
           std::unique_ptr<CompactionProxy>(
               compaction ? new RealCompaction(compaction) : nullptr),
-          compaction_filter, shutting_down, info_log, full_history_ts_low) {}
+          compaction_filter, shutting_down, info_log, full_history_ts_low,
+          penultimate_level_cutoff_seqno) {}
 
 CompactionIterator::CompactionIterator(
     InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
@@ -58,7 +60,8 @@ CompactionIterator::CompactionIterator(
     const CompactionFilter* compaction_filter,
     const std::atomic<bool>* shutting_down,
     const std::shared_ptr<Logger> info_log,
-    const std::string* full_history_ts_low)
+    const std::string* full_history_ts_low,
+    const SequenceNumber penultimate_level_cutoff_seqno)
     : input_(input, cmp,
              !compaction || compaction->DoesInputReferenceBlobFiles()),
       cmp_(cmp),
@@ -77,6 +80,15 @@ CompactionIterator::CompactionIterator(
       compaction_filter_(compaction_filter),
       shutting_down_(shutting_down),
       manual_compaction_canceled_(manual_compaction_canceled),
+      bottommost_level_(!compaction_ ? false
+                                     : compaction_->bottommost_level() &&
+                                           !compaction_->allow_ingest_behind()),
+      // snapshots_ cannot be nullptr, but we will assert later in the body of
+      // the constructor.
+      visible_at_tip_(snapshots_ ? snapshots_->empty() : false),
+      earliest_snapshot_(!snapshots_ || snapshots_->empty()
+                             ? kMaxSequenceNumber
+                             : snapshots_->at(0)),
       info_log_(info_log),
       allow_data_in_errors_(allow_data_in_errors),
       enforce_single_del_contracts_(enforce_single_del_contracts),
@@ -92,27 +104,13 @@ CompactionIterator::CompactionIterator(
           CreatePrefetchBufferCollectionIfNeeded(compaction_.get())),
       current_key_committed_(false),
       cmp_with_history_ts_low_(0),
-      level_(compaction_ == nullptr ? 0 : compaction_->level()) {
+      level_(compaction_ == nullptr ? 0 : compaction_->level()),
+      penultimate_level_cutoff_seqno_(penultimate_level_cutoff_seqno) {
   assert(snapshots_ != nullptr);
-  bottommost_level_ = compaction_ == nullptr
-                          ? false
-                          : compaction_->bottommost_level() &&
-                                !compaction_->allow_ingest_behind();
+
   if (compaction_ != nullptr) {
     level_ptrs_ = std::vector<size_t>(compaction_->number_levels(), 0);
   }
-  if (snapshots_->size() == 0) {
-    // optimize for fast path if there are no snapshots
-    visible_at_tip_ = true;
-    earliest_snapshot_iter_ = snapshots_->end();
-    earliest_snapshot_ = kMaxSequenceNumber;
-    latest_snapshot_ = 0;
-  } else {
-    visible_at_tip_ = false;
-    earliest_snapshot_iter_ = snapshots_->begin();
-    earliest_snapshot_ = snapshots_->at(0);
-    latest_snapshot_ = snapshots_->back();
-  }
 #ifndef NDEBUG
   // findEarliestVisibleSnapshot assumes this ordering.
   for (size_t i = 1; i < snapshots_->size(); ++i) {
@@ -169,7 +167,7 @@ void CompactionIterator::Next() {
       current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
       key_ = current_key_.GetInternalKey();
       ikey_.user_key = current_key_.GetUserKey();
-      valid_ = true;
+      validity_info_.SetValid(ValidContext::kMerge1);
     } else {
       // We consumed all pinned merge operands, release pinned iterators
       pinned_iters_mgr_.ReleasePinnedData();
@@ -187,7 +185,7 @@ void CompactionIterator::Next() {
     NextFromInput();
   }
 
-  if (valid_) {
+  if (Valid()) {
     // Record that we've outputted a record for the current key.
     has_outputted_key_ = true;
   }
@@ -224,7 +222,6 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
   {
     StopWatchNano timer(clock_, report_detailed_time_);
     if (kTypeBlobIndex == ikey_.type) {
-      blob_value_.Reset();
       filter = compaction_filter_->FilterBlobByKey(
           level_, filter_key, &compaction_filter_value_,
           compaction_filter_skip_until_.rep());
@@ -233,7 +230,7 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
         if (compaction_ == nullptr) {
           status_ =
               Status::Corruption("Unexpected blob index outside of compaction");
-          valid_ = false;
+          validity_info_.Invalidate();
           return false;
         }
 
@@ -248,7 +245,7 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
         Status s = blob_index.DecodeFrom(value_);
         if (!s.ok()) {
           status_ = s;
-          valid_ = false;
+          validity_info_.Invalidate();
           return false;
         }
 
@@ -266,7 +263,7 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
                                      &bytes_read);
         if (!s.ok()) {
           status_ = s;
-          valid_ = false;
+          validity_info_.Invalidate();
           return false;
         }
 
@@ -290,7 +287,7 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
     // Should not reach here, since FilterV2 should never return kUndetermined.
     status_ =
         Status::NotSupported("FilterV2() should never return kUndetermined");
-    valid_ = false;
+    validity_info_.Invalidate();
     return false;
   }
 
@@ -339,7 +336,7 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
       status_ = Status::NotSupported(
           "Only stacked BlobDB's internal compaction filter can return "
           "kChangeBlobIndex.");
-      valid_ = false;
+      validity_info_.Invalidate();
       return false;
     }
     if (ikey_.type == kTypeValue) {
@@ -352,7 +349,7 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
     if (!compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) {
       status_ = Status::NotSupported(
           "CompactionFilter for integrated BlobDB should not return kIOError");
-      valid_ = false;
+      validity_info_.Invalidate();
       return false;
     }
     status_ = Status::IOError("Failed to access blob during compaction filter");
@@ -363,12 +360,13 @@ bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
 
 void CompactionIterator::NextFromInput() {
   at_next_ = false;
-  valid_ = false;
+  validity_info_.Invalidate();
 
-  while (!valid_ && input_.Valid() && !IsPausingManualCompaction() &&
+  while (!Valid() && input_.Valid() && !IsPausingManualCompaction() &&
          !IsShuttingDown()) {
     key_ = input_.key();
     value_ = input_.value();
+    blob_value_.Reset();
     iter_stats_.num_input_records++;
 
     Status pik_status = ParseInternalKey(key_, &ikey_, allow_data_in_errors_);
@@ -385,7 +383,7 @@ void CompactionIterator::NextFromInput() {
       has_current_user_key_ = false;
       current_user_key_sequence_ = kMaxSequenceNumber;
       current_user_key_snapshot_ = 0;
-      valid_ = true;
+      validity_info_.SetValid(ValidContext::kParseKeyError);
       break;
     }
     TEST_SYNC_POINT_CALLBACK("CompactionIterator:ProcessKV", &ikey_);
@@ -498,7 +496,7 @@ void CompactionIterator::NextFromInput() {
 
     if (UNLIKELY(!current_key_committed_)) {
       assert(snapshot_checker_ != nullptr);
-      valid_ = true;
+      validity_info_.SetValid(ValidContext::kCurrentKeyUncommitted);
       break;
     }
 
@@ -541,7 +539,7 @@ void CompactionIterator::NextFromInput() {
       }
 
       value_.clear();
-      valid_ = true;
+      validity_info_.SetValid(ValidContext::kKeepSDAndClearPut);
       clear_and_output_next_key_ = false;
     } else if (ikey_.type == kTypeSingleDeletion) {
       // We can compact out a SingleDelete if:
@@ -665,7 +663,7 @@ void CompactionIterator::NextFromInput() {
             ++iter_stats_.num_single_del_mismatch;
             if (enforce_single_del_contracts_) {
               ROCKS_LOG_ERROR(info_log_, "%s", oss.str().c_str());
-              valid_ = false;
+              validity_info_.Invalidate();
               status_ = Status::Corruption(oss.str());
               return;
             }
@@ -674,7 +672,7 @@ void CompactionIterator::NextFromInput() {
             // We cannot drop the SingleDelete as timestamp is enabled, and
             // timestamp of this key is greater than or equal to
             // *full_history_ts_low_. We will output the SingleDelete.
-            valid_ = true;
+            validity_info_.SetValid(ValidContext::kKeepTsHistory);
           } else if (has_outputted_key_ ||
                      DefinitelyInSnapshot(ikey_.sequence,
                                           earliest_write_conflict_snapshot_) ||
@@ -709,7 +707,7 @@ void CompactionIterator::NextFromInput() {
             // outputted on the next iteration.)
 
             // Setting valid_ to true will output the current SingleDelete
-            valid_ = true;
+            validity_info_.SetValid(ValidContext::kKeepSDForConflictCheck);
 
             // Set up the Put to be outputted in the next iteration.
             // (Optimization 3).
@@ -721,7 +719,7 @@ void CompactionIterator::NextFromInput() {
         } else {
           // We hit the next snapshot without hitting a put, so the iterator
           // returns the single delete.
-          valid_ = true;
+          validity_info_.SetValid(ValidContext::kKeepSDForSnapshot);
           TEST_SYNC_POINT_CALLBACK(
               "CompactionIterator::NextFromInput:SingleDelete:3",
               const_cast<Compaction*>(c));
@@ -754,11 +752,11 @@ void CompactionIterator::NextFromInput() {
           assert(bottommost_level_);
         } else {
           // Output SingleDelete
-          valid_ = true;
+          validity_info_.SetValid(ValidContext::kKeepSD);
         }
       }
 
-      if (valid_) {
+      if (Valid()) {
         at_next_ = true;
       }
     } else if (last_snapshot == current_user_key_snapshot_ ||
@@ -857,7 +855,7 @@ void CompactionIterator::NextFromInput() {
           (ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_)
                .ok()) &&
           cmp_->EqualWithoutTimestamp(ikey_.user_key, next_ikey.user_key)) {
-        valid_ = true;
+        validity_info_.SetValid(ValidContext::kKeepDel);
         at_next_ = true;
       }
     } else if (ikey_.type == kTypeMerge) {
@@ -901,7 +899,7 @@ void CompactionIterator::NextFromInput() {
         current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
         key_ = current_key_.GetInternalKey();
         ikey_.user_key = current_key_.GetUserKey();
-        valid_ = true;
+        validity_info_.SetValid(ValidContext::kMerge2);
       } else {
         // all merge operands were filtered out. reset the user key, since the
         // batch consumed by the merge operator should not shadow any keys
@@ -923,7 +921,7 @@ void CompactionIterator::NextFromInput() {
         ++iter_stats_.num_record_drop_range_del;
         AdvanceInputIter();
       } else {
-        valid_ = true;
+        validity_info_.SetValid(ValidContext::kNewUserKey);
       }
     }
 
@@ -932,13 +930,18 @@ void CompactionIterator::NextFromInput() {
     }
   }
 
-  if (!valid_ && IsShuttingDown()) {
+  if (!Valid() && IsShuttingDown()) {
     status_ = Status::ShutdownInProgress();
   }
 
   if (IsPausingManualCompaction()) {
     status_ = Status::Incomplete(Status::SubCode::kManualCompactionPaused);
   }
+
+  // Propagate corruption status from memtable itereator
+  if (!input_.Valid() && input_.status().IsCorruption()) {
+    status_ = input_.status();
+  }
 }
 
 bool CompactionIterator::ExtractLargeValueIfNeededImpl() {
@@ -951,7 +954,7 @@ bool CompactionIterator::ExtractLargeValueIfNeededImpl() {
 
   if (!s.ok()) {
     status_ = s;
-    valid_ = false;
+    validity_info_.Invalidate();
 
     return false;
   }
@@ -996,7 +999,7 @@ void CompactionIterator::GarbageCollectBlobIfNeeded() {
 
       if (!s.ok()) {
         status_ = s;
-        valid_ = false;
+        validity_info_.Invalidate();
 
         return;
       }
@@ -1022,7 +1025,7 @@ void CompactionIterator::GarbageCollectBlobIfNeeded() {
 
       if (!s.ok()) {
         status_ = s;
-        valid_ = false;
+        validity_info_.Invalidate();
 
         return;
       }
@@ -1055,14 +1058,14 @@ void CompactionIterator::GarbageCollectBlobIfNeeded() {
     if (blob_decision == CompactionFilter::BlobDecision::kCorruption) {
       status_ =
           Status::Corruption("Corrupted blob reference encountered during GC");
-      valid_ = false;
+      validity_info_.Invalidate();
 
       return;
     }
 
     if (blob_decision == CompactionFilter::BlobDecision::kIOError) {
       status_ = Status::IOError("Could not relocate blob during GC");
-      valid_ = false;
+      validity_info_.Invalidate();
 
       return;
     }
@@ -1077,8 +1080,7 @@ void CompactionIterator::GarbageCollectBlobIfNeeded() {
 
 void CompactionIterator::DecideOutputLevel() {
 #ifndef NDEBUG
-  // TODO: will be set by sequence number or key range, for now, it will only be
-  // set by unittest
+  // Could be overridden by unittest
   PerKeyPlacementContext context(level_, ikey_.user_key, value_,
                                  ikey_.sequence);
   TEST_SYNC_POINT_CALLBACK("CompactionIterator::PrepareOutput.context",
@@ -1086,9 +1088,10 @@ void CompactionIterator::DecideOutputLevel() {
   output_to_penultimate_level_ = context.output_to_penultimate_level;
 #endif /* !NDEBUG */
 
-  // if the key is within the earliest snapshot, it has to output to the
-  // penultimate level.
-  if (ikey_.sequence > earliest_snapshot_) {
+  // if the key is newer than the cutoff sequence or within the earliest
+  // snapshot, it should output to the penultimate level.
+  if (ikey_.sequence > penultimate_level_cutoff_seqno_ ||
+      ikey_.sequence > earliest_snapshot_) {
     output_to_penultimate_level_ = true;
   }
 
@@ -1103,13 +1106,13 @@ void CompactionIterator::DecideOutputLevel() {
         compaction_->WithinPenultimateLevelOutputRange(ikey_.user_key);
     if (!safe_to_penultimate_level) {
       output_to_penultimate_level_ = false;
-      // It could happen when disable/enable `bottommost_temperature` while
-      // holding a snapshot. When `bottommost_temperature` is not set
+      // It could happen when disable/enable `last_level_temperature` while
+      // holding a snapshot. When `last_level_temperature` is not set
       // (==kUnknown), the data newer than any snapshot is pushed to the last
       // level, but when the per_key_placement feature is enabled on the fly,
       // the data later than the snapshot has to be moved to the penultimate
       // level, which may or may not be safe. So the user needs to make sure all
-      // snapshot is released before enabling `bottommost_temperature` feature
+      // snapshot is released before enabling `last_level_temperature` feature
       // We will migrate the feature to `last_level_temperature` and maybe make
       // it not dynamically changeable.
       if (ikey_.sequence > earliest_snapshot_) {
@@ -1122,7 +1125,7 @@ void CompactionIterator::DecideOutputLevel() {
 }
 
 void CompactionIterator::PrepareOutput() {
-  if (valid_) {
+  if (Valid()) {
     if (ikey_.type == kTypeValue) {
       ExtractLargeValueIfNeeded();
     } else if (ikey_.type == kTypeBlobIndex) {
@@ -1144,11 +1147,12 @@ void CompactionIterator::PrepareOutput() {
     //
     // Can we do the same for levels above bottom level as long as
     // KeyNotExistsBeyondOutputLevel() return true?
-    if (valid_ && compaction_ != nullptr &&
+    if (Valid() && compaction_ != nullptr &&
         !compaction_->allow_ingest_behind() && bottommost_level_ &&
         DefinitelyInSnapshot(ikey_.sequence, earliest_snapshot_) &&
         ikey_.type != kTypeMerge && current_key_committed_ &&
-        !output_to_penultimate_level_) {
+        !output_to_penultimate_level_ &&
+        ikey_.sequence < penultimate_level_cutoff_seqno_) {
       if (ikey_.type == kTypeDeletion ||
           (ikey_.type == kTypeSingleDeletion && timestamp_size_ == 0)) {
         ROCKS_LOG_FATAL(
@@ -1157,13 +1161,14 @@ void CompactionIterator::PrepareOutput() {
             "earliest_snapshot %" PRIu64
             ", earliest_write_conflict_snapshot %" PRIu64
             " job_snapshot %" PRIu64
-            ". timestamp_size: %d full_history_ts_low_ %s",
+            ". timestamp_size: %d full_history_ts_low_ %s. validity %x",
             ikey_.DebugString(allow_data_in_errors_, true).c_str(),
             earliest_snapshot_, earliest_write_conflict_snapshot_,
             job_snapshot_, static_cast<int>(timestamp_size_),
             full_history_ts_low_ != nullptr
                 ? Slice(*full_history_ts_low_).ToString(true).c_str()
-                : "null");
+                : "null",
+            validity_info_.rep);
         assert(false);
       }
       ikey_.sequence = 0;
@@ -1279,7 +1284,10 @@ std::unique_ptr<BlobFetcher> CompactionIterator::CreateBlobFetcherIfNeeded(
     return nullptr;
   }
 
-  return std::unique_ptr<BlobFetcher>(new BlobFetcher(version, ReadOptions()));
+  ReadOptions read_options;
+  read_options.fill_cache = false;
+
+  return std::unique_ptr<BlobFetcher>(new BlobFetcher(version, read_options));
 }
 
 std::unique_ptr<PrefetchBufferCollection>

db/compaction/compaction_iterator.h

@@ -181,42 +181,40 @@ class CompactionIterator {
     const Compaction* compaction_;
   };
 
-  CompactionIterator(InternalIterator* input, const Comparator* cmp,
-                     MergeHelper* merge_helper, SequenceNumber last_sequence,
-                     std::vector<SequenceNumber>* snapshots,
-                     SequenceNumber earliest_write_conflict_snapshot,
-                     SequenceNumber job_snapshot,
-                     const SnapshotChecker* snapshot_checker, Env* env,
-                     bool report_detailed_time, bool expect_valid_internal_key,
-                     CompactionRangeDelAggregator* range_del_agg,
-                     BlobFileBuilder* blob_file_builder,
-                     bool allow_data_in_errors,
-                     bool enforce_single_del_contracts,
-                     const std::atomic<bool>& manual_compaction_canceled,
-                     const Compaction* compaction = nullptr,
-                     const CompactionFilter* compaction_filter = nullptr,
-                     const std::atomic<bool>* shutting_down = nullptr,
-                     const std::shared_ptr<Logger> info_log = nullptr,
-                     const std::string* full_history_ts_low = nullptr);
+  CompactionIterator(
+      InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
+      SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+      SequenceNumber earliest_write_conflict_snapshot,
+      SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker,
+      Env* env, bool report_detailed_time, bool expect_valid_internal_key,
+      CompactionRangeDelAggregator* range_del_agg,
+      BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
+      bool enforce_single_del_contracts,
+      const std::atomic<bool>& manual_compaction_canceled,
+      const Compaction* compaction = nullptr,
+      const CompactionFilter* compaction_filter = nullptr,
+      const std::atomic<bool>* shutting_down = nullptr,
+      const std::shared_ptr<Logger> info_log = nullptr,
+      const std::string* full_history_ts_low = nullptr,
+      const SequenceNumber penultimate_level_cutoff_seqno = kMaxSequenceNumber);
 
   // Constructor with custom CompactionProxy, used for tests.
-  CompactionIterator(InternalIterator* input, const Comparator* cmp,
-                     MergeHelper* merge_helper, SequenceNumber last_sequence,
-                     std::vector<SequenceNumber>* snapshots,
-                     SequenceNumber earliest_write_conflict_snapshot,
-                     SequenceNumber job_snapshot,
-                     const SnapshotChecker* snapshot_checker, Env* env,
-                     bool report_detailed_time, bool expect_valid_internal_key,
-                     CompactionRangeDelAggregator* range_del_agg,
-                     BlobFileBuilder* blob_file_builder,
-                     bool allow_data_in_errors,
-                     bool enforce_single_del_contracts,
-                     const std::atomic<bool>& manual_compaction_canceled,
-                     std::unique_ptr<CompactionProxy> compaction,
-                     const CompactionFilter* compaction_filter = nullptr,
-                     const std::atomic<bool>* shutting_down = nullptr,
-                     const std::shared_ptr<Logger> info_log = nullptr,
-                     const std::string* full_history_ts_low = nullptr);
+  CompactionIterator(
+      InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
+      SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+      SequenceNumber earliest_write_conflict_snapshot,
+      SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker,
+      Env* env, bool report_detailed_time, bool expect_valid_internal_key,
+      CompactionRangeDelAggregator* range_del_agg,
+      BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
+      bool enforce_single_del_contracts,
+      const std::atomic<bool>& manual_compaction_canceled,
+      std::unique_ptr<CompactionProxy> compaction,
+      const CompactionFilter* compaction_filter = nullptr,
+      const std::atomic<bool>* shutting_down = nullptr,
+      const std::shared_ptr<Logger> info_log = nullptr,
+      const std::string* full_history_ts_low = nullptr,
+      const SequenceNumber penultimate_level_cutoff_seqno = kMaxSequenceNumber);
 
   ~CompactionIterator();
 
@@ -237,7 +235,7 @@ class CompactionIterator {
   const Slice& value() const { return value_; }
   const Status& status() const { return status_; }
   const ParsedInternalKey& ikey() const { return ikey_; }
-  bool Valid() const { return valid_; }
+  inline bool Valid() const { return validity_info_.IsValid(); }
   const Slice& user_key() const { return current_user_key_; }
   const CompactionIterationStats& iter_stats() const { return iter_stats_; }
   uint64_t num_input_entry_scanned() const { return input_.num_itered(); }
@@ -334,29 +332,26 @@ class CompactionIterator {
   // earliest visible snapshot of an older value.
   // See WritePreparedTransactionTest::ReleaseSnapshotDuringCompaction3.
   std::unordered_set<SequenceNumber> released_snapshots_;
-  std::vector<SequenceNumber>::const_iterator earliest_snapshot_iter_;
   const SequenceNumber earliest_write_conflict_snapshot_;
   const SequenceNumber job_snapshot_;
   const SnapshotChecker* const snapshot_checker_;
   Env* env_;
   SystemClock* clock_;
-  bool report_detailed_time_;
-  bool expect_valid_internal_key_;
+  const bool report_detailed_time_;
+  const bool expect_valid_internal_key_;
   CompactionRangeDelAggregator* range_del_agg_;
   BlobFileBuilder* blob_file_builder_;
   std::unique_ptr<CompactionProxy> compaction_;
   const CompactionFilter* compaction_filter_;
   const std::atomic<bool>* shutting_down_;
   const std::atomic<bool>& manual_compaction_canceled_;
-  bool bottommost_level_;
-  bool valid_ = false;
-  bool visible_at_tip_;
-  SequenceNumber earliest_snapshot_;
-  SequenceNumber latest_snapshot_;
+  const bool bottommost_level_;
+  const bool visible_at_tip_;
+  const SequenceNumber earliest_snapshot_;
 
   std::shared_ptr<Logger> info_log_;
 
-  bool allow_data_in_errors_;
+  const bool allow_data_in_errors_;
 
   const bool enforce_single_del_contracts_;
 
@@ -372,8 +367,33 @@ class CompactionIterator {
 
   // State
   //
+  enum ValidContext : uint8_t {
+    kMerge1 = 0,
+    kMerge2 = 1,
+    kParseKeyError = 2,
+    kCurrentKeyUncommitted = 3,
+    kKeepSDAndClearPut = 4,
+    kKeepTsHistory = 5,
+    kKeepSDForConflictCheck = 6,
+    kKeepSDForSnapshot = 7,
+    kKeepSD = 8,
+    kKeepDel = 9,
+    kNewUserKey = 10,
+  };
+
+  struct ValidityInfo {
+    inline bool IsValid() const { return rep & 1; }
+    ValidContext GetContext() const {
+      return static_cast<ValidContext>(rep >> 1);
+    }
+    inline void SetValid(uint8_t ctx) { rep = (ctx << 1) | 1; }
+    inline void Invalidate() { rep = 0; }
+
+    uint8_t rep{0};
+  } validity_info_;
+
   // Points to a copy of the current compaction iterator output (current_key_)
-  // if valid_.
+  // if valid.
   Slice key_;
   // Points to the value in the underlying iterator that corresponds to the
   // current output.
@@ -446,6 +466,10 @@ class CompactionIterator {
   // output to.
   bool output_to_penultimate_level_{false};
 
+  // any key later than this sequence number should have
+  // output_to_penultimate_level_ set to true
+  const SequenceNumber penultimate_level_cutoff_seqno_ = kMaxSequenceNumber;
+
   void AdvanceInputIter() { input_.Next(); }
 
   void SkipUntil(const Slice& skip_until) { input_.Seek(skip_until); }

db/compaction/compaction_job.cc

@@ -12,6 +12,7 @@
 #include <algorithm>
 #include <cinttypes>
 #include <memory>
+#include <optional>
 #include <set>
 #include <utility>
 #include <vector>
@@ -30,6 +31,7 @@
 #include "db/log_writer.h"
 #include "db/merge_helper.h"
 #include "db/range_del_aggregator.h"
+#include "db/version_edit.h"
 #include "db/version_set.h"
 #include "file/filename.h"
 #include "file/read_write_util.h"
@@ -44,6 +46,7 @@
 #include "port/port.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
+#include "rocksdb/options.h"
 #include "rocksdb/statistics.h"
 #include "rocksdb/status.h"
 #include "rocksdb/table.h"
@@ -120,7 +123,8 @@ CompactionJob::CompactionJob(
     const std::atomic<bool>& manual_compaction_canceled,
     const std::string& db_id, const std::string& db_session_id,
     std::string full_history_ts_low, std::string trim_ts,
-    BlobFileCompletionCallback* blob_callback)
+    BlobFileCompletionCallback* blob_callback, int* bg_compaction_scheduled,
+    int* bg_bottom_compaction_scheduled)
     : compact_(new CompactionState(compaction)),
       compaction_stats_(compaction->compaction_reason(), 1),
       db_options_(db_options),
@@ -159,9 +163,13 @@ CompactionJob::CompactionJob(
       thread_pri_(thread_pri),
       full_history_ts_low_(std::move(full_history_ts_low)),
       trim_ts_(std::move(trim_ts)),
-      blob_callback_(blob_callback) {
+      blob_callback_(blob_callback),
+      extra_num_subcompaction_threads_reserved_(0),
+      bg_compaction_scheduled_(bg_compaction_scheduled),
+      bg_bottom_compaction_scheduled_(bg_bottom_compaction_scheduled) {
   assert(compaction_job_stats_ != nullptr);
   assert(log_buffer_ != nullptr);
+
   const auto* cfd = compact_->compaction->column_family_data();
   ThreadStatusUtil::SetColumnFamily(cfd, cfd->ioptions()->env,
                                     db_options_.enable_thread_tracking);
@@ -223,34 +231,167 @@ void CompactionJob::Prepare() {
 
   // Generate file_levels_ for compaction before making Iterator
   auto* c = compact_->compaction;
-  assert(c->column_family_data() != nullptr);
-  assert(c->column_family_data()->current()->storage_info()->NumLevelFiles(
+  ColumnFamilyData* cfd = c->column_family_data();
+  assert(cfd != nullptr);
+  assert(cfd->current()->storage_info()->NumLevelFiles(
              compact_->compaction->level()) > 0);
 
-  write_hint_ =
-      c->column_family_data()->CalculateSSTWriteHint(c->output_level());
+  write_hint_ = cfd->CalculateSSTWriteHint(c->output_level());
   bottommost_level_ = c->bottommost_level();
 
   if (c->ShouldFormSubcompactions()) {
-    {
-      StopWatch sw(db_options_.clock, stats_, SUBCOMPACTION_SETUP_TIME);
-      GenSubcompactionBoundaries();
-    }
-
+    StopWatch sw(db_options_.clock, stats_, SUBCOMPACTION_SETUP_TIME);
+    GenSubcompactionBoundaries();
+  }
+  if (boundaries_.size() > 1) {
     for (size_t i = 0; i <= boundaries_.size(); i++) {
-      Slice* start = i == 0 ? nullptr : &boundaries_[i - 1];
-      Slice* end = i == boundaries_.size() ? nullptr : &boundaries_[i];
-      compact_->sub_compact_states.emplace_back(c, start, end,
-                                                static_cast<uint32_t>(i));
+      compact_->sub_compact_states.emplace_back(
+          c, (i != 0) ? std::optional<Slice>(boundaries_[i - 1]) : std::nullopt,
+          (i != boundaries_.size()) ? std::optional<Slice>(boundaries_[i])
+                                    : std::nullopt,
+          static_cast<uint32_t>(i));
+      // assert to validate that boundaries don't have same user keys (without
+      // timestamp part).
+      assert(i == 0 || i == boundaries_.size() ||
+             cfd->user_comparator()->CompareWithoutTimestamp(
+                 boundaries_[i - 1], true, boundaries_[i], true) < 0);
     }
     RecordInHistogram(stats_, NUM_SUBCOMPACTIONS_SCHEDULED,
                       compact_->sub_compact_states.size());
   } else {
-    constexpr Slice* start = nullptr;
-    constexpr Slice* end = nullptr;
-
-    compact_->sub_compact_states.emplace_back(c, start, end, /*sub_job_id*/ 0);
+    compact_->sub_compact_states.emplace_back(c, std::nullopt, std::nullopt,
+                                              /*sub_job_id*/ 0);
   }
+
+  if (c->immutable_options()->preclude_last_level_data_seconds > 0) {
+    // TODO(zjay): move to a function
+    seqno_time_mapping_.SetMaxTimeDuration(
+        c->immutable_options()->preclude_last_level_data_seconds);
+    // setup seqno_time_mapping_
+    for (const auto& each_level : *c->inputs()) {
+      for (const auto& fmd : each_level.files) {
+        std::shared_ptr<const TableProperties> tp;
+        Status s = cfd->current()->GetTableProperties(&tp, fmd, nullptr);
+        if (s.ok()) {
+          seqno_time_mapping_.Add(tp->seqno_to_time_mapping)
+              .PermitUncheckedError();
+          seqno_time_mapping_.Add(fmd->fd.smallest_seqno,
+                                  fmd->oldest_ancester_time);
+        }
+      }
+    }
+
+    auto status = seqno_time_mapping_.Sort();
+    if (!status.ok()) {
+      ROCKS_LOG_WARN(db_options_.info_log,
+                     "Invalid sequence number to time mapping: Status: %s",
+                     status.ToString().c_str());
+    }
+    int64_t _current_time = 0;
+    status = db_options_.clock->GetCurrentTime(&_current_time);
+    if (!status.ok()) {
+      ROCKS_LOG_WARN(db_options_.info_log,
+                     "Failed to get current time in compaction: Status: %s",
+                     status.ToString().c_str());
+      penultimate_level_cutoff_seqno_ = 0;
+    } else {
+      penultimate_level_cutoff_seqno_ =
+          seqno_time_mapping_.TruncateOldEntries(_current_time);
+    }
+  }
+}
+
+uint64_t CompactionJob::GetSubcompactionsLimit() {
+  return extra_num_subcompaction_threads_reserved_ +
+         std::max(
+             std::uint64_t(1),
+             static_cast<uint64_t>(compact_->compaction->max_subcompactions()));
+}
+
+void CompactionJob::AcquireSubcompactionResources(
+    int num_extra_required_subcompactions) {
+  TEST_SYNC_POINT("CompactionJob::AcquireSubcompactionResources:0");
+  TEST_SYNC_POINT("CompactionJob::AcquireSubcompactionResources:1");
+  int max_db_compactions =
+      DBImpl::GetBGJobLimits(
+          mutable_db_options_copy_.max_background_flushes,
+          mutable_db_options_copy_.max_background_compactions,
+          mutable_db_options_copy_.max_background_jobs,
+          versions_->GetColumnFamilySet()
+              ->write_controller()
+              ->NeedSpeedupCompaction())
+          .max_compactions;
+  // Apply min function first since We need to compute the extra subcompaction
+  // against compaction limits. And then try to reserve threads for extra
+  // subcompactions. The actual number of reserved threads could be less than
+  // the desired number.
+  int available_bg_compactions_against_db_limit =
+      std::max(max_db_compactions - *bg_compaction_scheduled_ -
+                   *bg_bottom_compaction_scheduled_,
+               0);
+  db_mutex_->Lock();
+  // Reservation only supports backgrdoun threads of which the priority is
+  // between BOTTOM and HIGH. Need to degrade the priority to HIGH if the
+  // origin thread_pri_ is higher than that. Similar to ReleaseThreads().
+  extra_num_subcompaction_threads_reserved_ =
+      env_->ReserveThreads(std::min(num_extra_required_subcompactions,
+                                    available_bg_compactions_against_db_limit),
+                           std::min(thread_pri_, Env::Priority::HIGH));
+
+  // Update bg_compaction_scheduled_ or bg_bottom_compaction_scheduled_
+  // depending on if this compaction has the bottommost priority
+  if (thread_pri_ == Env::Priority::BOTTOM) {
+    *bg_bottom_compaction_scheduled_ +=
+        extra_num_subcompaction_threads_reserved_;
+  } else {
+    *bg_compaction_scheduled_ += extra_num_subcompaction_threads_reserved_;
+  }
+  db_mutex_->Unlock();
+}
+
+void CompactionJob::ShrinkSubcompactionResources(uint64_t num_extra_resources) {
+  // Do nothing when we have zero resources to shrink
+  if (num_extra_resources == 0) return;
+  db_mutex_->Lock();
+  // We cannot release threads more than what we reserved before
+  int extra_num_subcompaction_threads_released = env_->ReleaseThreads(
+      (int)num_extra_resources, std::min(thread_pri_, Env::Priority::HIGH));
+  // Update the number of reserved threads and the number of background
+  // scheduled compactions for this compaction job
+  extra_num_subcompaction_threads_reserved_ -=
+      extra_num_subcompaction_threads_released;
+  // TODO (zichen): design a test case with new subcompaction partitioning
+  // when the number of actual partitions is less than the number of planned
+  // partitions
+  assert(extra_num_subcompaction_threads_released == (int)num_extra_resources);
+  // Update bg_compaction_scheduled_ or bg_bottom_compaction_scheduled_
+  // depending on if this compaction has the bottommost priority
+  if (thread_pri_ == Env::Priority::BOTTOM) {
+    *bg_bottom_compaction_scheduled_ -=
+        extra_num_subcompaction_threads_released;
+  } else {
+    *bg_compaction_scheduled_ -= extra_num_subcompaction_threads_released;
+  }
+  db_mutex_->Unlock();
+  TEST_SYNC_POINT("CompactionJob::ShrinkSubcompactionResources:0");
+}
+
+void CompactionJob::ReleaseSubcompactionResources() {
+  if (extra_num_subcompaction_threads_reserved_ == 0) {
+    return;
+  }
+  // The number of reserved threads becomes larger than 0 only if the
+  // compaction prioity is round robin and there is no sufficient
+  // sub-compactions available
+
+  // The scheduled compaction must be no less than 1 + extra number
+  // subcompactions using acquired resources since this compaction job has not
+  // finished yet
+  assert(*bg_bottom_compaction_scheduled_ >=
+             1 + extra_num_subcompaction_threads_reserved_ ||
+         *bg_compaction_scheduled_ >=
+             1 + extra_num_subcompaction_threads_reserved_);
+  ShrinkSubcompactionResources(extra_num_subcompaction_threads_reserved_);
 }
 
 struct RangeWithSize {
@@ -262,15 +403,51 @@ struct RangeWithSize {
 };
 
 void CompactionJob::GenSubcompactionBoundaries() {
+  // The goal is to find some boundary keys so that we can evenly partition
+  // the compaction input data into max_subcompactions ranges.
+  // For every input file, we ask TableReader to estimate 128 anchor points
+  // that evenly partition the input file into 128 ranges and the range
+  // sizes. This can be calculated by scanning index blocks of the file.
+  // Once we have the anchor points for all the input files, we merge them
+  // together and try to find keys dividing ranges evenly.
+  // For example, if we have two input files, and each returns following
+  // ranges:
+  //   File1: (a1, 1000), (b1, 1200), (c1, 1100)
+  //   File2: (a2, 1100), (b2, 1000), (c2, 1000)
+  // We total sort the keys to following:
+  //  (a1, 1000), (a2, 1100), (b1, 1200), (b2, 1000), (c1, 1100), (c2, 1000)
+  // We calculate the total size by adding up all ranges' size, which is 6400.
+  // If we would like to partition into 2 subcompactions, the target of the
+  // range size is 3200. Based on the size, we take "b1" as the partition key
+  // since the first three ranges would hit 3200.
+  //
+  // Note that the ranges are actually overlapping. For example, in the example
+  // above, the range ending with "b1" is overlapping with the range ending with
+  // "b2". So the size 1000+1100+1200 is an underestimation of data size up to
+  // "b1". In extreme cases where we only compact N L0 files, a range can
+  // overlap with N-1 other ranges. Since we requested a relatively large number
+  // (128) of ranges from each input files, even N range overlapping would
+  // cause relatively small inaccuracy.
+
   auto* c = compact_->compaction;
+  if (c->max_subcompactions() <= 1 &&
+      !(c->immutable_options()->compaction_pri == kRoundRobin &&
+        c->immutable_options()->compaction_style == kCompactionStyleLevel)) {
+    return;
+  }
   auto* cfd = c->column_family_data();
   const Comparator* cfd_comparator = cfd->user_comparator();
-  std::vector<Slice> bounds;
+  const InternalKeyComparator& icomp = cfd->internal_comparator();
+
+  auto* v = compact_->compaction->input_version();
+  int base_level = v->storage_info()->base_level();
+  InstrumentedMutexUnlock unlock_guard(db_mutex_);
+
+  uint64_t total_size = 0;
+  std::vector<TableReader::Anchor> all_anchors;
   int start_lvl = c->start_level();
   int out_lvl = c->output_level();
 
-  // Add the starting and/or ending key of certain input files as a potential
-  // boundary
   for (size_t lvl_idx = 0; lvl_idx < c->num_input_levels(); lvl_idx++) {
     int lvl = c->level(lvl_idx);
     if (lvl >= start_lvl && lvl <= out_lvl) {
@@ -281,108 +458,115 @@ void CompactionJob::GenSubcompactionBoundaries() {
         continue;
       }
 
-      if (lvl == 0) {
-        // For level 0 add the starting and ending key of each file since the
-        // files may have greatly differing key ranges (not range-partitioned)
-        for (size_t i = 0; i < num_files; i++) {
-          bounds.emplace_back(flevel->files[i].smallest_key);
-          bounds.emplace_back(flevel->files[i].largest_key);
+      for (size_t i = 0; i < num_files; i++) {
+        FileMetaData* f = flevel->files[i].file_metadata;
+        std::vector<TableReader::Anchor> my_anchors;
+        Status s = cfd->table_cache()->ApproximateKeyAnchors(
+            ReadOptions(), icomp, f->fd, my_anchors);
+        if (!s.ok() || my_anchors.empty()) {
+          my_anchors.emplace_back(f->largest.user_key(), f->fd.GetFileSize());
         }
-      } else {
-        // For all other levels add the smallest/largest key in the level to
-        // encompass the range covered by that level
-        bounds.emplace_back(flevel->files[0].smallest_key);
-        bounds.emplace_back(flevel->files[num_files - 1].largest_key);
-        if (lvl == out_lvl) {
-          // For the last level include the starting keys of all files since
-          // the last level is the largest and probably has the widest key
-          // range. Since it's range partitioned, the ending key of one file
-          // and the starting key of the next are very close (or identical).
-          for (size_t i = 1; i < num_files; i++) {
-            bounds.emplace_back(flevel->files[i].smallest_key);
-          }
+        for (auto& ac : my_anchors) {
+          // Can be optimize to avoid this loop.
+          total_size += ac.range_size;
         }
+
+        all_anchors.insert(all_anchors.end(), my_anchors.begin(),
+                           my_anchors.end());
       }
     }
   }
+  // Here we total sort all the anchor points across all files and go through
+  // them in the sorted order to find partitioning boundaries.
+  // Not the most efficient implementation. A much more efficient algorithm
+  // probably exists. But they are more complex. If performance turns out to
+  // be a problem, we can optimize.
+  std::sort(
+      all_anchors.begin(), all_anchors.end(),
+      [cfd_comparator](TableReader::Anchor& a, TableReader::Anchor& b) -> bool {
+        return cfd_comparator->CompareWithoutTimestamp(a.user_key, true,
+                                                       b.user_key, true) < 0;
+      });
 
-  std::sort(bounds.begin(), bounds.end(),
-            [cfd_comparator](const Slice& a, const Slice& b) -> bool {
-              return cfd_comparator->Compare(ExtractUserKey(a),
-                                             ExtractUserKey(b)) < 0;
-            });
-  // Remove duplicated entries from bounds
-  bounds.erase(
-      std::unique(bounds.begin(), bounds.end(),
-                  [cfd_comparator](const Slice& a, const Slice& b) -> bool {
-                    return cfd_comparator->Compare(ExtractUserKey(a),
-                                                   ExtractUserKey(b)) == 0;
+  // Remove duplicated entries from boundaries.
+  all_anchors.erase(
+      std::unique(all_anchors.begin(), all_anchors.end(),
+                  [cfd_comparator](TableReader::Anchor& a,
+                                   TableReader::Anchor& b) -> bool {
+                    return cfd_comparator->CompareWithoutTimestamp(
+                               a.user_key, true, b.user_key, true) == 0;
                   }),
-      bounds.end());
+      all_anchors.end());
 
-  // Combine consecutive pairs of boundaries into ranges with an approximate
-  // size of data covered by keys in that range
-  uint64_t sum = 0;
-  std::vector<RangeWithSize> ranges;
-  // Get input version from CompactionState since it's already referenced
-  // earlier in SetInputVersioCompaction::SetInputVersion and will not change
-  // when db_mutex_ is released below
-  auto* v = compact_->compaction->input_version();
-  for (auto it = bounds.begin();;) {
-    const Slice a = *it;
-    ++it;
+  // Get the number of planned subcompactions, may update reserve threads
+  // and update extra_num_subcompaction_threads_reserved_ for round-robin
+  uint64_t num_planned_subcompactions;
+  if (c->immutable_options()->compaction_pri == kRoundRobin &&
+      c->immutable_options()->compaction_style == kCompactionStyleLevel) {
+    // For round-robin compaction prioity, we need to employ more
+    // subcompactions (may exceed the max_subcompaction limit). The extra
+    // subcompactions will be executed using reserved threads and taken into
+    // account bg_compaction_scheduled or bg_bottom_compaction_scheduled.
 
-    if (it == bounds.end()) {
-      break;
+    // Initialized by the number of input files
+    num_planned_subcompactions = static_cast<uint64_t>(c->num_input_files(0));
+    uint64_t max_subcompactions_limit = GetSubcompactionsLimit();
+    if (max_subcompactions_limit < num_planned_subcompactions) {
+      // Assert two pointers are not empty so that we can use extra
+      // subcompactions against db compaction limits
+      assert(bg_bottom_compaction_scheduled_ != nullptr);
+      assert(bg_compaction_scheduled_ != nullptr);
+      // Reserve resources when max_subcompaction is not sufficient
+      AcquireSubcompactionResources(
+          (int)(num_planned_subcompactions - max_subcompactions_limit));
+      // Subcompactions limit changes after acquiring additional resources.
+      // Need to call GetSubcompactionsLimit() again to update the number
+      // of planned subcompactions
+      num_planned_subcompactions =
+          std::min(num_planned_subcompactions, GetSubcompactionsLimit());
+    } else {
+      num_planned_subcompactions = max_subcompactions_limit;
     }
-
-    const Slice b = *it;
-
-    // ApproximateSize could potentially create table reader iterator to seek
-    // to the index block and may incur I/O cost in the process. Unlock db
-    // mutex to reduce contention
-    db_mutex_->Unlock();
-    uint64_t size = versions_->ApproximateSize(SizeApproximationOptions(), v, a,
-                                               b, start_lvl, out_lvl + 1,
-                                               TableReaderCaller::kCompaction);
-    db_mutex_->Lock();
-    ranges.emplace_back(a, b, size);
-    sum += size;
+  } else {
+    num_planned_subcompactions = GetSubcompactionsLimit();
   }
 
+  TEST_SYNC_POINT_CALLBACK("CompactionJob::GenSubcompactionBoundaries:0",
+                           &num_planned_subcompactions);
+  if (num_planned_subcompactions == 1) return;
+
   // Group the ranges into subcompactions
-  const double min_file_fill_percent = 4.0 / 5;
-  int base_level = v->storage_info()->base_level();
-  uint64_t max_output_files = static_cast<uint64_t>(std::ceil(
-      sum / min_file_fill_percent /
+  uint64_t target_range_size = std::max(
+      total_size / num_planned_subcompactions,
       MaxFileSizeForLevel(
           *(c->mutable_cf_options()), out_lvl,
           c->immutable_options()->compaction_style, base_level,
-          c->immutable_options()->level_compaction_dynamic_level_bytes)));
-  uint64_t subcompactions =
-      std::min({static_cast<uint64_t>(ranges.size()),
-                static_cast<uint64_t>(c->max_subcompactions()),
-                max_output_files});
+          c->immutable_options()->level_compaction_dynamic_level_bytes));
 
-  if (subcompactions > 1) {
-    double mean = sum * 1.0 / subcompactions;
-    // Greedily add ranges to the subcompaction until the sum of the ranges'
-    // sizes becomes >= the expected mean size of a subcompaction
-    sum = 0;
-    for (size_t i = 0; i + 1 < ranges.size(); i++) {
-      sum += ranges[i].size;
-      if (subcompactions == 1) {
-        // If there's only one left to schedule then it goes to the end so no
-        // need to put an end boundary
-        continue;
-      }
-      if (sum >= mean) {
-        boundaries_.emplace_back(ExtractUserKey(ranges[i].range.limit));
-        subcompactions--;
-        sum = 0;
-      }
+  if (target_range_size >= total_size) {
+    return;
+  }
+
+  uint64_t next_threshold = target_range_size;
+  uint64_t cumulative_size = 0;
+  uint64_t num_actual_subcompactions = 1U;
+  for (TableReader::Anchor& anchor : all_anchors) {
+    cumulative_size += anchor.range_size;
+    if (cumulative_size > next_threshold) {
+      next_threshold += target_range_size;
+      num_actual_subcompactions++;
+      boundaries_.push_back(anchor.user_key);
+    }
+    if (num_actual_subcompactions == num_planned_subcompactions) {
+      break;
     }
   }
+  TEST_SYNC_POINT_CALLBACK("CompactionJob::GenSubcompactionBoundaries:1",
+                           &num_actual_subcompactions);
+  // Shrink extra subcompactions resources when extra resrouces are acquired
+  ShrinkSubcompactionResources(
+      std::min((int)(num_planned_subcompactions - num_actual_subcompactions),
+               extra_num_subcompaction_threads_reserved_));
 }
 
 Status CompactionJob::Run() {
@@ -545,6 +729,7 @@ Status CompactionJob::Run() {
     for (auto& thread : thread_pool) {
       thread.join();
     }
+
     for (const auto& state : compact_->sub_compact_states) {
       if (!state.status.ok()) {
         status = state.status;
@@ -553,6 +738,10 @@ Status CompactionJob::Run() {
     }
   }
 
+  ReleaseSubcompactionResources();
+  TEST_SYNC_POINT("CompactionJob::ReleaseSubcompactionResources:0");
+  TEST_SYNC_POINT("CompactionJob::ReleaseSubcompactionResources:1");
+
   TablePropertiesCollection tp;
   for (const auto& state : compact_->sub_compact_states) {
     for (const auto& output : state.GetOutputs()) {
@@ -848,8 +1037,11 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
 
   // TODO: since we already use C++17, should use
   // std::optional<const Slice> instead.
-  const Slice* const start = sub_compact->start;
-  const Slice* const end = sub_compact->end;
+  const std::optional<Slice> start = sub_compact->start;
+  const std::optional<Slice> end = sub_compact->end;
+
+  std::optional<Slice> start_without_ts;
+  std::optional<Slice> end_without_ts;
 
   ReadOptions read_options;
   read_options.verify_checksums = true;
@@ -861,21 +1053,29 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   // (b) CompactionFilter::Decision::kRemoveAndSkipUntil.
   read_options.total_order_seek = true;
 
-  // Note: if we're going to support subcompactions for user-defined timestamps,
-  // the timestamp part will have to be stripped from the bounds here.
-  assert((!start && !end) || cfd->user_comparator()->timestamp_size() == 0);
-  read_options.iterate_lower_bound = start;
-  read_options.iterate_upper_bound = end;
+  // Remove the timestamps from boundaries because boundaries created in
+  // GenSubcompactionBoundaries doesn't strip away the timestamp.
+  size_t ts_sz = cfd->user_comparator()->timestamp_size();
+  if (start.has_value()) {
+    read_options.iterate_lower_bound = &start.value();
+    if (ts_sz > 0) {
+      start_without_ts = StripTimestampFromUserKey(start.value(), ts_sz);
+      read_options.iterate_lower_bound = &start_without_ts.value();
+    }
+  }
+  if (end.has_value()) {
+    read_options.iterate_upper_bound = &end.value();
+    if (ts_sz > 0) {
+      end_without_ts = StripTimestampFromUserKey(end.value(), ts_sz);
+      read_options.iterate_upper_bound = &end_without_ts.value();
+    }
+  }
 
   // Although the v2 aggregator is what the level iterator(s) know about,
   // the AddTombstones calls will be propagated down to the v1 aggregator.
   std::unique_ptr<InternalIterator> raw_input(versions_->MakeInputIterator(
       read_options, sub_compact->compaction, range_del_agg.get(),
-      file_options_for_read_,
-      (start == nullptr) ? std::optional<const Slice>{}
-                         : std::optional<const Slice>{*start},
-      (end == nullptr) ? std::optional<const Slice>{}
-                       : std::optional<const Slice>{*end}));
+      file_options_for_read_, start, end));
   InternalIterator* input = raw_input.get();
 
   IterKey start_ikey;
@@ -883,20 +1083,21 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   Slice start_slice;
   Slice end_slice;
 
-  if (start) {
-    start_ikey.SetInternalKey(*start, kMaxSequenceNumber, kValueTypeForSeek);
+  if (start.has_value()) {
+    start_ikey.SetInternalKey(start.value(), kMaxSequenceNumber,
+                              kValueTypeForSeek);
     start_slice = start_ikey.GetInternalKey();
   }
-  if (end) {
-    end_ikey.SetInternalKey(*end, kMaxSequenceNumber, kValueTypeForSeek);
+  if (end.has_value()) {
+    end_ikey.SetInternalKey(end.value(), kMaxSequenceNumber, kValueTypeForSeek);
     end_slice = end_ikey.GetInternalKey();
   }
 
   std::unique_ptr<InternalIterator> clip;
-  if (start || end) {
+  if (start.has_value() || end.has_value()) {
     clip = std::make_unique<ClippingIterator>(
-        raw_input.get(), start ? &start_slice : nullptr,
-        end ? &end_slice : nullptr, &cfd->internal_comparator());
+        raw_input.get(), start.has_value() ? &start_slice : nullptr,
+        end.has_value() ? &end_slice : nullptr, &cfd->internal_comparator());
     input = clip.get();
   }
 
@@ -962,10 +1163,10 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
           ? new BlobFileBuilder(
                 versions_, fs_.get(),
                 sub_compact->compaction->immutable_options(),
-                mutable_cf_options, &file_options_, job_id_, cfd->GetID(),
-                cfd->GetName(), Env::IOPriority::IO_LOW, write_hint_,
-                io_tracer_, blob_callback_, BlobFileCreationReason::kCompaction,
-                &blob_file_paths,
+                mutable_cf_options, &file_options_, db_id_, db_session_id_,
+                job_id_, cfd->GetID(), cfd->GetName(), Env::IOPriority::IO_LOW,
+                write_hint_, io_tracer_, blob_callback_,
+                BlobFileCreationReason::kCompaction, &blob_file_paths,
                 sub_compact->Current().GetBlobFileAdditionsPtr())
           : nullptr);
 
@@ -989,7 +1190,8 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
       blob_file_builder.get(), db_options_.allow_data_in_errors,
       db_options_.enforce_single_del_contracts, manual_compaction_canceled_,
       sub_compact->compaction, compaction_filter, shutting_down_,
-      db_options_.info_log, full_history_ts_low);
+      db_options_.info_log, full_history_ts_low,
+      penultimate_level_cutoff_seqno_);
   c_iter->SeekToFirst();
 
   // Assign range delete aggregator to the target output level, which makes sure
@@ -1023,8 +1225,8 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
     // Invariant: c_iter.status() is guaranteed to be OK if c_iter->Valid()
     // returns true.
 
-    assert(!end ||
-           cfd->user_comparator()->Compare(c_iter->user_key(), *end) < 0);
+    assert(!end.has_value() || cfd->user_comparator()->Compare(
+                                   c_iter->user_key(), end.value()) < 0);
 
     if (c_iter_stats.num_input_records % kRecordStatsEvery ==
         kRecordStatsEvery - 1) {
@@ -1242,10 +1444,12 @@ Status CompactionJob::FinishCompactionOutputFile(
     // output_to_penultimate_level compaction here, as it's only used to decide
     // if range dels could be dropped.
     if (outputs.HasRangeDel()) {
-      s = outputs.AddRangeDels(sub_compact->start, sub_compact->end,
-                               range_del_out_stats, bottommost_level_,
-                               cfd->internal_comparator(), earliest_snapshot,
-                               next_table_min_key);
+      s = outputs.AddRangeDels(
+          sub_compact->start.has_value() ? &(sub_compact->start.value())
+                                         : nullptr,
+          sub_compact->end.has_value() ? &(sub_compact->end.value()) : nullptr,
+          range_del_out_stats, bottommost_level_, cfd->internal_comparator(),
+          earliest_snapshot, next_table_min_key);
     }
     RecordDroppedKeys(range_del_out_stats, &sub_compact->compaction_job_stats);
     TEST_SYNC_POINT("CompactionJob::FinishCompactionOutputFile1");
@@ -1253,7 +1457,7 @@ Status CompactionJob::FinishCompactionOutputFile(
 
   const uint64_t current_entries = outputs.NumEntries();
 
-  s = outputs.Finish(s);
+  s = outputs.Finish(s, seqno_time_mapping_);
 
   if (s.ok()) {
     // With accurate smallest and largest key, we can get a slightly more
@@ -1457,7 +1661,8 @@ Status CompactionJob::InstallCompactionResults(
     if (start_level > 0) {
       auto vstorage = compaction->input_version()->storage_info();
       edit->AddCompactCursor(start_level,
-                             vstorage->GetNextCompactCursor(start_level));
+                             vstorage->GetNextCompactCursor(
+                                 start_level, compaction->num_input_files(0)));
     }
   }
 
@@ -1511,13 +1716,16 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
                            &syncpoint_arg);
 #endif
 
-  // Pass temperature of botommost files to FileSystem.
+  // Pass temperature of the last level files to FileSystem.
   FileOptions fo_copy = file_options_;
   Temperature temperature = sub_compact->compaction->output_temperature();
-  if (temperature == Temperature::kUnknown && bottommost_level_ &&
+  // only set for the last level compaction and also it's not output to
+  // penultimate level (when preclude_last_level feature is enabled)
+  if (temperature == Temperature::kUnknown &&
+      sub_compact->compaction->is_last_level() &&
       !sub_compact->IsCurrentPenultimateLevel()) {
     temperature =
-        sub_compact->compaction->mutable_cf_options()->bottommost_temperature;
+        sub_compact->compaction->mutable_cf_options()->last_level_temperature;
   }
   fo_copy.temperature = temperature;
 
@@ -1557,16 +1765,16 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
   }
   uint64_t current_time = static_cast<uint64_t>(temp_current_time);
   InternalKey tmp_start, tmp_end;
-  if (sub_compact->start != nullptr) {
-    tmp_start.SetMinPossibleForUserKey(*(sub_compact->start));
+  if (sub_compact->start.has_value()) {
+    tmp_start.SetMinPossibleForUserKey(sub_compact->start.value());
   }
-  if (sub_compact->end != nullptr) {
-    tmp_end.SetMinPossibleForUserKey(*(sub_compact->end));
+  if (sub_compact->end.has_value()) {
+    tmp_end.SetMinPossibleForUserKey(sub_compact->end.value());
   }
   uint64_t oldest_ancester_time =
       sub_compact->compaction->MinInputFileOldestAncesterTime(
-          (sub_compact->start != nullptr) ? &tmp_start : nullptr,
-          (sub_compact->end != nullptr) ? &tmp_end : nullptr);
+          sub_compact->start.has_value() ? &tmp_start : nullptr,
+          sub_compact->end.has_value() ? &tmp_end : nullptr);
   if (oldest_ancester_time == std::numeric_limits<uint64_t>::max()) {
     oldest_ancester_time = current_time;
   }
@@ -1617,9 +1825,8 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
       sub_compact->compaction->output_compression_opts(), cfd->GetID(),
       cfd->GetName(), sub_compact->compaction->output_level(),
       bottommost_level_, TableFileCreationReason::kCompaction,
-      oldest_ancester_time, 0 /* oldest_key_time */, current_time, db_id_,
-      db_session_id_, sub_compact->compaction->max_output_file_size(),
-      file_number);
+      0 /* oldest_key_time */, current_time, db_id_, db_session_id_,
+      sub_compact->compaction->max_output_file_size(), file_number);
 
   outputs.NewBuilder(tboptions);
 
@@ -1754,7 +1961,10 @@ void CompactionJob::LogCompaction() {
       stream.EndArray();
     }
     stream << "score" << compaction->score() << "input_data_size"
-           << compaction->CalculateTotalInputSize();
+           << compaction->CalculateTotalInputSize() << "oldest_snapshot_seqno"
+           << (existing_snapshots_.empty()
+                   ? int64_t{-1}  // Use -1 for "none"
+                   : static_cast<int64_t>(existing_snapshots_[0]));
   }
 }
 

db/compaction/compaction_job.h

@@ -27,6 +27,7 @@
 #include "db/log_writer.h"
 #include "db/memtable_list.h"
 #include "db/range_del_aggregator.h"
+#include "db/seqno_to_time_mapping.h"
 #include "db/version_edit.h"
 #include "db/write_controller.h"
 #include "db/write_thread.h"
@@ -163,7 +164,9 @@ class CompactionJob {
       const std::atomic<bool>& manual_compaction_canceled,
       const std::string& db_id = "", const std::string& db_session_id = "",
       std::string full_history_ts_low = "", std::string trim_ts = "",
-      BlobFileCompletionCallback* blob_callback = nullptr);
+      BlobFileCompletionCallback* blob_callback = nullptr,
+      int* bg_compaction_scheduled = nullptr,
+      int* bg_bottom_compaction_scheduled = nullptr);
 
   virtual ~CompactionJob();
 
@@ -224,6 +227,26 @@ class CompactionJob {
   // consecutive groups such that each group has a similar size.
   void GenSubcompactionBoundaries();
 
+  // Get the number of planned subcompactions based on max_subcompactions and
+  // extra reserved resources
+  uint64_t GetSubcompactionsLimit();
+
+  // Additional reserved threads are reserved and the number is stored in
+  // extra_num_subcompaction_threads_reserved__. For now, this happens only if
+  // the compaction priority is round-robin and max_subcompactions is not
+  // sufficient (extra resources may be needed)
+  void AcquireSubcompactionResources(int num_extra_required_subcompactions);
+
+  // Additional threads may be reserved during IncreaseSubcompactionResources()
+  // if num_actual_subcompactions is less than num_planned_subcompactions.
+  // Additional threads will be released and the bg_compaction_scheduled_ or
+  // bg_bottom_compaction_scheduled_ will be updated if they are used.
+  // DB Mutex lock is required.
+  void ShrinkSubcompactionResources(uint64_t num_extra_resources);
+
+  // Release all reserved threads and update the compaction limits.
+  void ReleaseSubcompactionResources();
+
   CompactionServiceJobStatus ProcessKeyValueCompactionWithCompactionService(
       SubcompactionState* sub_compact);
 
@@ -291,13 +314,33 @@ class CompactionJob {
   bool paranoid_file_checks_;
   bool measure_io_stats_;
   // Stores the Slices that designate the boundaries for each subcompaction
-  std::vector<Slice> boundaries_;
+  std::vector<std::string> boundaries_;
   Env::Priority thread_pri_;
   std::string full_history_ts_low_;
   std::string trim_ts_;
   BlobFileCompletionCallback* blob_callback_;
 
   uint64_t GetCompactionId(SubcompactionState* sub_compact) const;
+  // Stores the number of reserved threads in shared env_ for the number of
+  // extra subcompaction in kRoundRobin compaction priority
+  int extra_num_subcompaction_threads_reserved_;
+
+  // Stores the pointer to bg_compaction_scheduled_,
+  // bg_bottom_compaction_scheduled_ in DBImpl. Mutex is required when accessing
+  // or updating it.
+  int* bg_compaction_scheduled_;
+  int* bg_bottom_compaction_scheduled_;
+
+  // Stores the sequence number to time mapping gathered from all input files
+  // it also collects the smallest_seqno -> oldest_ancester_time from the SST.
+  SeqnoToTimeMapping seqno_time_mapping_;
+
+  // cutoff sequence number for penultimate level, only set when
+  // per_key_placement feature is enabled.
+  // If a key with sequence number larger than penultimate_level_cutoff_seqno_,
+  // it will be placed on the penultimate_level and seqnuence number won't be
+  // zeroed out.
+  SequenceNumber penultimate_level_cutoff_seqno_ = kMaxSequenceNumber;
 
   // Get table file name in where it's outputting to, which should also be in
   // `output_directory_`.

db/compaction/compaction_job_test.cc

@@ -372,8 +372,7 @@ class CompactionJobTestBase : public testing::Test {
                  smallest_seqno, largest_seqno, false, Temperature::kUnknown,
                  oldest_blob_file_number, kUnknownOldestAncesterTime,
                  kUnknownFileCreationTime, kUnknownFileChecksum,
-                 kUnknownFileChecksumFuncName, kDisableUserTimestamp,
-                 kDisableUserTimestamp, kNullUniqueId64x2);
+                 kUnknownFileChecksumFuncName, kNullUniqueId64x2);
 
     mutex_.Lock();
     EXPECT_OK(

db/compaction/compaction_outputs.cc

@@ -18,12 +18,19 @@ void CompactionOutputs::NewBuilder(const TableBuilderOptions& tboptions) {
   builder_.reset(NewTableBuilder(tboptions, file_writer_.get()));
 }
 
-Status CompactionOutputs::Finish(const Status& intput_status) {
+Status CompactionOutputs::Finish(const Status& intput_status,
+                                 const SeqnoToTimeMapping& seqno_time_mapping) {
   FileMetaData* meta = GetMetaData();
   assert(meta != nullptr);
   Status s = intput_status;
   if (s.ok()) {
+    std::string seqno_time_mapping_str;
+    seqno_time_mapping.Encode(seqno_time_mapping_str, meta->fd.smallest_seqno,
+                              meta->fd.largest_seqno, meta->file_creation_time);
+    builder_->SetSeqnoTimeTableProperties(seqno_time_mapping_str,
+                                          meta->oldest_ancester_time);
     s = builder_->Finish();
+
   } else {
     builder_->Abandon();
   }

db/compaction/compaction_outputs.h

@@ -111,7 +111,8 @@ class CompactionOutputs {
   }
 
   // Finish the current output file
-  Status Finish(const Status& intput_status);
+  Status Finish(const Status& intput_status,
+                const SeqnoToTimeMapping& seqno_time_mapping);
 
   // Update output table properties from table builder
   void UpdateTableProperties() {

db/compaction/compaction_picker.cc

@@ -462,7 +462,7 @@ bool CompactionPicker::SetupOtherInputs(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     VersionStorageInfo* vstorage, CompactionInputFiles* inputs,
     CompactionInputFiles* output_level_inputs, int* parent_index,
-    int base_index) {
+    int base_index, bool only_expand_towards_right) {
   assert(!inputs->empty());
   assert(output_level_inputs->empty());
   const int input_level = inputs->level;
@@ -515,8 +515,16 @@ bool CompactionPicker::SetupOtherInputs(
     InternalKey all_start, all_limit;
     GetRange(*inputs, *output_level_inputs, &all_start, &all_limit);
     bool try_overlapping_inputs = true;
-    vstorage->GetOverlappingInputs(input_level, &all_start, &all_limit,
-                                   &expanded_inputs.files, base_index, nullptr);
+    if (only_expand_towards_right) {
+      // Round-robin compaction only allows expansion towards the larger side.
+      vstorage->GetOverlappingInputs(input_level, &smallest, &all_limit,
+                                     &expanded_inputs.files, base_index,
+                                     nullptr);
+    } else {
+      vstorage->GetOverlappingInputs(input_level, &all_start, &all_limit,
+                                     &expanded_inputs.files, base_index,
+                                     nullptr);
+    }
     uint64_t expanded_inputs_size =
         TotalCompensatedFileSize(expanded_inputs.files);
     if (!ExpandInputsToCleanCut(cf_name, vstorage, &expanded_inputs)) {

db/compaction/compaction_picker.h

@@ -189,7 +189,8 @@ class CompactionPicker {
                         VersionStorageInfo* vstorage,
                         CompactionInputFiles* inputs,
                         CompactionInputFiles* output_level_inputs,
-                        int* parent_index, int base_index);
+                        int* parent_index, int base_index,
+                        bool only_expand_towards_right = false);
 
   void GetGrandparents(VersionStorageInfo* vstorage,
                        const CompactionInputFiles& inputs,

db/compaction/compaction_picker_fifo.cc

@@ -83,7 +83,7 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
           break;
         }
       }
-      total_size -= f->compensated_file_size;
+      total_size -= f->fd.file_size;
       inputs[0].files.push_back(f);
     }
   }
@@ -191,7 +191,7 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
 
   for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
     auto f = *ritr;
-    total_size -= f->compensated_file_size;
+    total_size -= f->fd.file_size;
     inputs[0].files.push_back(f);
     char tmp_fsize[16];
     AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize));

db/compaction/compaction_picker_level.cc

@@ -76,6 +76,9 @@ class LevelCompactionBuilder {
   // files if needed.
   bool SetupOtherL0FilesIfNeeded();
 
+  // Compaction with round-robin compaction priority allows more files to be
+  // picked to form a large compaction
+  void SetupOtherFilesWithRoundRobinExpansion();
   // Based on initial files, setup other files need to be compacted
   // in this compaction, accordingly.
   bool SetupOtherInputsIfNeeded();
@@ -84,7 +87,9 @@ class LevelCompactionBuilder {
 
   // For the specfied level, pick a file that we want to compact.
   // Returns false if there is no file to compact.
-  // If it returns true, inputs->files.size() will be exactly one.
+  // If it returns true, inputs->files.size() will be exactly one for
+  // all compaction priorities except round-robin. For round-robin,
+  // multiple consecutive files may be put into inputs->files.
   // If level is 0 and there is already a compaction on that level, this
   // function will return false.
   bool PickFileToCompact();
@@ -278,16 +283,141 @@ bool LevelCompactionBuilder::SetupOtherL0FilesIfNeeded() {
   return true;
 }
 
+void LevelCompactionBuilder::SetupOtherFilesWithRoundRobinExpansion() {
+  // We only expand when the start level is not L0 under round robin
+  assert(start_level_ >= 1);
+
+  // For round-robin compaction priority, we have 3 constraints when picking
+  // multiple files.
+  // Constraint 1: We can only pick consecutive files
+  //  -> Constraint 1a: When a file is being compacted (or some input files
+  //                    are being compacted after expanding, we cannot
+  //                    choose it and have to stop choosing more files
+  //  -> Constraint 1b: When we reach the last file (with largest keys), we
+  //                    cannot choose more files (the next file will be the
+  //                    first one)
+  // Constraint 2: We should ensure the total compaction bytes (including the
+  //               overlapped files from the next level) is no more than
+  //               mutable_cf_options_.max_compaction_bytes
+  // Constraint 3: We try our best to pick as many files as possible so that
+  //               the post-compaction level size is less than
+  //               MaxBytesForLevel(start_level_)
+  // Constraint 4: We do not expand if it is possible to apply a trivial move
+  // Constraint 5 (TODO): Try to pick minimal files to split into the target
+  //               number of subcompactions
+  TEST_SYNC_POINT("LevelCompactionPicker::RoundRobin");
+
+  // Only expand the inputs when we have selected a file in start_level_inputs_
+  if (start_level_inputs_.size() == 0) return;
+
+  uint64_t start_lvl_bytes_no_compacting = 0;
+  uint64_t curr_bytes_to_compact = 0;
+  uint64_t start_lvl_max_bytes_to_compact = 0;
+  const std::vector<FileMetaData*>& level_files =
+      vstorage_->LevelFiles(start_level_);
+  // Constraint 3 (pre-calculate the ideal max bytes to compact)
+  for (auto f : level_files) {
+    if (!f->being_compacted) {
+      start_lvl_bytes_no_compacting += f->compensated_file_size;
+    }
+  }
+  if (start_lvl_bytes_no_compacting >
+      vstorage_->MaxBytesForLevel(start_level_)) {
+    start_lvl_max_bytes_to_compact = start_lvl_bytes_no_compacting -
+                                     vstorage_->MaxBytesForLevel(start_level_);
+  }
+
+  size_t start_index = vstorage_->FilesByCompactionPri(start_level_)[0];
+  InternalKey smallest, largest;
+  // Constraint 4 (No need to check again later)
+  compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest);
+  CompactionInputFiles output_level_inputs;
+  output_level_inputs.level = output_level_;
+  vstorage_->GetOverlappingInputs(output_level_, &smallest, &largest,
+                                  &output_level_inputs.files);
+  if (output_level_inputs.empty()) {
+    if (TryExtendNonL0TrivialMove((int)start_index)) {
+      return;
+    }
+  }
+  // Constraint 3
+  if (start_level_inputs_[0]->compensated_file_size >=
+      start_lvl_max_bytes_to_compact) {
+    return;
+  }
+  CompactionInputFiles tmp_start_level_inputs;
+  tmp_start_level_inputs = start_level_inputs_;
+  // TODO (zichen): Future parallel round-robin may also need to update this
+  // Constraint 1b (only expand till the end)
+  for (size_t i = start_index + 1; i < level_files.size(); i++) {
+    auto* f = level_files[i];
+    if (f->being_compacted) {
+      // Constraint 1a
+      return;
+    }
+
+    tmp_start_level_inputs.files.push_back(f);
+    if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+                                                    &tmp_start_level_inputs) ||
+        compaction_picker_->FilesRangeOverlapWithCompaction(
+            {tmp_start_level_inputs}, output_level_)) {
+      // Constraint 1a
+      tmp_start_level_inputs.clear();
+      return;
+    }
+
+    curr_bytes_to_compact = 0;
+    for (auto start_lvl_f : tmp_start_level_inputs.files) {
+      curr_bytes_to_compact += start_lvl_f->compensated_file_size;
+    }
+
+    // Check whether any output level files are locked
+    compaction_picker_->GetRange(tmp_start_level_inputs, &smallest, &largest);
+    vstorage_->GetOverlappingInputs(output_level_, &smallest, &largest,
+                                    &output_level_inputs.files);
+    if (!output_level_inputs.empty() &&
+        !compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+                                                    &output_level_inputs)) {
+      // Constraint 1a
+      tmp_start_level_inputs.clear();
+      return;
+    }
+
+    uint64_t start_lvl_curr_bytes_to_compact = curr_bytes_to_compact;
+    for (auto output_lvl_f : output_level_inputs.files) {
+      curr_bytes_to_compact += output_lvl_f->compensated_file_size;
+    }
+    if (curr_bytes_to_compact > mutable_cf_options_.max_compaction_bytes) {
+      // Constraint 2
+      tmp_start_level_inputs.clear();
+      return;
+    }
+
+    start_level_inputs_.files = tmp_start_level_inputs.files;
+    // Constraint 3
+    if (start_lvl_curr_bytes_to_compact > start_lvl_max_bytes_to_compact) {
+      return;
+    }
+  }
+}
+
 bool LevelCompactionBuilder::SetupOtherInputsIfNeeded() {
   // Setup input files from output level. For output to L0, we only compact
   // spans of files that do not interact with any pending compactions, so don't
   // need to consider other levels.
   if (output_level_ != 0) {
     output_level_inputs_.level = output_level_;
+    bool round_robin_expanding =
+        ioptions_.compaction_pri == kRoundRobin &&
+        compaction_reason_ == CompactionReason::kLevelMaxLevelSize;
+    if (round_robin_expanding) {
+      SetupOtherFilesWithRoundRobinExpansion();
+    }
     if (!is_l0_trivial_move_ &&
         !compaction_picker_->SetupOtherInputs(
             cf_name_, mutable_cf_options_, vstorage_, &start_level_inputs_,
-            &output_level_inputs_, &parent_index_, base_index_)) {
+            &output_level_inputs_, &parent_index_, base_index_,
+            round_robin_expanding)) {
       return false;
     }
 
@@ -606,9 +736,6 @@ bool LevelCompactionBuilder::PickFileToCompact() {
       // user-key overlap.
       start_level_inputs_.clear();
 
-      // To ensure every file is selcted in a round-robin manner, we cannot
-      // skip the current file. So we return false and wait for the next time
-      // we can pick this file to compact
       if (ioptions_.compaction_pri == kRoundRobin) {
         return false;
       }
@@ -641,6 +768,7 @@ bool LevelCompactionBuilder::PickFileToCompact() {
         continue;
       }
     }
+
     base_index_ = index;
     break;
   }

db/compaction/compaction_picker_test.cc

@@ -116,8 +116,7 @@ class CompactionPickerTest : public testing::Test {
         InternalKey(largest, largest_seq, kTypeValue), smallest_seq,
         largest_seq, marked_for_compact, temperature, kInvalidBlobFileNumber,
         kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-        kUnknownFileChecksum, kUnknownFileChecksumFuncName,
-        kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
+        kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
     f->compensated_file_size =
         (compensated_file_size != 0) ? compensated_file_size : file_size;
     f->oldest_ancester_time = oldest_ancestor_time;
@@ -1318,7 +1317,7 @@ TEST_F(CompactionPickerTest, CompactionPriRoundRobin) {
   std::vector<uint32_t> selected_files = {8U, 6U, 6U};
 
   ioptions_.compaction_pri = kRoundRobin;
-  mutable_cf_options_.max_bytes_for_level_base = 10000000;
+  mutable_cf_options_.max_bytes_for_level_base = 12000000;
   mutable_cf_options_.max_bytes_for_level_multiplier = 10;
   for (size_t i = 0; i < test_cursors.size(); i++) {
     // start a brand new version in each test.
@@ -1342,6 +1341,9 @@ TEST_F(CompactionPickerTest, CompactionPriRoundRobin) {
             cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
             &log_buffer_));
     ASSERT_TRUE(compaction.get() != nullptr);
+    // Since the max bytes for level 2 is 120M, picking one file to compact
+    // makes the post-compaction level size less than 120M, there is exactly one
+    // file picked for round-robin compaction
     ASSERT_EQ(1U, compaction->num_input_files(0));
     ASSERT_EQ(selected_files[i], compaction->input(0, 0)->fd.GetNumber());
     // release the version storage
@@ -1349,6 +1351,132 @@ TEST_F(CompactionPickerTest, CompactionPriRoundRobin) {
   }
 }
 
+TEST_F(CompactionPickerTest, CompactionPriMultipleFilesRoundRobin1) {
+  ioptions_.compaction_pri = kRoundRobin;
+  mutable_cf_options_.max_compaction_bytes = 100000000u;
+  mutable_cf_options_.max_bytes_for_level_base = 120;
+  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
+  // start a brand new version in each test.
+  NewVersionStorage(6, kCompactionStyleLevel);
+  vstorage_->ResizeCompactCursors(6);
+  // Set the cursor (file picking should start with 7U)
+  vstorage_->AddCursorForOneLevel(2, InternalKey("199", 100, kTypeValue));
+  Add(2, 6U, "150", "199", 500U);
+  Add(2, 7U, "200", "249", 500U);
+  Add(2, 8U, "300", "600", 500U);
+  Add(2, 9U, "700", "800", 500U);
+  Add(2, 10U, "850", "950", 500U);
+
+  Add(3, 26U, "130", "165", 600U);
+  Add(3, 27U, "166", "170", 600U);
+  Add(3, 28U, "270", "340", 600U);
+  Add(3, 29U, "401", "500", 600U);
+  Add(3, 30U, "601", "800", 600U);
+  Add(3, 31U, "830", "890", 600U);
+  UpdateVersionStorageInfo();
+  LevelCompactionPicker local_level_compaction_picker =
+      LevelCompactionPicker(ioptions_, &icmp_);
+  std::unique_ptr<Compaction> compaction(
+      local_level_compaction_picker.PickCompaction(
+          cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+          &log_buffer_));
+  ASSERT_TRUE(compaction.get() != nullptr);
+
+  // The maximum compaction bytes is very large in this case so we can igore its
+  // constraint in this test case. The maximum bytes for level 2 is 1200
+  // bytes, and thus at least 3 files should be picked so that the bytes in
+  // level 2 is less than the maximum
+  ASSERT_EQ(3U, compaction->num_input_files(0));
+  ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
+  ASSERT_EQ(8U, compaction->input(0, 1)->fd.GetNumber());
+  ASSERT_EQ(9U, compaction->input(0, 2)->fd.GetNumber());
+  // release the version storage
+  DeleteVersionStorage();
+}
+
+TEST_F(CompactionPickerTest, CompactionPriMultipleFilesRoundRobin2) {
+  ioptions_.compaction_pri = kRoundRobin;
+  mutable_cf_options_.max_compaction_bytes = 2500u;
+  mutable_cf_options_.max_bytes_for_level_base = 120;
+  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
+  // start a brand new version in each test.
+  NewVersionStorage(6, kCompactionStyleLevel);
+  vstorage_->ResizeCompactCursors(6);
+  // Set the cursor (file picking should start with 6U)
+  vstorage_->AddCursorForOneLevel(2, InternalKey("1000", 100, kTypeValue));
+  Add(2, 6U, "150", "199", 500U);  // Overlap with 26U, 27U
+  Add(2, 7U, "200", "249", 500U);  // Overlap with 27U
+  Add(2, 8U, "300", "600", 500U);  // Overlap with 28U, 29U
+  Add(2, 9U, "700", "800", 500U);
+  Add(2, 10U, "850", "950", 500U);
+
+  Add(3, 26U, "130", "165", 600U);
+  Add(3, 27U, "166", "230", 600U);
+  Add(3, 28U, "270", "340", 600U);
+  Add(3, 29U, "401", "500", 600U);
+  Add(3, 30U, "601", "800", 600U);
+  Add(3, 31U, "830", "890", 600U);
+  UpdateVersionStorageInfo();
+  LevelCompactionPicker local_level_compaction_picker =
+      LevelCompactionPicker(ioptions_, &icmp_);
+  std::unique_ptr<Compaction> compaction(
+      local_level_compaction_picker.PickCompaction(
+          cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+          &log_buffer_));
+  ASSERT_TRUE(compaction.get() != nullptr);
+
+  // The maximum compaction bytes is only 2500 bytes now. Even though we are
+  // required to choose 3 files so that the post-compaction level size is less
+  // than 1200 bytes. We cannot pick 3 files to compact since the maximum
+  // compaction size is 2500. After picking files 6U and 7U, the number of
+  // compaction bytes has reached 2200, and thus no more space to add another
+  // input file with 50M bytes.
+  ASSERT_EQ(2U, compaction->num_input_files(0));
+  ASSERT_EQ(6U, compaction->input(0, 0)->fd.GetNumber());
+  ASSERT_EQ(7U, compaction->input(0, 1)->fd.GetNumber());
+  // release the version storage
+  DeleteVersionStorage();
+}
+
+TEST_F(CompactionPickerTest, CompactionPriMultipleFilesRoundRobin3) {
+  ioptions_.compaction_pri = kRoundRobin;
+  mutable_cf_options_.max_compaction_bytes = 1000000u;
+  mutable_cf_options_.max_bytes_for_level_base = 120;
+  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
+  // start a brand new version in each test.
+  NewVersionStorage(6, kCompactionStyleLevel);
+  vstorage_->ResizeCompactCursors(6);
+  // Set the cursor (file picking should start with 9U)
+  vstorage_->AddCursorForOneLevel(2, InternalKey("700", 100, kTypeValue));
+  Add(2, 6U, "150", "199", 500U);
+  Add(2, 7U, "200", "249", 500U);
+  Add(2, 8U, "300", "600", 500U);
+  Add(2, 9U, "700", "800", 500U);
+  Add(2, 10U, "850", "950", 500U);
+
+  Add(3, 26U, "130", "165", 600U);
+  Add(3, 27U, "166", "170", 600U);
+  Add(3, 28U, "270", "340", 600U);
+  Add(3, 29U, "401", "500", 600U);
+  Add(3, 30U, "601", "800", 600U);
+  Add(3, 31U, "830", "890", 600U);
+  UpdateVersionStorageInfo();
+  LevelCompactionPicker local_level_compaction_picker =
+      LevelCompactionPicker(ioptions_, &icmp_);
+  std::unique_ptr<Compaction> compaction(
+      local_level_compaction_picker.PickCompaction(
+          cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+          &log_buffer_));
+  ASSERT_TRUE(compaction.get() != nullptr);
+
+  // Cannot pick more files since we reach the last file in level 2
+  ASSERT_EQ(2U, compaction->num_input_files(0));
+  ASSERT_EQ(9U, compaction->input(0, 0)->fd.GetNumber());
+  ASSERT_EQ(10U, compaction->input(0, 1)->fd.GetNumber());
+  // release the version storage
+  DeleteVersionStorage();
+}
+
 TEST_F(CompactionPickerTest, CompactionPriMinOverlappingManyFiles) {
   NewVersionStorage(6, kCompactionStyleLevel);
   ioptions_.compaction_pri = kMinOverlappingRatio;
@@ -3047,6 +3175,148 @@ TEST_F(CompactionPickerTest, UniversalMarkedManualCompaction) {
   ASSERT_EQ(0U, vstorage_->FilesMarkedForCompaction().size());
 }
 
+TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNonLastLevel) {
+  // This test make sure size amplification compaction could still be triggered
+  // if the last sorted run is not the last level.
+  const uint64_t kFileSize = 100000;
+  const int kNumLevels = 7;
+  const int kLastLevel = kNumLevels - 1;
+
+  ioptions_.compaction_style = kCompactionStyleUniversal;
+  ioptions_.preclude_last_level_data_seconds = 1000;
+  mutable_cf_options_.compaction_options_universal
+      .max_size_amplification_percent = 200;
+  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
+
+  NewVersionStorage(kNumLevels, kCompactionStyleUniversal);
+  Add(0, 100U, "100", "300", 1 * kFileSize);
+  Add(0, 101U, "200", "400", 1 * kFileSize);
+  Add(4, 90U, "100", "600", 4 * kFileSize);
+  Add(5, 80U, "200", "300", 2 * kFileSize);
+  UpdateVersionStorageInfo();
+
+  std::unique_ptr<Compaction> compaction(
+      universal_compaction_picker.PickCompaction(
+          cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+          &log_buffer_));
+
+  // Make sure it's a size amp compaction and includes all files
+  ASSERT_EQ(compaction->compaction_reason(),
+            CompactionReason::kUniversalSizeAmplification);
+  ASSERT_EQ(compaction->output_level(), kLastLevel);
+  ASSERT_EQ(compaction->input_levels(0)->num_files, 2);
+  ASSERT_EQ(compaction->input_levels(4)->num_files, 1);
+  ASSERT_EQ(compaction->input_levels(5)->num_files, 1);
+}
+
+TEST_F(CompactionPickerTest, UniversalSizeRatioTierCompactionLastLevel) {
+  // This test makes sure the size amp calculation skips the last level (L6), so
+  // size amp compaction is not triggered, instead a size ratio compaction is
+  // triggered.
+  const uint64_t kFileSize = 100000;
+  const int kNumLevels = 7;
+  const int kLastLevel = kNumLevels - 1;
+  const int kPenultimateLevel = kLastLevel - 1;
+
+  ioptions_.compaction_style = kCompactionStyleUniversal;
+  ioptions_.preclude_last_level_data_seconds = 1000;
+  mutable_cf_options_.compaction_options_universal
+      .max_size_amplification_percent = 200;
+  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
+
+  NewVersionStorage(kNumLevels, kCompactionStyleUniversal);
+  Add(0, 100U, "100", "300", 1 * kFileSize);
+  Add(0, 101U, "200", "400", 1 * kFileSize);
+  Add(5, 90U, "100", "600", 4 * kFileSize);
+  Add(6, 80U, "200", "300", 2 * kFileSize);
+  UpdateVersionStorageInfo();
+
+  std::unique_ptr<Compaction> compaction(
+      universal_compaction_picker.PickCompaction(
+          cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+          &log_buffer_));
+
+  // Internally, size amp compaction is evaluated before size ratio compaction.
+  // Here to make sure it's size ratio compaction instead of size amp
+  ASSERT_EQ(compaction->compaction_reason(),
+            CompactionReason::kUniversalSizeRatio);
+  ASSERT_EQ(compaction->output_level(), kPenultimateLevel - 1);
+  ASSERT_EQ(compaction->input_levels(0)->num_files, 2);
+  ASSERT_EQ(compaction->input_levels(5)->num_files, 0);
+  ASSERT_EQ(compaction->input_levels(6)->num_files, 0);
+}
+
+TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionNotSuport) {
+  // Tiered compaction only support level_num > 2 (otherwise the penultimate
+  // level is going to be level 0, which may make thing more complicated), so
+  // when there's only 2 level, still treating level 1 as the last level for
+  // size amp compaction
+  const uint64_t kFileSize = 100000;
+  const int kNumLevels = 2;
+  const int kLastLevel = kNumLevels - 1;
+
+  ioptions_.compaction_style = kCompactionStyleUniversal;
+  ioptions_.preclude_last_level_data_seconds = 1000;
+  mutable_cf_options_.compaction_options_universal
+      .max_size_amplification_percent = 200;
+  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
+
+  NewVersionStorage(kNumLevels, kCompactionStyleUniversal);
+  Add(0, 100U, "100", "300", 1 * kFileSize);
+  Add(0, 101U, "200", "400", 1 * kFileSize);
+  Add(0, 90U, "100", "600", 4 * kFileSize);
+  Add(1, 80U, "200", "300", 2 * kFileSize);
+  UpdateVersionStorageInfo();
+
+  std::unique_ptr<Compaction> compaction(
+      universal_compaction_picker.PickCompaction(
+          cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+          &log_buffer_));
+
+  // size amp compaction is still triggered even preclude_last_level is set
+  ASSERT_EQ(compaction->compaction_reason(),
+            CompactionReason::kUniversalSizeAmplification);
+  ASSERT_EQ(compaction->output_level(), kLastLevel);
+  ASSERT_EQ(compaction->input_levels(0)->num_files, 3);
+  ASSERT_EQ(compaction->input_levels(1)->num_files, 1);
+}
+
+TEST_F(CompactionPickerTest, UniversalSizeAmpTierCompactionLastLevel) {
+  // This test makes sure the size amp compaction for tiered storage could still
+  // be triggered, but only for non-last-level files
+  const uint64_t kFileSize = 100000;
+  const int kNumLevels = 7;
+  const int kLastLevel = kNumLevels - 1;
+  const int kPenultimateLevel = kLastLevel - 1;
+
+  ioptions_.compaction_style = kCompactionStyleUniversal;
+  ioptions_.preclude_last_level_data_seconds = 1000;
+  mutable_cf_options_.compaction_options_universal
+      .max_size_amplification_percent = 200;
+  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
+
+  NewVersionStorage(kNumLevels, kCompactionStyleUniversal);
+  Add(0, 100U, "100", "300", 3 * kFileSize);
+  Add(0, 101U, "200", "400", 2 * kFileSize);
+  Add(5, 90U, "100", "600", 2 * kFileSize);
+  Add(6, 80U, "200", "300", 2 * kFileSize);
+  UpdateVersionStorageInfo();
+
+  std::unique_ptr<Compaction> compaction(
+      universal_compaction_picker.PickCompaction(
+          cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+          &log_buffer_));
+
+  // It's a Size Amp compaction, but doesn't include the last level file and
+  // output to the penultimate level.
+  ASSERT_EQ(compaction->compaction_reason(),
+            CompactionReason::kUniversalSizeAmplification);
+  ASSERT_EQ(compaction->output_level(), kPenultimateLevel);
+  ASSERT_EQ(compaction->input_levels(0)->num_files, 2);
+  ASSERT_EQ(compaction->input_levels(5)->num_files, 1);
+  ASSERT_EQ(compaction->input_levels(6)->num_files, 0);
+}
+
 class PerKeyPlacementCompactionPickerTest
     : public CompactionPickerTest,
       public testing::WithParamInterface<bool> {

db/compaction/compaction_picker_universal.cc

@@ -106,6 +106,9 @@ class UniversalCompactionBuilder {
   Compaction* PickCompactionToOldest(size_t start_index,
                                      CompactionReason compaction_reason);
 
+  Compaction* PickCompactionWithSortedRunRange(
+      size_t start_index, size_t end_index, CompactionReason compaction_reason);
+
   // Try to pick periodic compaction. The caller should only call it
   // if there is at least one file marked for periodic compaction.
   // null will be returned if no such a compaction can be formed
@@ -811,8 +814,20 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() {
         cf_name_.c_str(), file_num_buf, start_index, " to reduce size amp.\n");
   }
 
+  // size of the base sorted run for size amp calculation
+  uint64_t base_sr_size = sorted_runs_.back().size;
+  size_t sr_end_idx = sorted_runs_.size() - 1;
+  // If tiered compaction is enabled and the last sorted run is the last level
+  if (ioptions_.preclude_last_level_data_seconds > 0 &&
+      ioptions_.num_levels > 2 &&
+      sorted_runs_.back().level == ioptions_.num_levels - 1 &&
+      sorted_runs_.size() > 1) {
+    sr_end_idx = sorted_runs_.size() - 2;
+    base_sr_size = sorted_runs_[sr_end_idx].size;
+  }
+
   // keep adding up all the remaining files
-  for (size_t loop = start_index; loop + 1 < sorted_runs_.size(); loop++) {
+  for (size_t loop = start_index; loop < sr_end_idx; loop++) {
     sr = &sorted_runs_[loop];
     if (sr->being_compacted) {
       // TODO with incremental compaction is supported, we might want to
@@ -832,23 +847,20 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() {
     return nullptr;
   }
 
-  // size of earliest file
-  uint64_t earliest_file_size = sorted_runs_.back().size;
-
   // size amplification = percentage of additional size
-  if (candidate_size * 100 < ratio * earliest_file_size) {
+  if (candidate_size * 100 < ratio * base_sr_size) {
     ROCKS_LOG_BUFFER(
         log_buffer_,
         "[%s] Universal: size amp not needed. newer-files-total-size %" PRIu64
         " earliest-file-size %" PRIu64,
-        cf_name_.c_str(), candidate_size, earliest_file_size);
+        cf_name_.c_str(), candidate_size, base_sr_size);
     return nullptr;
   } else {
     ROCKS_LOG_BUFFER(
         log_buffer_,
         "[%s] Universal: size amp needed. newer-files-total-size %" PRIu64
         " earliest-file-size %" PRIu64,
-        cf_name_.c_str(), candidate_size, earliest_file_size);
+        cf_name_.c_str(), candidate_size, base_sr_size);
   }
   // Since incremental compaction can't include more than second last
   // level, it can introduce penalty, compared to full compaction. We
@@ -860,7 +872,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() {
   // This also prevent the case when compaction falls behind and we
   // need to compact more levels for compactions to catch up.
   if (mutable_cf_options_.compaction_options_universal.incremental) {
-    double fanout_threshold = static_cast<double>(earliest_file_size) /
+    double fanout_threshold = static_cast<double>(base_sr_size) /
                               static_cast<double>(candidate_size) * 1.8;
     Compaction* picked = PickIncrementalForReduceSizeAmp(fanout_threshold);
     if (picked != nullptr) {
@@ -869,8 +881,8 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() {
       return picked;
     }
   }
-  return PickCompactionToOldest(start_index,
-                                CompactionReason::kUniversalSizeAmplification);
+  return PickCompactionWithSortedRunRange(
+      start_index, sr_end_idx, CompactionReason::kUniversalSizeAmplification);
 }
 
 Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
@@ -1233,11 +1245,17 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
 
 Compaction* UniversalCompactionBuilder::PickCompactionToOldest(
     size_t start_index, CompactionReason compaction_reason) {
+  return PickCompactionWithSortedRunRange(start_index, sorted_runs_.size() - 1,
+                                          compaction_reason);
+}
+
+Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange(
+    size_t start_index, size_t end_index, CompactionReason compaction_reason) {
   assert(start_index < sorted_runs_.size());
 
   // Estimate total file size
   uint64_t estimated_total_size = 0;
-  for (size_t loop = start_index; loop < sorted_runs_.size(); loop++) {
+  for (size_t loop = start_index; loop <= end_index; loop++) {
     estimated_total_size += sorted_runs_[loop].size;
   }
   uint32_t path_id =
@@ -1248,7 +1266,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToOldest(
   for (size_t i = 0; i < inputs.size(); ++i) {
     inputs[i].level = start_level + static_cast<int>(i);
   }
-  for (size_t loop = start_index; loop < sorted_runs_.size(); loop++) {
+  for (size_t loop = start_index; loop <= end_index; loop++) {
     auto& picking_sr = sorted_runs_[loop];
     if (picking_sr.level == 0) {
       FileMetaData* f = picking_sr.file;
@@ -1279,12 +1297,19 @@ Compaction* UniversalCompactionBuilder::PickCompactionToOldest(
                      file_num_buf);
   }
 
-  // output files at the bottom most level, unless it's reserved
-  int output_level = vstorage_->num_levels() - 1;
-  // last level is reserved for the files ingested behind
-  if (ioptions_.allow_ingest_behind) {
-    assert(output_level > 1);
-    output_level--;
+  int output_level;
+  if (end_index == sorted_runs_.size() - 1) {
+    // output files at the last level, unless it's reserved
+    output_level = vstorage_->num_levels() - 1;
+    // last level is reserved for the files ingested behind
+    if (ioptions_.allow_ingest_behind) {
+      assert(output_level > 1);
+      output_level--;
+    }
+  } else {
+    // if it's not including all sorted_runs, it can only output to the level
+    // above the `end_index + 1` sorted_run.
+    output_level = sorted_runs_[end_index + 1].level - 1;
   }
 
   // We never check size for

db/compaction/compaction_service_job.cc

@@ -47,10 +47,10 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(
   compaction_input.db_options =
       BuildDBOptions(db_options_, mutable_db_options_copy_);
   compaction_input.snapshots = existing_snapshots_;
-  compaction_input.has_begin = sub_compact->start;
+  compaction_input.has_begin = sub_compact->start.has_value();
   compaction_input.begin =
       compaction_input.has_begin ? sub_compact->start->ToString() : "";
-  compaction_input.has_end = sub_compact->end;
+  compaction_input.has_end = sub_compact->end.has_value();
   compaction_input.end =
       compaction_input.has_end ? sub_compact->end->ToString() : "";
 
@@ -264,8 +264,12 @@ Status CompactionServiceCompactionJob::Run() {
   Slice begin = compaction_input_.begin;
   Slice end = compaction_input_.end;
   compact_->sub_compact_states.emplace_back(
-      c, compaction_input_.has_begin ? &begin : nullptr,
-      compaction_input_.has_end ? &end : nullptr, /*sub_job_id*/ 0);
+      c,
+      compaction_input_.has_begin ? std::optional<Slice>(begin)
+                                  : std::optional<Slice>(),
+      compaction_input_.has_end ? std::optional<Slice>(end)
+                                : std::optional<Slice>(),
+      /*sub_job_id*/ 0);
 
   log_buffer_->FlushBufferToLog();
   LogCompaction();

db/compaction/subcompaction_state.h

@@ -10,6 +10,8 @@
 
 #pragma once
 
+#include <optional>
+
 #include "db/blob/blob_file_addition.h"
 #include "db/blob/blob_garbage_meter.h"
 #include "db/compaction/compaction.h"
@@ -52,7 +54,7 @@ class SubcompactionState {
   // The boundaries of the key-range this compaction is interested in. No two
   // sub-compactions may have overlapping key-ranges.
   // 'start' is inclusive, 'end' is exclusive, and nullptr means unbounded
-  const Slice *start, *end;
+  const std::optional<Slice> start, end;
 
   // The return status of this sub-compaction
   Status status;
@@ -117,8 +119,8 @@ class SubcompactionState {
   SubcompactionState(const SubcompactionState&) = delete;
   SubcompactionState& operator=(const SubcompactionState&) = delete;
 
-  SubcompactionState(Compaction* c, Slice* _start, Slice* _end,
-                     uint32_t _sub_job_id)
+  SubcompactionState(Compaction* c, const std::optional<Slice> _start,
+                     const std::optional<Slice> _end, uint32_t _sub_job_id)
       : compaction(c),
         start(_start),
         end(_end),
@@ -132,12 +134,12 @@ class SubcompactionState {
     // Invalid output_split_key indicates that we do not need to split
     if (output_split_key != nullptr) {
       // We may only split the output when the cursor is in the range. Split
-      if ((end == nullptr || icmp->user_comparator()->Compare(
-                                 ExtractUserKey(output_split_key->Encode()),
-                                 ExtractUserKey(*end)) < 0) &&
-          (start == nullptr || icmp->user_comparator()->Compare(
-                                   ExtractUserKey(output_split_key->Encode()),
-                                   ExtractUserKey(*start)) > 0)) {
+      if ((!end.has_value() ||
+           icmp->user_comparator()->Compare(
+               ExtractUserKey(output_split_key->Encode()), end.value()) < 0) &&
+          (!start.has_value() || icmp->user_comparator()->Compare(
+                                     ExtractUserKey(output_split_key->Encode()),
+                                     start.value()) > 0)) {
         local_output_split_key_ = output_split_key;
       }
     }

db/compaction/tiered_compaction_test.cc

@@ -16,7 +16,8 @@ namespace ROCKSDB_NAMESPACE {
 
 #if !defined(ROCKSDB_LITE)
 
-class TieredCompactionTest : public DBTestBase {
+class TieredCompactionTest : public DBTestBase,
+                             public testing::WithParamInterface<bool> {
  public:
   TieredCompactionTest()
       : DBTestBase("tiered_compaction_test", /*env_do_fsync=*/true),
@@ -53,26 +54,17 @@ class TieredCompactionTest : public DBTestBase {
   InternalStats::CompactionOutputsStats kBasicPerLevelStats;
   InternalStats::CompactionStats kBasicFlushStats;
 
+  std::atomic_bool enable_per_key_placement = true;
+
   void SetUp() override {
     SyncPoint::GetInstance()->SetCallBack(
         "Compaction::SupportsPerKeyPlacement:Enabled", [&](void* arg) {
           auto supports_per_key_placement = static_cast<bool*>(arg);
-          *supports_per_key_placement = true;
+          *supports_per_key_placement = enable_per_key_placement;
         });
     SyncPoint::GetInstance()->EnableProcessing();
   }
 
-#ifndef ROCKSDB_LITE
-  uint64_t GetSstSizeHelper(Temperature temperature) {
-    std::string prop;
-    EXPECT_TRUE(dbfull()->GetProperty(
-        DB::Properties::kLiveSstFilesSizeAtTemperature +
-            std::to_string(static_cast<uint8_t>(temperature)),
-        &prop));
-    return static_cast<uint64_t>(std::atoi(prop.c_str()));
-  }
-#endif  // ROCKSDB_LITE
-
   const std::vector<InternalStats::CompactionStats>& GetCompactionStats() {
     VersionSet* const versions = dbfull()->GetVersionSet();
     assert(versions);
@@ -129,6 +121,16 @@ class TieredCompactionTest : public DBTestBase {
     pl_stats.Clear();
   }
 
+  // bottommost_temperature is renaming to last_level_temperature, set either
+  // of them should have the same effect.
+  void SetColdTemperature(Options& options) {
+    if (GetParam()) {
+      options.bottommost_temperature = Temperature::kCold;
+    } else {
+      options.last_level_temperature = Temperature::kCold;
+    }
+  }
+
  private:
   void CompareStats(uint64_t val, uint64_t expect) {
     if (expect > 0) {
@@ -168,7 +170,7 @@ class TieredCompactionTest : public DBTestBase {
   }
 };
 
-TEST_F(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
+TEST_P(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 100;
@@ -176,7 +178,7 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
 
   auto options = CurrentOptions();
   options.compaction_style = kCompactionStyleUniversal;
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.statistics = CreateDBStatistics();
   options.max_subcompactions = 10;
@@ -330,7 +332,7 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageUniversal) {
   ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }
 
-TEST_F(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
+TEST_P(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 100;
@@ -338,7 +340,7 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
 
   auto options = CurrentOptions();
   options.compaction_style = kCompactionStyleUniversal;
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.statistics = CreateDBStatistics();
   options.max_subcompactions = 10;
@@ -522,13 +524,14 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageUniversal) {
       1);
 }
 
-TEST_F(TieredCompactionTest, LevelColdRangeDelete) {
+TEST_P(TieredCompactionTest, LevelColdRangeDelete) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 100;
+  const int kLastLevel = kNumLevels - 1;
 
   auto options = CurrentOptions();
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.num_levels = kNumLevels;
   options.statistics = CreateDBStatistics();
@@ -553,11 +556,13 @@ TEST_F(TieredCompactionTest, LevelColdRangeDelete) {
   CompactRangeOptions cro;
   cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
-  ASSERT_EQ("0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
+  ASSERT_EQ("0,1",
+            FilesPerLevel());  // bottommost but not last level file is hot
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
 
-  MoveFilesToLevel(kNumLevels - 1);
+  // explicitly move the data to the last level
+  MoveFilesToLevel(kLastLevel);
 
   ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
 
@@ -625,14 +630,14 @@ class SingleKeySstPartitionerFactory : public SstPartitionerFactory {
   }
 };
 
-TEST_F(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
+TEST_P(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
   const int kNumTrigger = 4;
   const int kNumLevels = 3;
   const int kNumKeys = 10;
 
   auto factory = std::make_shared<SingleKeySstPartitionerFactory>();
   auto options = CurrentOptions();
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.num_levels = kNumLevels;
   options.statistics = CreateDBStatistics();
@@ -743,7 +748,7 @@ TEST_F(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
   ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }
 
-TEST_F(TieredCompactionTest, UniversalRangeDelete) {
+TEST_P(TieredCompactionTest, UniversalRangeDelete) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 10;
@@ -752,7 +757,7 @@ TEST_F(TieredCompactionTest, UniversalRangeDelete) {
 
   auto options = CurrentOptions();
   options.compaction_style = kCompactionStyleUniversal;
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.statistics = CreateDBStatistics();
   options.sst_partitioner_factory = factory;
@@ -875,14 +880,14 @@ TEST_F(TieredCompactionTest, UniversalRangeDelete) {
   ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }
 
-TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
+TEST_P(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 100;
   const int kLastLevel = kNumLevels - 1;
 
   auto options = CurrentOptions();
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.num_levels = kNumLevels;
   options.statistics = CreateDBStatistics();
@@ -913,23 +918,22 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
   }
   ASSERT_OK(dbfull()->WaitForCompact(true));
 
-  // non-last-level compaction doesn't support per_key_placement
+  // non last level is hot
   ASSERT_EQ("0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
 
   expect_stats[1].Add(kBasicCompStats);
   expect_stats[1].Add(kBasicPerLevelStats);
   expect_stats[1].ResetCompactionReason(CompactionReason::kLevelL0FilesNum);
   VerifyCompactionStats(expect_stats, expect_pl_stats);
 
+  // move all data to the last level
   MoveFilesToLevel(kLastLevel);
 
   ResetAllStats(expect_stats, expect_pl_stats);
 
-  // the data should be all hot, and it's a last level compaction, but all
-  // sequence numbers have been zeroed out, so they're still treated as old
-  // data.
+  // The compaction won't move the data up
   CompactRangeOptions cro;
   cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
@@ -982,49 +986,19 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
 
   // move forward the cold_seq, try to split the data into cold and hot, but in
   // this case it's unsafe to split the data
+  // because it's non-last-level but bottommost file, the sequence number will
+  // be zeroed out and lost the time information (with
+  // `level_compaction_dynamic_level_bytes` or Universal Compaction, it should
+  // be rare.)
+  // TODO(zjay): ideally we should avoid zero out non-last-level bottommost file
   latest_cold_seq = seq_history[1];
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
   ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
 
   seq_history.clear();
 
-  // Add new data again
-  for (int i = 0; i < kNumTrigger; i++) {
-    for (int j = 0; j < kNumKeys; j++) {
-      ASSERT_OK(Put(Key(i * 10 + j), "value" + std::to_string(i)));
-    }
-    ASSERT_OK(Flush());
-    seq_history.emplace_back(dbfull()->GetLatestSequenceNumber());
-  }
-  ASSERT_OK(dbfull()->WaitForCompact(true));
-
-  ResetAllStats(expect_stats, expect_pl_stats);
-
-  // Try to split the last level cold data into hot and cold, which
-  // is not supported
-  latest_cold_seq = seq_history[0];
-  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
-  ASSERT_EQ("0,0,0,0,0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
-  ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
-
-  auto comp_stats = kBasicCompStats;
-  comp_stats.ResetCompactionReason(CompactionReason::kManualCompaction);
-  const int bottommost_level = 5;
-  expect_stats[bottommost_level].Add(comp_stats);
-  expect_stats[bottommost_level].Add(
-      comp_stats);  // bottommost level has 2 compactions
-  expect_stats[bottommost_level].Add(kBasicPerLevelStats);
-  expect_stats[bottommost_level].bytes_read_output_level = kHasValue;
-  expect_stats[bottommost_level].num_input_files_in_output_level = kHasValue;
-
-  for (int level = 2; level < bottommost_level; level++) {
-    expect_stats[level].bytes_moved = kHasValue;
-  }
-  VerifyCompactionStats(expect_stats, expect_pl_stats);
-
   // manually move all data (cold) to last level
   MoveFilesToLevel(kLastLevel);
   seq_history.clear();
@@ -1054,12 +1028,14 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
   ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
   ASSERT_EQ(GetSstSizeHelper(Temperature::kCold), 0);
 
+  latest_cold_seq = seq_history[2];
+
   MoveFilesToLevel(kLastLevel);
 
   // move forward the cold_seq again with range delete, take a snapshot to keep
   // the range dels in bottommost
   auto snap = db_->GetSnapshot();
-  latest_cold_seq = seq_history[2];
+
   std::string start = Key(25), end = Key(35);
   ASSERT_OK(
       db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), start, end));
@@ -1105,8 +1081,8 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
   db_->ReleaseSnapshot(snap);
 
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
-  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
   ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 
   // 3 range dels dropped, the first one is double counted as expected, which is
@@ -1123,18 +1099,18 @@ TEST_F(TieredCompactionTest, SequenceBasedTieredStorageLevel) {
   // input range
   latest_cold_seq = seq_history[1];
   ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown), 0);
+  ASSERT_EQ("0,0,0,0,0,1,1", FilesPerLevel());
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
   ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
 }
 
-TEST_F(TieredCompactionTest, RangeBasedTieredStorageLevel) {
+TEST_P(TieredCompactionTest, RangeBasedTieredStorageLevel) {
   const int kNumTrigger = 4;
   const int kNumLevels = 7;
   const int kNumKeys = 100;
 
   auto options = CurrentOptions();
-  options.bottommost_temperature = Temperature::kCold;
+  SetColdTemperature(options);
   options.level0_file_num_compaction_trigger = kNumTrigger;
   options.level_compaction_dynamic_level_bytes = true;
   options.num_levels = kNumLevels;
@@ -1236,6 +1212,9 @@ TEST_F(TieredCompactionTest, RangeBasedTieredStorageLevel) {
       1);
 }
 
+INSTANTIATE_TEST_CASE_P(TieredCompactionTest, TieredCompactionTest,
+                        testing::Bool());
+
 #endif  // !defined(ROCKSDB_LITE)
 
 }  // namespace ROCKSDB_NAMESPACE

db/corruption_test.cc

@@ -764,6 +764,7 @@ TEST_F(CorruptionTest, ParanoidFileChecksOnCompact) {
     delete db_;
     db_ = nullptr;
     s = DestroyDB(dbname_, options);
+    ASSERT_OK(s);
     std::shared_ptr<mock::MockTableFactory> mock =
         std::make_shared<mock::MockTableFactory>();
     options.table_factory = mock;

db/db_basic_test.cc

@@ -1209,9 +1209,9 @@ TEST_F(DBBasicTest, DBCloseAllDirectoryFDs) {
   s = db->Close();
   auto* counted_fs =
       options.env->GetFileSystem()->CheckedCast<CountedFileSystem>();
-  assert(counted_fs);
-  ASSERT_TRUE(counted_fs->counters()->dir_opens ==
-              counted_fs->counters()->dir_closes);
+  ASSERT_TRUE(counted_fs != nullptr);
+  ASSERT_EQ(counted_fs->counters()->dir_opens,
+            counted_fs->counters()->dir_closes);
   ASSERT_OK(s);
   delete db;
 }
@@ -2013,12 +2013,10 @@ TEST_P(DBMultiGetTestWithParam, MultiGetBatchedValueSize) {
 }
 
 TEST_P(DBMultiGetTestWithParam, MultiGetBatchedValueSizeMultiLevelMerge) {
-#ifndef USE_COROUTINES
   if (std::get<1>(GetParam())) {
-    ROCKSDB_GTEST_SKIP("This test requires coroutine support");
+    ROCKSDB_GTEST_SKIP("This test needs to be fixed for async IO");
     return;
   }
-#endif  // USE_COROUTINES
   // Skip for unbatched MultiGet
   if (!std::get<0>(GetParam())) {
     ROCKSDB_GTEST_BYPASS("This test is only for batched MultiGet");
@@ -2131,7 +2129,8 @@ INSTANTIATE_TEST_CASE_P(DBMultiGetTestWithParam, DBMultiGetTestWithParam,
                         testing::Combine(testing::Bool(), testing::Bool()));
 
 #if USE_COROUTINES
-class DBMultiGetAsyncIOTest : public DBBasicTest {
+class DBMultiGetAsyncIOTest : public DBBasicTest,
+                              public ::testing::WithParamInterface<bool> {
  public:
   DBMultiGetAsyncIOTest()
       : DBBasicTest(), statistics_(ROCKSDB_NAMESPACE::CreateDBStatistics()) {
@@ -2146,7 +2145,7 @@ class DBMultiGetAsyncIOTest : public DBBasicTest {
 
     // Put all keys in the bottommost level, and overwrite some keys
     // in L0 and L1
-    for (int i = 0; i < 128; ++i) {
+    for (int i = 0; i < 256; ++i) {
       EXPECT_OK(Put(Key(i), "val_l2_" + std::to_string(i)));
       num_keys++;
       if (num_keys == 8) {
@@ -2172,6 +2171,21 @@ class DBMultiGetAsyncIOTest : public DBBasicTest {
       EXPECT_OK(Flush());
       num_keys = 0;
     }
+    // Put some range deletes in L1
+    for (int i = 128; i < 256; i += 32) {
+      std::string range_begin = Key(i);
+      std::string range_end = Key(i + 16);
+      EXPECT_OK(dbfull()->DeleteRange(WriteOptions(),
+                                      dbfull()->DefaultColumnFamily(),
+                                      range_begin, range_end));
+      // Also do some Puts to force creation of bloom filter
+      for (int j = i + 16; j < i + 32; ++j) {
+        if (j % 3 == 0) {
+          EXPECT_OK(Put(Key(j), "val_l1_" + std::to_string(j)));
+        }
+      }
+      EXPECT_OK(Flush());
+    }
     MoveFilesToLevel(1);
 
     for (int i = 0; i < 128; i += 5) {
@@ -2195,7 +2209,7 @@ class DBMultiGetAsyncIOTest : public DBBasicTest {
   std::shared_ptr<Statistics> statistics_;
 };
 
-TEST_F(DBMultiGetAsyncIOTest, GetFromL0) {
+TEST_P(DBMultiGetAsyncIOTest, GetFromL0) {
   // All 3 keys in L0. The L0 files should be read serially.
   std::vector<std::string> key_strs{Key(0), Key(40), Key(80)};
   std::vector<Slice> keys{key_strs[0], key_strs[1], key_strs[2]};
@@ -2204,6 +2218,7 @@ TEST_F(DBMultiGetAsyncIOTest, GetFromL0) {
 
   ReadOptions ro;
   ro.async_io = true;
+  ro.optimize_multiget_for_io = GetParam();
   dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
                      keys.data(), values.data(), statuses.data());
   ASSERT_EQ(values.size(), 3);
@@ -2218,13 +2233,17 @@ TEST_F(DBMultiGetAsyncIOTest, GetFromL0) {
 
   statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size);
 
-  // No async IO in this case since we don't do parallel lookup in L0
-  ASSERT_EQ(multiget_io_batch_size.count, 0);
-  ASSERT_EQ(multiget_io_batch_size.max, 0);
-  ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 0);
+  // With async IO, lookups will happen in parallel for each key
+  if (GetParam()) {
+    ASSERT_EQ(multiget_io_batch_size.count, 1);
+    ASSERT_EQ(multiget_io_batch_size.max, 3);
+    ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 3);
+  } else {
+    ASSERT_EQ(multiget_io_batch_size.count, 0);
+  }
 }
 
-TEST_F(DBMultiGetAsyncIOTest, GetFromL1) {
+TEST_P(DBMultiGetAsyncIOTest, GetFromL1) {
   std::vector<std::string> key_strs;
   std::vector<Slice> keys;
   std::vector<PinnableSlice> values;
@@ -2241,6 +2260,7 @@ TEST_F(DBMultiGetAsyncIOTest, GetFromL1) {
 
   ReadOptions ro;
   ro.async_io = true;
+  ro.optimize_multiget_for_io = GetParam();
   dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
                      keys.data(), values.data(), statuses.data());
   ASSERT_EQ(values.size(), 3);
@@ -2261,7 +2281,7 @@ TEST_F(DBMultiGetAsyncIOTest, GetFromL1) {
   ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 3);
 }
 
-TEST_F(DBMultiGetAsyncIOTest, LastKeyInFile) {
+TEST_P(DBMultiGetAsyncIOTest, LastKeyInFile) {
   std::vector<std::string> key_strs;
   std::vector<Slice> keys;
   std::vector<PinnableSlice> values;
@@ -2279,6 +2299,7 @@ TEST_F(DBMultiGetAsyncIOTest, LastKeyInFile) {
 
   ReadOptions ro;
   ro.async_io = true;
+  ro.optimize_multiget_for_io = GetParam();
   dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
                      keys.data(), values.data(), statuses.data());
   ASSERT_EQ(values.size(), 3);
@@ -2301,7 +2322,7 @@ TEST_F(DBMultiGetAsyncIOTest, LastKeyInFile) {
   ASSERT_EQ(multiget_io_batch_size.max, 2);
 }
 
-TEST_F(DBMultiGetAsyncIOTest, GetFromL1AndL2) {
+TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2) {
   std::vector<std::string> key_strs;
   std::vector<Slice> keys;
   std::vector<PinnableSlice> values;
@@ -2319,6 +2340,7 @@ TEST_F(DBMultiGetAsyncIOTest, GetFromL1AndL2) {
 
   ReadOptions ro;
   ro.async_io = true;
+  ro.optimize_multiget_for_io = GetParam();
   dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
                      keys.data(), values.data(), statuses.data());
   ASSERT_EQ(values.size(), 3);
@@ -2333,11 +2355,101 @@ TEST_F(DBMultiGetAsyncIOTest, GetFromL1AndL2) {
 
   statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size);
 
-  // There is only one MultiGet key in the bottommost level - 56. Thus
-  // the bottommost level will not use async IO.
+  // There are 2 keys in L1 in twp separate files, and 1 in L2. With
+  // async IO, all three lookups will happen in parallel
   ASSERT_EQ(multiget_io_batch_size.count, 1);
-  ASSERT_EQ(multiget_io_batch_size.max, 2);
+  ASSERT_EQ(multiget_io_batch_size.max, GetParam() ? 3 : 2);
 }
+
+TEST_P(DBMultiGetAsyncIOTest, GetFromL2WithRangeOverlapL0L1) {
+  std::vector<std::string> key_strs;
+  std::vector<Slice> keys;
+  std::vector<PinnableSlice> values;
+  std::vector<Status> statuses;
+
+  // 19 and 26 are in L2, but overlap with L0 and L1 file ranges
+  key_strs.push_back(Key(19));
+  key_strs.push_back(Key(26));
+  keys.push_back(key_strs[0]);
+  keys.push_back(key_strs[1]);
+  values.resize(keys.size());
+  statuses.resize(keys.size());
+
+  ReadOptions ro;
+  ro.async_io = true;
+  ro.optimize_multiget_for_io = GetParam();
+  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
+                     keys.data(), values.data(), statuses.data());
+  ASSERT_EQ(values.size(), 2);
+  ASSERT_EQ(statuses[0], Status::OK());
+  ASSERT_EQ(statuses[1], Status::OK());
+  ASSERT_EQ(values[0], "val_l2_" + std::to_string(19));
+  ASSERT_EQ(values[1], "val_l2_" + std::to_string(26));
+
+  // Bloom filters in L0/L1 will avoid the coroutine calls in those levels
+  ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 2);
+}
+
+TEST_P(DBMultiGetAsyncIOTest, GetFromL2WithRangeDelInL1) {
+  std::vector<std::string> key_strs;
+  std::vector<Slice> keys;
+  std::vector<PinnableSlice> values;
+  std::vector<Status> statuses;
+
+  // 139 and 163 are in L2, but overlap with a range deletes in L1
+  key_strs.push_back(Key(139));
+  key_strs.push_back(Key(163));
+  keys.push_back(key_strs[0]);
+  keys.push_back(key_strs[1]);
+  values.resize(keys.size());
+  statuses.resize(keys.size());
+
+  ReadOptions ro;
+  ro.async_io = true;
+  ro.optimize_multiget_for_io = GetParam();
+  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
+                     keys.data(), values.data(), statuses.data());
+  ASSERT_EQ(values.size(), 2);
+  ASSERT_EQ(statuses[0], Status::NotFound());
+  ASSERT_EQ(statuses[1], Status::NotFound());
+
+  // Bloom filters in L0/L1 will avoid the coroutine calls in those levels
+  ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 2);
+}
+
+TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2WithRangeDelInL1) {
+  std::vector<std::string> key_strs;
+  std::vector<Slice> keys;
+  std::vector<PinnableSlice> values;
+  std::vector<Status> statuses;
+
+  // 139 and 163 are in L2, but overlap with a range deletes in L1
+  key_strs.push_back(Key(139));
+  key_strs.push_back(Key(144));
+  key_strs.push_back(Key(163));
+  keys.push_back(key_strs[0]);
+  keys.push_back(key_strs[1]);
+  keys.push_back(key_strs[2]);
+  values.resize(keys.size());
+  statuses.resize(keys.size());
+
+  ReadOptions ro;
+  ro.async_io = true;
+  ro.optimize_multiget_for_io = GetParam();
+  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
+                     keys.data(), values.data(), statuses.data());
+  ASSERT_EQ(values.size(), keys.size());
+  ASSERT_EQ(statuses[0], Status::NotFound());
+  ASSERT_EQ(statuses[1], Status::OK());
+  ASSERT_EQ(values[1], "val_l1_" + std::to_string(144));
+  ASSERT_EQ(statuses[2], Status::NotFound());
+
+  // Bloom filters in L0/L1 will avoid the coroutine calls in those levels
+  ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 3);
+}
+
+INSTANTIATE_TEST_CASE_P(DBMultiGetAsyncIOTest, DBMultiGetAsyncIOTest,
+                        testing::Bool());
 #endif  // USE_COROUTINES
 
 TEST_F(DBBasicTest, MultiGetStats) {
@@ -2618,13 +2730,20 @@ TEST_F(DBBasicTest, GetAllKeyVersions) {
     ASSERT_OK(Delete(std::to_string(i)));
   }
   std::vector<KeyVersion> key_versions;
-  ASSERT_OK(ROCKSDB_NAMESPACE::GetAllKeyVersions(
-      db_, Slice(), Slice(), std::numeric_limits<size_t>::max(),
-      &key_versions));
+  ASSERT_OK(GetAllKeyVersions(db_, Slice(), Slice(),
+                              std::numeric_limits<size_t>::max(),
+                              &key_versions));
   ASSERT_EQ(kNumInserts + kNumDeletes + kNumUpdates, key_versions.size());
-  ASSERT_OK(ROCKSDB_NAMESPACE::GetAllKeyVersions(
-      db_, handles_[0], Slice(), Slice(), std::numeric_limits<size_t>::max(),
-      &key_versions));
+  for (size_t i = 0; i < kNumInserts + kNumDeletes + kNumUpdates; i++) {
+    if (i % 3 == 0) {
+      ASSERT_EQ(key_versions[i].GetTypeName(), "TypeDeletion");
+    } else {
+      ASSERT_EQ(key_versions[i].GetTypeName(), "TypeValue");
+    }
+  }
+  ASSERT_OK(GetAllKeyVersions(db_, handles_[0], Slice(), Slice(),
+                              std::numeric_limits<size_t>::max(),
+                              &key_versions));
   ASSERT_EQ(kNumInserts + kNumDeletes + kNumUpdates, key_versions.size());
 
   // Check non-default column family
@@ -2637,11 +2756,21 @@ TEST_F(DBBasicTest, GetAllKeyVersions) {
   for (size_t i = 0; i + 1 != kNumDeletes; ++i) {
     ASSERT_OK(Delete(1, std::to_string(i)));
   }
-  ASSERT_OK(ROCKSDB_NAMESPACE::GetAllKeyVersions(
-      db_, handles_[1], Slice(), Slice(), std::numeric_limits<size_t>::max(),
-      &key_versions));
+  ASSERT_OK(GetAllKeyVersions(db_, handles_[1], Slice(), Slice(),
+                              std::numeric_limits<size_t>::max(),
+                              &key_versions));
   ASSERT_EQ(kNumInserts + kNumDeletes + kNumUpdates - 3, key_versions.size());
 }
+
+TEST_F(DBBasicTest, ValueTypeString) {
+  KeyVersion key_version;
+  // when adding new type, please also update `value_type_string_map`
+  for (unsigned char i = ValueType::kTypeDeletion; i < ValueType::kTypeMaxValid;
+       i++) {
+    key_version.type = i;
+    ASSERT_TRUE(key_version.GetTypeName() != "Invalid");
+  }
+}
 #endif  // !ROCKSDB_LITE
 
 TEST_F(DBBasicTest, MultiGetIOBufferOverrun) {

db/db_block_cache_test.cc

@@ -819,8 +819,8 @@ class MockCache : public LRUCache {
 
   MockCache()
       : LRUCache((size_t)1 << 25 /*capacity*/, 0 /*num_shard_bits*/,
-                 false /*strict_capacity_limit*/, 0.0 /*high_pri_pool_ratio*/) {
-  }
+                 false /*strict_capacity_limit*/, 0.0 /*high_pri_pool_ratio*/,
+                 0.0 /*low_pri_pool_ratio*/) {}
 
   using ShardedCache::Insert;
 
@@ -1294,10 +1294,12 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) {
   const size_t capacity = size_t{1} << 25;
   int iterations_tested = 0;
   for (bool partition : {false, true}) {
-    for (std::shared_ptr<Cache> cache : {NewLRUCache(capacity)}) {
-      // This test doesn't support FastLRUCache nor ClockCache because the
-      // keys used are not 16B long.
-      // TODO(guido) Add support for FastLRUCache and ClockCache.
+    for (std::shared_ptr<Cache> cache :
+         {NewLRUCache(capacity),
+          ExperimentalNewClockCache(capacity, 1 /*estimated_value_size*/,
+                                    -1 /*num_shard_bits*/,
+                                    false /*strict_capacity_limit*/,
+                                    kDefaultCacheMetadataChargePolicy)}) {
       if (!cache) {
         // Skip clock cache when not supported
         continue;
@@ -1450,9 +1452,16 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) {
       // even if the cache is full
       ClearCache(cache.get());
       Cache::Handle* h = nullptr;
-      ASSERT_OK(cache->Insert("Fill-it-up", nullptr, capacity + 1,
-                              GetNoopDeleterForRole<CacheEntryRole::kMisc>(),
-                              &h, Cache::Priority::HIGH));
+      if (strcmp(cache->Name(), "LRUCache") == 0) {
+        ASSERT_OK(cache->Insert("Fill-it-up", nullptr, capacity + 1,
+                                GetNoopDeleterForRole<CacheEntryRole::kMisc>(),
+                                &h, Cache::Priority::HIGH));
+      } else {
+        // For ClockCache we use a 16-byte key.
+        ASSERT_OK(cache->Insert("Fill-it-up-xxxxx", nullptr, capacity + 1,
+                                GetNoopDeleterForRole<CacheEntryRole::kMisc>(),
+                                &h, Cache::Priority::HIGH));
+      }
       ASSERT_GT(cache->GetUsage(), cache->GetCapacity());
       expected = {};
       // For CacheEntryStatsCollector
@@ -1709,7 +1718,7 @@ TEST_P(DBBlockCacheKeyTest, StableCacheKeys) {
   delete metadata_ptr_;
   metadata_ptr_ = nullptr;
 
-  DestroyDB(export_files_dir, options);
+  ASSERT_OK(DestroyDB(export_files_dir, options));
 
   ReopenWithColumnFamilies({"default", "yoyo"}, options);
 
@@ -1727,51 +1736,60 @@ TEST_P(DBBlockCacheKeyTest, StableCacheKeys) {
 
 class CacheKeyTest : public testing::Test {
  public:
-  void SetupStableBase() {
-    // Like SemiStructuredUniqueIdGen::GenerateNext
-    tp_.db_session_id = EncodeSessionId(base_session_upper_,
-                                        base_session_lower_ ^ session_counter_);
-    tp_.db_id = std::to_string(db_id_);
-    tp_.orig_file_number = file_number_;
-    bool is_stable;
-    std::string cur_session_id = "";  // ignored
-    uint64_t cur_file_number = 42;    // ignored
-    BlockBasedTable::SetupBaseCacheKey(&tp_, cur_session_id, cur_file_number,
-                                       file_size_, &base_cache_key_,
-                                       &is_stable);
-    ASSERT_TRUE(is_stable);
+  CacheKey GetBaseCacheKey() {
+    CacheKey rv = GetOffsetableCacheKey(0, /*min file_number*/ 1).WithOffset(0);
+    // Correct for file_number_ == 1
+    *reinterpret_cast<uint64_t*>(&rv) ^= ReverseBits(uint64_t{1});
+    return rv;
   }
-  CacheKey WithOffset(uint64_t offset) {
-    return BlockBasedTable::GetCacheKey(base_cache_key_,
-                                        BlockHandle(offset, /*size*/ 5));
+  CacheKey GetCacheKey(uint64_t session_counter, uint64_t file_number,
+                       uint64_t offset) {
+    OffsetableCacheKey offsetable =
+        GetOffsetableCacheKey(session_counter, file_number);
+    // * 4 to counteract optimization that strips lower 2 bits in encoding
+    // the offset in BlockBasedTable::GetCacheKey (which we prefer to include
+    // in unit tests to maximize functional coverage).
+    EXPECT_GE(offset * 4, offset);  // no overflow
+    return BlockBasedTable::GetCacheKey(offsetable,
+                                        BlockHandle(offset * 4, /*size*/ 5));
   }
 
  protected:
-  OffsetableCacheKey base_cache_key_;
+  OffsetableCacheKey GetOffsetableCacheKey(uint64_t session_counter,
+                                           uint64_t file_number) {
+    // Like SemiStructuredUniqueIdGen::GenerateNext
+    tp_.db_session_id = EncodeSessionId(base_session_upper_,
+                                        base_session_lower_ ^ session_counter);
+    tp_.db_id = std::to_string(db_id_);
+    tp_.orig_file_number = file_number;
+    bool is_stable;
+    std::string cur_session_id = "";  // ignored
+    uint64_t cur_file_number = 42;    // ignored
+    OffsetableCacheKey rv;
+    BlockBasedTable::SetupBaseCacheKey(&tp_, cur_session_id, cur_file_number,
+                                       &rv, &is_stable);
+    EXPECT_TRUE(is_stable);
+    EXPECT_TRUE(!rv.IsEmpty());
+    // BEGIN some assertions in relation to SST unique IDs
+    std::string external_unique_id_str;
+    EXPECT_OK(GetUniqueIdFromTableProperties(tp_, &external_unique_id_str));
+    UniqueId64x2 sst_unique_id = {};
+    EXPECT_OK(DecodeUniqueIdBytes(external_unique_id_str, &sst_unique_id));
+    ExternalUniqueIdToInternal(&sst_unique_id);
+    OffsetableCacheKey ock =
+        OffsetableCacheKey::FromInternalUniqueId(&sst_unique_id);
+    EXPECT_EQ(rv.WithOffset(0).AsSlice(), ock.WithOffset(0).AsSlice());
+    EXPECT_EQ(ock.ToInternalUniqueId(), sst_unique_id);
+    // END some assertions in relation to SST unique IDs
+    return rv;
+  }
+
   TableProperties tp_;
-  uint64_t file_size_ = 0;
   uint64_t base_session_upper_ = 0;
   uint64_t base_session_lower_ = 0;
-  uint64_t session_counter_ = 0;
-  uint64_t file_number_ = 0;
   uint64_t db_id_ = 0;
 };
 
-namespace {
-template <typename T>
-int CountBitsDifferent(const T& t1, const T& t2) {
-  int diff = 0;
-  const uint8_t* p1 = reinterpret_cast<const uint8_t*>(&t1);
-  const uint8_t* p2 = reinterpret_cast<const uint8_t*>(&t2);
-  static_assert(sizeof(*p1) == 1, "Expecting uint8_t byte");
-  for (size_t i = 0; i < sizeof(T); ++i) {
-    diff += BitsSetToOne(p1[i] ^ p2[i]);
-  }
-  return diff;
-}
-
-}  // namespace
-
 TEST_F(CacheKeyTest, DBImplSessionIdStructure) {
   // We have to generate our own session IDs for simulation purposes in other
   // tests. Here we verify that the DBImpl implementation seems to match
@@ -1790,171 +1808,202 @@ TEST_F(CacheKeyTest, DBImplSessionIdStructure) {
   ASSERT_NE(Lower32of64(lower1), Lower32of64(lower2));
 }
 
-TEST_F(CacheKeyTest, StandardEncodingLimit) {
-  base_session_upper_ = 1234;
-  base_session_lower_ = 5678;
-  session_counter_ = 42;
-  file_number_ = 42;
-  db_id_ = 1234;
-
-  file_size_ = 42;
-  SetupStableBase();
-  CacheKey ck1;
-  ASSERT_TRUE(ck1.IsEmpty());
-  ck1 = WithOffset(0);
-  ASSERT_FALSE(ck1.IsEmpty());
-
-  // Should use same encoding
-  file_size_ = BlockBasedTable::kMaxFileSizeStandardEncoding;
-  SetupStableBase();
-  CacheKey ck2 = WithOffset(0);
-  ASSERT_EQ(CountBitsDifferent(ck1, ck2), 0);
-
-  // Should use different encoding
-  ++file_size_;
-  SetupStableBase();
-  CacheKey ck3 = WithOffset(0);
-  ASSERT_GT(CountBitsDifferent(ck2, ck3), 0);
+namespace {
+// Deconstruct cache key, based on knowledge of implementation details.
+void DeconstructNonemptyCacheKey(const CacheKey& key, uint64_t* file_num_etc64,
+                                 uint64_t* offset_etc64) {
+  *file_num_etc64 = *reinterpret_cast<const uint64_t*>(key.AsSlice().data());
+  *offset_etc64 = *reinterpret_cast<const uint64_t*>(key.AsSlice().data() + 8);
+  assert(*file_num_etc64 != 0);
+  if (*offset_etc64 == 0) {
+    std::swap(*file_num_etc64, *offset_etc64);
+  }
+  assert(*offset_etc64 != 0);
 }
 
+// Make a bit mask of 0 to 64 bits
+uint64_t MakeMask64(int bits) {
+  if (bits >= 64) {
+    return uint64_t{0} - 1;
+  } else {
+    return (uint64_t{1} << bits) - 1;
+  }
+}
+
+// See CacheKeyTest::Encodings
+struct CacheKeyDecoder {
+  // Inputs
+  uint64_t base_file_num_etc64, base_offset_etc64;
+  int session_counter_bits, file_number_bits, offset_bits;
+
+  // Derived
+  uint64_t session_counter_mask, file_number_mask, offset_mask;
+
+  // Outputs
+  uint64_t decoded_session_counter, decoded_file_num, decoded_offset;
+
+  void SetBaseCacheKey(const CacheKey& base) {
+    DeconstructNonemptyCacheKey(base, &base_file_num_etc64, &base_offset_etc64);
+  }
+
+  void SetRanges(int _session_counter_bits, int _file_number_bits,
+                 int _offset_bits) {
+    session_counter_bits = _session_counter_bits;
+    session_counter_mask = MakeMask64(session_counter_bits);
+    file_number_bits = _file_number_bits;
+    file_number_mask = MakeMask64(file_number_bits);
+    offset_bits = _offset_bits;
+    offset_mask = MakeMask64(offset_bits);
+  }
+
+  void Decode(const CacheKey& key) {
+    uint64_t file_num_etc64, offset_etc64;
+    DeconstructNonemptyCacheKey(key, &file_num_etc64, &offset_etc64);
+
+    // First decode session counter
+    if (offset_bits + session_counter_bits <= 64) {
+      // fully recoverable from offset_etc64
+      decoded_session_counter =
+          ReverseBits((offset_etc64 ^ base_offset_etc64)) &
+          session_counter_mask;
+    } else if (file_number_bits + session_counter_bits <= 64) {
+      // fully recoverable from file_num_etc64
+      decoded_session_counter = DownwardInvolution(
+          (file_num_etc64 ^ base_file_num_etc64) & session_counter_mask);
+    } else {
+      // Need to combine parts from each word.
+      // Piece1 will contain some correct prefix of the bottom bits of
+      // session counter.
+      uint64_t piece1 =
+          ReverseBits((offset_etc64 ^ base_offset_etc64) & ~offset_mask);
+      int piece1_bits = 64 - offset_bits;
+      // Piece2 will contain involuded bits that we can combine with piece1
+      // to infer rest of session counter
+      int piece2_bits = std::min(64 - file_number_bits, 64 - piece1_bits);
+      ASSERT_LT(piece2_bits, 64);
+      uint64_t piece2_mask = MakeMask64(piece2_bits);
+      uint64_t piece2 = (file_num_etc64 ^ base_file_num_etc64) & piece2_mask;
+
+      // Cancel out the part of piece2 that we can infer from piece1
+      // (DownwardInvolution distributes over xor)
+      piece2 ^= DownwardInvolution(piece1) & piece2_mask;
+
+      // Now we need to solve for the unknown original bits in higher
+      // positions than piece1 provides. We use Gaussian elimination
+      // because we know that a piece2_bits X piece2_bits submatrix of
+      // the matrix underlying DownwardInvolution times the vector of
+      // unknown original bits equals piece2.
+      //
+      // Build an augmented row matrix for that submatrix, built column by
+      // column.
+      std::array<uint64_t, 64> aug_rows{};
+      for (int i = 0; i < piece2_bits; ++i) {  // over columns
+        uint64_t col_i = DownwardInvolution(uint64_t{1} << piece1_bits << i);
+        ASSERT_NE(col_i & 1U, 0);
+        for (int j = 0; j < piece2_bits; ++j) {  // over rows
+          aug_rows[j] |= (col_i & 1U) << i;
+          col_i >>= 1;
+        }
+      }
+      // Augment with right hand side
+      for (int j = 0; j < piece2_bits; ++j) {  // over rows
+        aug_rows[j] |= (piece2 & 1U) << piece2_bits;
+        piece2 >>= 1;
+      }
+      // Run Gaussian elimination
+      for (int i = 0; i < piece2_bits; ++i) {  // over columns
+        // Find a row that can be used to cancel others
+        uint64_t canceller = 0;
+        // Note: Rows 0 through i-1 contain 1s in columns already eliminated
+        for (int j = i; j < piece2_bits; ++j) {  // over rows
+          if (aug_rows[j] & (uint64_t{1} << i)) {
+            // Swap into appropriate row
+            std::swap(aug_rows[i], aug_rows[j]);
+            // Keep a handy copy for row reductions
+            canceller = aug_rows[i];
+            break;
+          }
+        }
+        ASSERT_NE(canceller, 0);
+        for (int j = 0; j < piece2_bits; ++j) {  // over rows
+          if (i != j && ((aug_rows[j] >> i) & 1) != 0) {
+            // Row reduction
+            aug_rows[j] ^= canceller;
+          }
+        }
+      }
+      // Extract result
+      decoded_session_counter = piece1;
+      for (int j = 0; j < piece2_bits; ++j) {  // over rows
+        ASSERT_EQ(aug_rows[j] & piece2_mask, uint64_t{1} << j);
+        decoded_session_counter |= aug_rows[j] >> piece2_bits << piece1_bits
+                                                              << j;
+      }
+    }
+
+    decoded_offset =
+        offset_etc64 ^ base_offset_etc64 ^ ReverseBits(decoded_session_counter);
+
+    decoded_file_num = ReverseBits(file_num_etc64 ^ base_file_num_etc64 ^
+                                   DownwardInvolution(decoded_session_counter));
+  }
+};
+}  // namespace
+
 TEST_F(CacheKeyTest, Encodings) {
-  // Claim from cache_key.cc:
-  // In fact, if our SST files are all < 4TB (see
-  // BlockBasedTable::kMaxFileSizeStandardEncoding), then SST files generated
-  // in a single process are guaranteed to have unique cache keys, unless/until
-  // number session ids * max file number = 2**86, e.g. 1 trillion DB::Open in
-  // a single process and 64 trillion files generated.
+  // This test primarily verifies this claim from cache_key.cc:
+  // // In fact, if DB ids were not involved, we would be guaranteed unique
+  // // cache keys for files generated in a single process until total bits for
+  // // biggest session_id_counter, orig_file_number, and offset_in_file
+  // // reach 128 bits.
+  //
+  // To demonstrate this, CacheKeyDecoder can reconstruct the structured inputs
+  // to the cache key when provided an output cache key, the unstructured
+  // inputs, and bounds on the structured inputs.
+  //
+  // See OffsetableCacheKey comments in cache_key.cc.
 
-  // We can generalize that. For
-  // * z bits in maximum file size
-  // * n bits in maximum file number
-  // * s bits in maximum session counter
-  // uniqueness is guaranteed at least when all of these hold:
-  // *  z + n + s <= 121  (128 - 2 meta + 2 offset trim - (8-1) byte granularity
-  //                       in encoding)
-  // *  n + s <= 86       (encoding limitation)
-  // *  s <= 62           (because of 2-bit metadata)
-
-  // We can verify this indirectly by how input bits get into the cache key,
-  // but we have to be mindful that for sufficiently large file sizes,
-  // different encodings might be used. But for cases mixing large and small
-  // files, we have to verify uniqueness between encodings.
-
-  // Going through all combinations would be a little expensive, so we test
-  // only one random "stripe" of the configuration space per run.
-  constexpr uint32_t kStripeBits = 8;
-  constexpr uint32_t kStripeMask = (uint32_t{1} << kStripeBits) - 1;
-
-  // Also cycle through stripes on repeated runs (not thread safe)
-  static uint32_t stripe =
-      static_cast<uint32_t>(std::random_device{}()) & kStripeMask;
-  stripe = (stripe + 1) & kStripeMask;
-
-  fprintf(stderr, "%u\n", stripe);
-
-  // We are going to randomly initialize some values which *should* not affect
+  // We are going to randomly initialize some values that *should* not affect
   // result
   Random64 r{std::random_device{}()};
 
-  int max_num_encodings = 0;
-  uint32_t config_num = 0;
-  uint32_t session_counter_bits, file_number_bits, max_file_size_bits;
+  CacheKeyDecoder decoder;
+  db_id_ = r.Next();
+  base_session_upper_ = r.Next();
+  base_session_lower_ = r.Next();
+  if (base_session_lower_ == 0) {
+    base_session_lower_ = 1;
+  }
 
-  // Inner loop body, used later in a loop over configurations
-  auto TestConfig = [&]() {
-    base_session_upper_ = r.Next();
-    base_session_lower_ = r.Next();
-    session_counter_ = r.Next();
-    if (session_counter_bits < 64) {
-      // Avoid shifting UB
-      session_counter_ = session_counter_ >> 1 >> (63 - session_counter_bits);
-    }
-    file_number_ = r.Next() >> (64 - file_number_bits);
-    // Need two bits set to avoid temporary zero below
-    if (BitsSetToOne(file_number_) < 2) {
-      file_number_ = 3;
-    }
-    db_id_ = r.Next();
+  decoder.SetBaseCacheKey(GetBaseCacheKey());
 
-    // Work-around clang-analyzer which thinks empty last_base is garbage
-    CacheKey last_base = CacheKey::CreateUniqueForProcessLifetime();
-
-    std::unordered_set<std::string> seen;
-    int num_encodings = 0;
-
-    // Loop over encodings by increasing file size bits
-    for (uint32_t file_size_bits = 1; file_size_bits <= max_file_size_bits;
-         ++file_size_bits) {
-      file_size_ = uint64_t{1} << (file_size_bits - 1);
-      SetupStableBase();
-      CacheKey new_base = WithOffset(0);
-      if (CountBitsDifferent(last_base, new_base) == 0) {
-        // Same as previous encoding
-        continue;
-      }
-
-      // New encoding
-      ++num_encodings;
-      ASSERT_TRUE(seen.insert(new_base.AsSlice().ToString()).second);
-      last_base = new_base;
-      for (uint32_t i = 0; i < file_size_bits; ++i) {
-        CacheKey ck = WithOffset(uint64_t{1} << i);
-        if (i < 2) {
-          // These cases are not relevant and optimized by dropping two
-          // lowest bits because there's always at least 5 bytes between
-          // blocks.
-          ASSERT_EQ(CountBitsDifferent(ck, new_base), 0);
-        } else {
-          // Normal case
-          // 1 bit different from base and never been seen implies the bit
-          // is encoded into cache key without overlapping other structured
-          // data.
-          ASSERT_EQ(CountBitsDifferent(ck, new_base), 1);
-          ASSERT_TRUE(seen.insert(ck.AsSlice().ToString()).second);
-        }
-      }
-      for (uint32_t i = 0; i < session_counter_bits; ++i) {
-        SaveAndRestore<uint64_t> tmp(&session_counter_,
-                                     session_counter_ ^ (uint64_t{1} << i));
-        SetupStableBase();
-        CacheKey ck = WithOffset(0);
-        ASSERT_EQ(CountBitsDifferent(ck, new_base), 1);
-        ASSERT_TRUE(seen.insert(ck.AsSlice().ToString()).second);
-      }
-      for (uint32_t i = 0; i < file_number_bits; ++i) {
-        SaveAndRestore<uint64_t> tmp(&file_number_,
-                                     file_number_ ^ (uint64_t{1} << i));
-        SetupStableBase();
-        CacheKey ck = WithOffset(0);
-        ASSERT_EQ(CountBitsDifferent(ck, new_base), 1);
-        ASSERT_TRUE(seen.insert(ck.AsSlice().ToString()).second);
-      }
-      max_num_encodings = std::max(max_num_encodings, num_encodings);
-    }
-  };
-
-  // Loop over configurations and test those in stripe
-  for (session_counter_bits = 0; session_counter_bits <= 62;
+  // Loop over configurations and test those
+  for (int session_counter_bits = 0; session_counter_bits <= 64;
        ++session_counter_bits) {
-    uint32_t max_file_number_bits =
-        std::min(uint32_t{64}, uint32_t{86} - session_counter_bits);
-    // Start with 2 to avoid file_number_ == 0 in testing
-    for (file_number_bits = 2; file_number_bits <= max_file_number_bits;
-         ++file_number_bits) {
-      uint32_t max_max_file_size_bits =
-          std::min(uint32_t{64},
-                   uint32_t{121} - file_number_bits - session_counter_bits);
-      for (max_file_size_bits = 1; max_file_size_bits <= max_max_file_size_bits;
-           ++max_file_size_bits) {
-        if ((config_num++ & kStripeMask) == stripe) {
-          TestConfig();
+    for (int file_number_bits = 1; file_number_bits <= 64; ++file_number_bits) {
+      // 62 bits max because unoptimized offset will be 64 bits in that case
+      for (int offset_bits = 0; offset_bits <= 62; ++offset_bits) {
+        if (session_counter_bits + file_number_bits + offset_bits > 128) {
+          break;
         }
+
+        decoder.SetRanges(session_counter_bits, file_number_bits, offset_bits);
+
+        uint64_t session_counter = r.Next() & decoder.session_counter_mask;
+        uint64_t file_number = r.Next() & decoder.file_number_mask;
+        if (file_number == 0) {
+          // Minimum
+          file_number = 1;
+        }
+        uint64_t offset = r.Next() & decoder.offset_mask;
+        decoder.Decode(GetCacheKey(session_counter, file_number, offset));
+
+        EXPECT_EQ(decoder.decoded_session_counter, session_counter);
+        EXPECT_EQ(decoder.decoded_file_num, file_number);
+        EXPECT_EQ(decoder.decoded_offset, offset);
       }
     }
   }
-
-  // Make sure the current implementation is exercised
-  ASSERT_EQ(max_num_encodings, 4);
 }
 
 INSTANTIATE_TEST_CASE_P(DBBlockCacheKeyTest, DBBlockCacheKeyTest,

db/db_compaction_test.cc

@@ -78,17 +78,6 @@ class DBCompactionTest : public DBTestBase {
       : DBTestBase("db_compaction_test", /*env_do_fsync=*/true) {}
 
  protected:
-#ifndef ROCKSDB_LITE
-  uint64_t GetSstSizeHelper(Temperature temperature) {
-    std::string prop;
-    EXPECT_TRUE(dbfull()->GetProperty(
-        DB::Properties::kLiveSstFilesSizeAtTemperature +
-            std::to_string(static_cast<uint8_t>(temperature)),
-        &prop));
-    return static_cast<uint64_t>(std::atoi(prop.c_str()));
-  }
-#endif  // ROCKSDB_LITE
-
   /*
    * Verifies compaction stats of cfd are valid.
    *
@@ -172,8 +161,32 @@ class ChangeLevelConflictsWithAuto
   ChangeLevelConflictsWithAuto() : DBCompactionTest() {}
 };
 
-namespace {
+// Param = true: grab the compaction pressure token (enable
+// parallel compactions)
+// Param = false: Not grab the token (no parallel compactions)
+class RoundRobinSubcompactionsAgainstPressureToken
+    : public DBCompactionTest,
+      public ::testing::WithParamInterface<bool> {
+ public:
+  RoundRobinSubcompactionsAgainstPressureToken() {
+    grab_pressure_token_ = GetParam();
+  }
+  bool grab_pressure_token_;
+};
 
+class RoundRobinSubcompactionsAgainstResources
+    : public DBCompactionTest,
+      public ::testing::WithParamInterface<std::tuple<int, int>> {
+ public:
+  RoundRobinSubcompactionsAgainstResources() {
+    total_low_pri_threads_ = std::get<0>(GetParam());
+    max_compaction_limits_ = std::get<1>(GetParam());
+  }
+  int total_low_pri_threads_;
+  int max_compaction_limits_;
+};
+
+namespace {
 class FlushedFileCollector : public EventListener {
  public:
   FlushedFileCollector() {}
@@ -2647,8 +2660,9 @@ TEST_F(DBCompactionTest, ManualAutoRace) {
   TEST_SYNC_POINT("DBCompactionTest::ManualAutoRace:1");
   // The auto compaction will wait until the manual compaction is registerd
   // before processing so that it will be cancelled.
-  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                                   nullptr));
+  CompactRangeOptions cro;
+  cro.exclusive_manual_compaction = true;
+  ASSERT_OK(dbfull()->CompactRange(cro, handles_[1], nullptr, nullptr));
   ASSERT_EQ("0,1", FilesPerLevel(1));
 
   // Eventually the cancelled compaction will be rescheduled and executed.
@@ -5317,6 +5331,255 @@ TEST_F(DBCompactionTest, PersistRoundRobinCompactCursor) {
   }
 }
 
+TEST_P(RoundRobinSubcompactionsAgainstPressureToken, PressureTokenTest) {
+  const int kKeysPerBuffer = 100;
+  Options options = CurrentOptions();
+  options.num_levels = 4;
+  options.max_bytes_for_level_multiplier = 2;
+  options.level0_file_num_compaction_trigger = 4;
+  options.target_file_size_base = kKeysPerBuffer * 1024;
+  options.compaction_pri = CompactionPri::kRoundRobin;
+  options.max_bytes_for_level_base = 8 * kKeysPerBuffer * 1024;
+  options.disable_auto_compactions = true;
+  // Setup 7 threads but limited subcompactions so that
+  // RoundRobin requires extra compactions from reserved threads
+  options.max_subcompactions = 1;
+  options.max_background_compactions = 7;
+  options.max_compaction_bytes = 100000000;
+  DestroyAndReopen(options);
+  env_->SetBackgroundThreads(7, Env::LOW);
+
+  Random rnd(301);
+  const std::vector<int> files_per_level = {0, 15, 25};
+  for (int lvl = 2; lvl > 0; lvl--) {
+    for (int i = 0; i < files_per_level[lvl]; i++) {
+      for (int j = 0; j < kKeysPerBuffer; j++) {
+        // Add (lvl-1) to ensure nearly equivallent number of files
+        // in L2 are overlapped with fils selected to compact from
+        // L1
+        ASSERT_OK(Put(Key(2 * i * kKeysPerBuffer + 2 * j + (lvl - 1)),
+                      rnd.RandomString(1010)));
+      }
+      ASSERT_OK(Flush());
+    }
+    MoveFilesToLevel(lvl);
+    ASSERT_EQ(files_per_level[lvl], NumTableFilesAtLevel(lvl, 0));
+  }
+  // 15 files in L1; 25 files in L2
+
+  // This is a variable for making sure the following callback is called
+  // and the assertions in it are indeed excuted.
+  bool num_planned_subcompactions_verified = false;
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::GenSubcompactionBoundaries:0", [&](void* arg) {
+        uint64_t num_planned_subcompactions = *(static_cast<uint64_t*>(arg));
+        if (grab_pressure_token_) {
+          // 7 files are selected for round-robin under auto
+          // compaction. The number of planned subcompaction is restricted by
+          // the limited number of max_background_compactions
+          ASSERT_EQ(num_planned_subcompactions, 7);
+        } else {
+          ASSERT_EQ(num_planned_subcompactions, 1);
+        }
+        num_planned_subcompactions_verified = true;
+      });
+
+  // The following 3 dependencies have to be added to ensure the auto
+  // compaction and the pressure token is correctly enabled. Same for
+  // RoundRobinSubcompactionsUsingResources and
+  // DBCompactionTest.RoundRobinSubcompactionsShrinkResources
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"RoundRobinSubcompactionsAgainstPressureToken:0",
+        "BackgroundCallCompaction:0"},
+       {"CompactionJob::AcquireSubcompactionResources:0",
+        "RoundRobinSubcompactionsAgainstPressureToken:1"},
+       {"RoundRobinSubcompactionsAgainstPressureToken:2",
+        "CompactionJob::AcquireSubcompactionResources:1"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(dbfull()->EnableAutoCompaction({dbfull()->DefaultColumnFamily()}));
+  TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstPressureToken:0");
+  TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstPressureToken:1");
+  std::unique_ptr<WriteControllerToken> pressure_token;
+  if (grab_pressure_token_) {
+    pressure_token =
+        dbfull()->TEST_write_controler().GetCompactionPressureToken();
+  }
+  TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstPressureToken:2");
+
+  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_TRUE(num_planned_subcompactions_verified);
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
+INSTANTIATE_TEST_CASE_P(RoundRobinSubcompactionsAgainstPressureToken,
+                        RoundRobinSubcompactionsAgainstPressureToken,
+                        testing::Bool());
+
+TEST_P(RoundRobinSubcompactionsAgainstResources, SubcompactionsUsingResources) {
+  const int kKeysPerBuffer = 200;
+  Options options = CurrentOptions();
+  options.num_levels = 4;
+  options.level0_file_num_compaction_trigger = 3;
+  options.target_file_size_base = kKeysPerBuffer * 1024;
+  options.compaction_pri = CompactionPri::kRoundRobin;
+  options.max_bytes_for_level_base = 30 * kKeysPerBuffer * 1024;
+  options.disable_auto_compactions = true;
+  options.max_subcompactions = 1;
+  options.max_background_compactions = max_compaction_limits_;
+  // Set a large number for max_compaction_bytes so that one round-robin
+  // compaction is enough to make post-compaction L1 size less than
+  // the maximum size (this test assumes only one round-robin compaction
+  // is triggered by kLevelMaxLevelSize)
+  options.max_compaction_bytes = 100000000;
+
+  DestroyAndReopen(options);
+  env_->SetBackgroundThreads(total_low_pri_threads_, Env::LOW);
+
+  Random rnd(301);
+  const std::vector<int> files_per_level = {0, 40, 100};
+  for (int lvl = 2; lvl > 0; lvl--) {
+    for (int i = 0; i < files_per_level[lvl]; i++) {
+      for (int j = 0; j < kKeysPerBuffer; j++) {
+        // Add (lvl-1) to ensure nearly equivallent number of files
+        // in L2 are overlapped with fils selected to compact from
+        // L1
+        ASSERT_OK(Put(Key(2 * i * kKeysPerBuffer + 2 * j + (lvl - 1)),
+                      rnd.RandomString(1010)));
+      }
+      ASSERT_OK(Flush());
+    }
+    MoveFilesToLevel(lvl);
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_EQ(files_per_level[lvl], NumTableFilesAtLevel(lvl, 0));
+  }
+
+  // 40 files in L1; 100 files in L2
+  // This is a variable for making sure the following callback is called
+  // and the assertions in it are indeed excuted.
+  bool num_planned_subcompactions_verified = false;
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::GenSubcompactionBoundaries:0", [&](void* arg) {
+        uint64_t num_planned_subcompactions = *(static_cast<uint64_t*>(arg));
+        // More than 10 files are selected for round-robin under auto
+        // compaction. The number of planned subcompaction is restricted by
+        // the minimum number between available threads and compaction limits
+        ASSERT_EQ(num_planned_subcompactions - options.max_subcompactions,
+                  std::min(total_low_pri_threads_, max_compaction_limits_) - 1);
+        num_planned_subcompactions_verified = true;
+      });
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"RoundRobinSubcompactionsAgainstResources:0",
+        "BackgroundCallCompaction:0"},
+       {"CompactionJob::AcquireSubcompactionResources:0",
+        "RoundRobinSubcompactionsAgainstResources:1"},
+       {"RoundRobinSubcompactionsAgainstResources:2",
+        "CompactionJob::AcquireSubcompactionResources:1"},
+       {"CompactionJob::ReleaseSubcompactionResources:0",
+        "RoundRobinSubcompactionsAgainstResources:3"},
+       {"RoundRobinSubcompactionsAgainstResources:4",
+        "CompactionJob::ReleaseSubcompactionResources:1"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->EnableAutoCompaction({dbfull()->DefaultColumnFamily()}));
+  TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstResources:0");
+  TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstResources:1");
+  auto pressure_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
+
+  TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstResources:2");
+  TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstResources:3");
+  // We can reserve more threads now except one is being used
+  ASSERT_EQ(total_low_pri_threads_ - 1,
+            env_->ReserveThreads(total_low_pri_threads_, Env::Priority::LOW));
+  ASSERT_EQ(
+      total_low_pri_threads_ - 1,
+      env_->ReleaseThreads(total_low_pri_threads_ - 1, Env::Priority::LOW));
+  TEST_SYNC_POINT("RoundRobinSubcompactionsAgainstResources:4");
+  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_TRUE(num_planned_subcompactions_verified);
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
+INSTANTIATE_TEST_CASE_P(RoundRobinSubcompactionsAgainstResources,
+                        RoundRobinSubcompactionsAgainstResources,
+                        ::testing::Values(std::make_tuple(1, 5),
+                                          std::make_tuple(5, 1),
+                                          std::make_tuple(10, 5),
+                                          std::make_tuple(5, 10),
+                                          std::make_tuple(10, 10)));
+
+TEST_P(DBCompactionTestWithParam, RoundRobinWithoutAdditionalResources) {
+  const int kKeysPerBuffer = 200;
+  Options options = CurrentOptions();
+  options.num_levels = 4;
+  options.level0_file_num_compaction_trigger = 3;
+  options.target_file_size_base = kKeysPerBuffer * 1024;
+  options.compaction_pri = CompactionPri::kRoundRobin;
+  options.max_bytes_for_level_base = 30 * kKeysPerBuffer * 1024;
+  options.disable_auto_compactions = true;
+  options.max_subcompactions = max_subcompactions_;
+  options.max_background_compactions = 1;
+  options.max_compaction_bytes = 100000000;
+  // Similar experiment setting as above except the max_subcompactions
+  // is given by max_subcompactions_ (1 or 4), and we fix the
+  // additional resources as (1, 1) and thus no more extra resources
+  // can be used
+  DestroyAndReopen(options);
+  env_->SetBackgroundThreads(1, Env::LOW);
+
+  Random rnd(301);
+  const std::vector<int> files_per_level = {0, 33, 100};
+  for (int lvl = 2; lvl > 0; lvl--) {
+    for (int i = 0; i < files_per_level[lvl]; i++) {
+      for (int j = 0; j < kKeysPerBuffer; j++) {
+        // Add (lvl-1) to ensure nearly equivallent number of files
+        // in L2 are overlapped with fils selected to compact from
+        // L1
+        ASSERT_OK(Put(Key(2 * i * kKeysPerBuffer + 2 * j + (lvl - 1)),
+                      rnd.RandomString(1010)));
+      }
+      ASSERT_OK(Flush());
+    }
+    MoveFilesToLevel(lvl);
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_EQ(files_per_level[lvl], NumTableFilesAtLevel(lvl, 0));
+  }
+
+  // 33 files in L1; 100 files in L2
+  // This is a variable for making sure the following callback is called
+  // and the assertions in it are indeed excuted.
+  bool num_planned_subcompactions_verified = false;
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::GenSubcompactionBoundaries:0", [&](void* arg) {
+        uint64_t num_planned_subcompactions = *(static_cast<uint64_t*>(arg));
+        // At most 4 files are selected for round-robin under auto
+        // compaction. The number of planned subcompaction is restricted by
+        // the max_subcompactions since no extra resources can be used
+        ASSERT_EQ(num_planned_subcompactions, options.max_subcompactions);
+        num_planned_subcompactions_verified = true;
+      });
+  // No need to setup dependency for pressure token since
+  // AcquireSubcompactionResources may not be called and it anyway cannot
+  // reserve any additional resources
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"DBCompactionTest::RoundRobinWithoutAdditionalResources:0",
+        "BackgroundCallCompaction:0"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_OK(dbfull()->EnableAutoCompaction({dbfull()->DefaultColumnFamily()}));
+  TEST_SYNC_POINT("DBCompactionTest::RoundRobinWithoutAdditionalResources:0");
+
+  ASSERT_OK(dbfull()->WaitForCompact());
+  ASSERT_TRUE(num_planned_subcompactions_verified);
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 TEST_F(DBCompactionTest, RoundRobinCutOutputAtCompactCursor) {
   Options options = CurrentOptions();
   options.num_levels = 3;
@@ -5670,18 +5933,10 @@ TEST_P(DBCompactionTestWithParam, FixFileIngestionCompactionDeadlock) {
     for (int j = 0; j != kNumKeysPerFile; ++j) {
       ASSERT_OK(Put(Key(j), rnd.RandomString(990)));
     }
-    if (0 == i) {
-      // When we reach here, the memtables have kNumKeysPerFile keys. Note that
-      // flush is not yet triggered. We need to write an extra key so that the
-      // write path will call PreprocessWrite and flush the previous key-value
-      // pairs to e flushed. After that, there will be the newest key in the
-      // memtable, and a bunch of L0 files. Since there is already one key in
-      // the memtable, then for i = 1, 2, ..., we do not have to write this
-      // extra key to trigger flush.
-      ASSERT_OK(Put("", ""));
+    if (i > 0) {
+      ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
+      ASSERT_EQ(NumTableFilesAtLevel(0 /*level*/, 0 /*cf*/), i);
     }
-    ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
-    ASSERT_EQ(NumTableFilesAtLevel(0 /*level*/, 0 /*cf*/), i + 1);
   }
   // When we reach this point, there will be level0_stop_writes_trigger L0
   // files and one extra key (99) in memory, which overlaps with the external

db/db_impl/compacted_db_impl.cc

@@ -72,9 +72,9 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
       user_comparator_->timestamp_size() > 0 ? timestamp : nullptr;
   LookupKey lkey(key, kMaxSequenceNumber, options.timestamp);
   GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
-                         GetContext::kNotFound, lkey.user_key(), value, ts,
-                         nullptr, nullptr, true, nullptr, nullptr, nullptr,
-                         nullptr, &read_cb);
+                         GetContext::kNotFound, lkey.user_key(), value,
+                         /*columns=*/nullptr, ts, nullptr, nullptr, true,
+                         nullptr, nullptr, nullptr, nullptr, &read_cb);
 
   const FdWithKeyRange& f = files_.files[FindFile(lkey.user_key())];
   if (user_comparator_->CompareWithoutTimestamp(
@@ -159,7 +159,7 @@ std::vector<Status> CompactedDBImpl::MultiGet(
       std::string* timestamp = timestamps ? &(*timestamps)[idx] : nullptr;
       GetContext get_context(
           user_comparator_, nullptr, nullptr, nullptr, GetContext::kNotFound,
-          lkey.user_key(), &pinnable_val,
+          lkey.user_key(), &pinnable_val, /*columns=*/nullptr,
           user_comparator_->timestamp_size() > 0 ? timestamp : nullptr, nullptr,
           nullptr, true, nullptr, nullptr, nullptr, nullptr, &read_cb);
       Status s = r->Get(options, lkey.internal_key(), &get_context, nullptr);

db/db_impl/db_impl.cc

@@ -185,7 +185,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
       log_dir_synced_(false),
       log_empty_(true),
       persist_stats_cf_handle_(nullptr),
-      log_sync_cv_(&mutex_),
+      log_sync_cv_(&log_write_mutex_),
       total_log_size_(0),
       is_snapshot_supported_(true),
       write_buffer_manager_(immutable_db_options_.write_buffer_manager.get()),
@@ -273,6 +273,8 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
   mutable_db_options_.Dump(immutable_db_options_.info_log.get());
   DumpSupportInfo(immutable_db_options_.info_log.get());
 
+  max_total_wal_size_.store(mutable_db_options_.max_total_wal_size,
+                            std::memory_order_relaxed);
   if (write_buffer_manager_) {
     wbm_stall_.reset(new WBMStallInterface());
   }
@@ -480,6 +482,7 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
 #ifndef ROCKSDB_LITE
   if (periodic_work_scheduler_ != nullptr) {
     periodic_work_scheduler_->Unregister(this);
+    periodic_work_scheduler_->UnregisterRecordSeqnoTimeWorker(this);
   }
 #endif  // !ROCKSDB_LITE
 
@@ -624,26 +627,28 @@ Status DBImpl::CloseHelper() {
     job_context.Clean();
     mutex_.Lock();
   }
-
-  for (auto l : logs_to_free_) {
-    delete l;
-  }
-  for (auto& log : logs_) {
-    uint64_t log_number = log.writer->get_log_number();
-    Status s = log.ClearWriter();
-    if (!s.ok()) {
-      ROCKS_LOG_WARN(
-          immutable_db_options_.info_log,
-          "Unable to Sync WAL file %s with error -- %s",
-          LogFileName(immutable_db_options_.GetWalDir(), log_number).c_str(),
-          s.ToString().c_str());
-      // Retain the first error
-      if (ret.ok()) {
-        ret = s;
+  {
+    InstrumentedMutexLock lock(&log_write_mutex_);
+    for (auto l : logs_to_free_) {
+      delete l;
+    }
+    for (auto& log : logs_) {
+      uint64_t log_number = log.writer->get_log_number();
+      Status s = log.ClearWriter();
+      if (!s.ok()) {
+        ROCKS_LOG_WARN(
+            immutable_db_options_.info_log,
+            "Unable to Sync WAL file %s with error -- %s",
+            LogFileName(immutable_db_options_.GetWalDir(), log_number).c_str(),
+            s.ToString().c_str());
+        // Retain the first error
+        if (ret.ok()) {
+          ret = s;
+        }
       }
     }
+    logs_.clear();
   }
-  logs_.clear();
 
   // Table cache may have table handles holding blocks from the block cache.
   // We need to release them before the block cache is destroyed. The block
@@ -791,6 +796,66 @@ Status DBImpl::StartPeriodicWorkScheduler() {
 #endif  // !ROCKSDB_LITE
 }
 
+Status DBImpl::RegisterRecordSeqnoTimeWorker() {
+#ifndef ROCKSDB_LITE
+  if (!periodic_work_scheduler_) {
+    return Status::OK();
+  }
+  uint64_t min_time_duration = std::numeric_limits<uint64_t>::max();
+  uint64_t max_time_duration = std::numeric_limits<uint64_t>::min();
+  {
+    InstrumentedMutexLock l(&mutex_);
+
+    for (auto cfd : *versions_->GetColumnFamilySet()) {
+      uint64_t preclude_last_option =
+          cfd->ioptions()->preclude_last_level_data_seconds;
+      if (!cfd->IsDropped() && preclude_last_option > 0) {
+        min_time_duration = std::min(preclude_last_option, min_time_duration);
+        max_time_duration = std::max(preclude_last_option, max_time_duration);
+      }
+    }
+    if (min_time_duration == std::numeric_limits<uint64_t>::max()) {
+      seqno_time_mapping_.Resize(0, 0);
+    } else {
+      seqno_time_mapping_.Resize(min_time_duration, max_time_duration);
+    }
+  }
+
+  uint64_t seqno_time_cadence = 0;
+  if (min_time_duration != std::numeric_limits<uint64_t>::max()) {
+    seqno_time_cadence =
+        min_time_duration / SeqnoToTimeMapping::kMaxSeqnoTimePairsPerCF;
+  }
+
+  Status s;
+  if (seqno_time_cadence != record_seqno_time_cadence_) {
+    if (seqno_time_cadence == 0) {
+      periodic_work_scheduler_->UnregisterRecordSeqnoTimeWorker(this);
+    } else {
+      s = periodic_work_scheduler_->RegisterRecordSeqnoTimeWorker(
+          this, seqno_time_cadence);
+    }
+
+    if (s.ok()) {
+      record_seqno_time_cadence_ = seqno_time_cadence;
+    }
+
+    if (s.IsNotSupported()) {
+      // TODO: Fix the timer cannot cancel and re-add the same task
+      ROCKS_LOG_WARN(
+          immutable_db_options_.info_log,
+          "Updating seqno to time worker cadence is not supported yet, to make "
+          "the change effective, please reopen the DB instance.");
+      s = Status::OK();
+    }
+  }
+
+  return s;
+#else
+  return Status::OK();
+#endif  // !ROCKSDB_LITE
+}
+
 // esitmate the total size of stats_history_
 size_t DBImpl::EstimateInMemoryStatsHistorySize() const {
   size_t size_total =
@@ -1047,6 +1112,7 @@ Status DBImpl::TablesRangeTombstoneSummary(ColumnFamilyHandle* column_family,
 }
 
 void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) {
+  mutex_.AssertHeld();
   if (!job_context->logs_to_free.empty()) {
     for (auto l : job_context->logs_to_free) {
       AddToLogsToFreeQueue(l);
@@ -1224,6 +1290,11 @@ Status DBImpl::SetDBOptions(
             new_options.stats_persist_period_sec);
         mutex_.Lock();
       }
+      if (new_options.max_total_wal_size !=
+          mutable_db_options_.max_total_wal_size) {
+        max_total_wal_size_.store(new_options.max_total_wal_size,
+                                  std::memory_order_release);
+      }
       write_controller_.set_max_delayed_write_rate(
           new_options.delayed_write_rate);
       table_cache_.get()->SetCapacity(new_options.max_open_files == -1
@@ -1344,7 +1415,7 @@ Status DBImpl::SyncWAL() {
   uint64_t current_log_number;
 
   {
-    InstrumentedMutexLock l(&mutex_);
+    InstrumentedMutexLock l(&log_write_mutex_);
     assert(!logs_.empty());
 
     // This SyncWAL() call only cares about logs up to this number.
@@ -1401,19 +1472,37 @@ Status DBImpl::SyncWAL() {
   TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2");
 
   TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:1");
+  VersionEdit synced_wals;
   {
-    InstrumentedMutexLock l(&mutex_);
+    InstrumentedMutexLock l(&log_write_mutex_);
     if (status.ok()) {
-      status = MarkLogsSynced(current_log_number, need_log_dir_sync);
+      MarkLogsSynced(current_log_number, need_log_dir_sync, &synced_wals);
     } else {
       MarkLogsNotSynced(current_log_number);
     }
   }
+  if (status.ok() && synced_wals.IsWalAddition()) {
+    InstrumentedMutexLock l(&mutex_);
+    status = ApplyWALToManifest(&synced_wals);
+  }
+
   TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:2");
 
   return status;
 }
 
+Status DBImpl::ApplyWALToManifest(VersionEdit* synced_wals) {
+  // not empty, write to MANIFEST.
+  mutex_.AssertHeld();
+  Status status =
+      versions_->LogAndApplyToDefaultColumnFamily(synced_wals, &mutex_);
+  if (!status.ok() && versions_->io_status().IsIOError()) {
+    status = error_handler_.SetBGError(versions_->io_status(),
+                                       BackgroundErrorReason::kManifestWrite);
+  }
+  return status;
+}
+
 Status DBImpl::LockWAL() {
   log_write_mutex_.Lock();
   auto cur_log_writer = logs_.back().writer;
@@ -1433,12 +1522,12 @@ Status DBImpl::UnlockWAL() {
   return Status::OK();
 }
 
-Status DBImpl::MarkLogsSynced(uint64_t up_to, bool synced_dir) {
-  mutex_.AssertHeld();
+void DBImpl::MarkLogsSynced(uint64_t up_to, bool synced_dir,
+                            VersionEdit* synced_wals) {
+  log_write_mutex_.AssertHeld();
   if (synced_dir && logfile_number_ == up_to) {
     log_dir_synced_ = true;
   }
-  VersionEdit synced_wals;
   for (auto it = logs_.begin(); it != logs_.end() && it->number <= up_to;) {
     auto& wal = *it;
     assert(wal.IsSyncing());
@@ -1446,11 +1535,9 @@ Status DBImpl::MarkLogsSynced(uint64_t up_to, bool synced_dir) {
     if (logs_.size() > 1) {
       if (immutable_db_options_.track_and_verify_wals_in_manifest &&
           wal.GetPreSyncSize() > 0) {
-        synced_wals.AddWal(wal.number, WalMetadata(wal.GetPreSyncSize()));
+        synced_wals->AddWal(wal.number, WalMetadata(wal.GetPreSyncSize()));
       }
       logs_to_free_.push_back(wal.ReleaseWriter());
-      // To modify logs_ both mutex_ and log_write_mutex_ must be held
-      InstrumentedMutexLock l(&log_write_mutex_);
       it = logs_.erase(it);
     } else {
       wal.FinishSync();
@@ -1459,22 +1546,11 @@ Status DBImpl::MarkLogsSynced(uint64_t up_to, bool synced_dir) {
   }
   assert(logs_.empty() || logs_[0].number > up_to ||
          (logs_.size() == 1 && !logs_[0].IsSyncing()));
-
-  Status s;
-  if (synced_wals.IsWalAddition()) {
-    // not empty, write to MANIFEST.
-    s = versions_->LogAndApplyToDefaultColumnFamily(&synced_wals, &mutex_);
-    if (!s.ok() && versions_->io_status().IsIOError()) {
-      s = error_handler_.SetBGError(versions_->io_status(),
-                                    BackgroundErrorReason::kManifestWrite);
-    }
-  }
   log_sync_cv_.SignalAll();
-  return s;
 }
 
 void DBImpl::MarkLogsNotSynced(uint64_t up_to) {
-  mutex_.AssertHeld();
+  log_write_mutex_.AssertHeld();
   for (auto it = logs_.begin(); it != logs_.end() && it->number <= up_to;
        ++it) {
     auto& wal = *it;
@@ -1598,9 +1674,16 @@ void DBImpl::BackgroundCallPurge() {
 }
 
 namespace {
-struct IterState {
-  IterState(DBImpl* _db, InstrumentedMutex* _mu, SuperVersion* _super_version,
-            bool _background_purge)
+
+// A `SuperVersionHandle` holds a non-null `SuperVersion*` pointing at a
+// `SuperVersion` referenced once for this object. It also contains the state
+// needed to clean up the `SuperVersion` reference from outside of `DBImpl`
+// using `CleanupSuperVersionHandle()`.
+struct SuperVersionHandle {
+  // `_super_version` must be non-nullptr and `Ref()`'d once as long as the
+  // `SuperVersionHandle` may use it.
+  SuperVersionHandle(DBImpl* _db, InstrumentedMutex* _mu,
+                     SuperVersion* _super_version, bool _background_purge)
       : db(_db),
         mu(_mu),
         super_version(_super_version),
@@ -1612,35 +1695,49 @@ struct IterState {
   bool background_purge;
 };
 
-static void CleanupIteratorState(void* arg1, void* /*arg2*/) {
-  IterState* state = reinterpret_cast<IterState*>(arg1);
+static void CleanupSuperVersionHandle(void* arg1, void* /*arg2*/) {
+  SuperVersionHandle* sv_handle = reinterpret_cast<SuperVersionHandle*>(arg1);
 
-  if (state->super_version->Unref()) {
+  if (sv_handle->super_version->Unref()) {
     // Job id == 0 means that this is not our background process, but rather
     // user thread
     JobContext job_context(0);
 
-    state->mu->Lock();
-    state->super_version->Cleanup();
-    state->db->FindObsoleteFiles(&job_context, false, true);
-    if (state->background_purge) {
-      state->db->ScheduleBgLogWriterClose(&job_context);
-      state->db->AddSuperVersionsToFreeQueue(state->super_version);
-      state->db->SchedulePurge();
+    sv_handle->mu->Lock();
+    sv_handle->super_version->Cleanup();
+    sv_handle->db->FindObsoleteFiles(&job_context, false, true);
+    if (sv_handle->background_purge) {
+      sv_handle->db->ScheduleBgLogWriterClose(&job_context);
+      sv_handle->db->AddSuperVersionsToFreeQueue(sv_handle->super_version);
+      sv_handle->db->SchedulePurge();
     }
-    state->mu->Unlock();
+    sv_handle->mu->Unlock();
 
-    if (!state->background_purge) {
-      delete state->super_version;
+    if (!sv_handle->background_purge) {
+      delete sv_handle->super_version;
     }
     if (job_context.HaveSomethingToDelete()) {
-      state->db->PurgeObsoleteFiles(job_context, state->background_purge);
+      sv_handle->db->PurgeObsoleteFiles(job_context,
+                                        sv_handle->background_purge);
     }
     job_context.Clean();
   }
 
+  delete sv_handle;
+}
+
+struct GetMergeOperandsState {
+  MergeContext merge_context;
+  PinnedIteratorsManager pinned_iters_mgr;
+  SuperVersionHandle* sv_handle;
+};
+
+static void CleanupGetMergeOperandsState(void* arg1, void* /*arg2*/) {
+  GetMergeOperandsState* state = static_cast<GetMergeOperandsState*>(arg1);
+  CleanupSuperVersionHandle(state->sv_handle /* arg1 */, nullptr /* arg2 */);
   delete state;
 }
+
 }  // namespace
 
 InternalIterator* DBImpl::NewInternalIterator(const ReadOptions& read_options,
@@ -1664,8 +1761,8 @@ InternalIterator* DBImpl::NewInternalIterator(const ReadOptions& read_options,
   std::unique_ptr<FragmentedRangeTombstoneIterator> range_del_iter;
   Status s;
   if (!read_options.ignore_range_deletions) {
-    range_del_iter.reset(
-        super_version->mem->NewRangeTombstoneIterator(read_options, sequence));
+    range_del_iter.reset(super_version->mem->NewRangeTombstoneIterator(
+        read_options, sequence, false /* immutable_memtable */));
     range_del_agg->AddTombstones(std::move(range_del_iter));
   }
   // Collect all needed child iterators for immutable memtables
@@ -1685,11 +1782,11 @@ InternalIterator* DBImpl::NewInternalIterator(const ReadOptions& read_options,
                                            allow_unprepared_value);
     }
     internal_iter = merge_iter_builder.Finish();
-    IterState* cleanup =
-        new IterState(this, &mutex_, super_version,
-                      read_options.background_purge_on_iterator_cleanup ||
-                      immutable_db_options_.avoid_unnecessary_blocking_io);
-    internal_iter->RegisterCleanup(CleanupIteratorState, cleanup, nullptr);
+    SuperVersionHandle* cleanup = new SuperVersionHandle(
+        this, &mutex_, super_version,
+        read_options.background_purge_on_iterator_cleanup ||
+            immutable_db_options_.avoid_unnecessary_blocking_io);
+    internal_iter->RegisterCleanup(CleanupSuperVersionHandle, cleanup, nullptr);
 
     return internal_iter;
   } else {
@@ -1725,10 +1822,61 @@ Status DBImpl::Get(const ReadOptions& read_options,
   return s;
 }
 
+Status DBImpl::GetEntity(const ReadOptions& read_options,
+                         ColumnFamilyHandle* column_family, const Slice& key,
+                         PinnableWideColumns* columns) {
+  if (!column_family) {
+    return Status::InvalidArgument(
+        "Cannot call GetEntity without a column family handle");
+  }
+
+  if (!columns) {
+    return Status::InvalidArgument(
+        "Cannot call GetEntity without a PinnableWideColumns object");
+  }
+
+  columns->Reset();
+
+  GetImplOptions get_impl_options;
+  get_impl_options.column_family = column_family;
+  get_impl_options.columns = columns;
+
+  return GetImpl(read_options, key, get_impl_options);
+}
+
+bool DBImpl::ShouldReferenceSuperVersion(const MergeContext& merge_context) {
+  // If both thresholds are reached, a function returning merge operands as
+  // `PinnableSlice`s should reference the `SuperVersion` to avoid large and/or
+  // numerous `memcpy()`s.
+  //
+  // The below constants enable the optimization conservatively. They are
+  // verified to not regress `GetMergeOperands()` latency in the following
+  // scenarios.
+  //
+  // - CPU: two socket Intel(R) Xeon(R) Gold 6138 CPU @ 2.00GHz
+  // - `GetMergeOperands()` threads: 1 - 32
+  // - Entry size: 32 bytes - 4KB
+  // - Merges per key: 1 - 16K
+  // - LSM component: memtable
+  //
+  // TODO(ajkr): expand measurement to SST files.
+  static const size_t kNumBytesForSvRef = 32768;
+  static const size_t kLog2AvgBytesForSvRef = 8;  // 256 bytes
+
+  size_t num_bytes = 0;
+  for (const Slice& sl : merge_context.GetOperands()) {
+    num_bytes += sl.size();
+  }
+  return num_bytes >= kNumBytesForSvRef &&
+         (num_bytes >> kLog2AvgBytesForSvRef) >=
+             merge_context.GetOperands().size();
+}
+
 Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
                        GetImplOptions& get_impl_options) {
   assert(get_impl_options.value != nullptr ||
-         get_impl_options.merge_operands != nullptr);
+         get_impl_options.merge_operands != nullptr ||
+         get_impl_options.columns != nullptr);
 
   assert(get_impl_options.column_family);
 
@@ -1772,6 +1920,14 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
     }
   }
 
+  if (get_impl_options.get_merge_operands_options != nullptr) {
+    for (int i = 0; i < get_impl_options.get_merge_operands_options
+                            ->expected_max_number_of_operands;
+         ++i) {
+      get_impl_options.merge_operands[i].Reset();
+    }
+  }
+
   // Acquire SuperVersion
   SuperVersion* sv = GetAndRefSuperVersion(cfd);
 
@@ -1847,29 +2003,46 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
   if (!skip_memtable) {
     // Get value associated with key
     if (get_impl_options.get_value) {
-      if (sv->mem->Get(lkey, get_impl_options.value->GetSelf(), timestamp, &s,
-                       &merge_context, &max_covering_tombstone_seq,
-                       read_options, get_impl_options.callback,
-                       get_impl_options.is_blob_index)) {
+      if (sv->mem->Get(
+              lkey,
+              get_impl_options.value ? get_impl_options.value->GetSelf()
+                                     : nullptr,
+              get_impl_options.columns, timestamp, &s, &merge_context,
+              &max_covering_tombstone_seq, read_options,
+              false /* immutable_memtable */, get_impl_options.callback,
+              get_impl_options.is_blob_index)) {
         done = true;
-        get_impl_options.value->PinSelf();
+
+        if (get_impl_options.value) {
+          get_impl_options.value->PinSelf();
+        }
+
         RecordTick(stats_, MEMTABLE_HIT);
       } else if ((s.ok() || s.IsMergeInProgress()) &&
-                 sv->imm->Get(lkey, get_impl_options.value->GetSelf(),
-                              timestamp, &s, &merge_context,
-                              &max_covering_tombstone_seq, read_options,
-                              get_impl_options.callback,
+                 sv->imm->Get(lkey,
+                              get_impl_options.value
+                                  ? get_impl_options.value->GetSelf()
+                                  : nullptr,
+                              get_impl_options.columns, timestamp, &s,
+                              &merge_context, &max_covering_tombstone_seq,
+                              read_options, get_impl_options.callback,
                               get_impl_options.is_blob_index)) {
         done = true;
-        get_impl_options.value->PinSelf();
+
+        if (get_impl_options.value) {
+          get_impl_options.value->PinSelf();
+        }
+
         RecordTick(stats_, MEMTABLE_HIT);
       }
     } else {
       // Get Merge Operands associated with key, Merge Operands should not be
       // merged and raw values should be returned to the user.
-      if (sv->mem->Get(lkey, /*value*/ nullptr, /*timestamp=*/nullptr, &s,
-                       &merge_context, &max_covering_tombstone_seq,
-                       read_options, nullptr, nullptr, false)) {
+      if (sv->mem->Get(lkey, /*value=*/nullptr, /*columns=*/nullptr,
+                       /*timestamp=*/nullptr, &s, &merge_context,
+                       &max_covering_tombstone_seq, read_options,
+                       false /* immutable_memtable */, nullptr, nullptr,
+                       false)) {
         done = true;
         RecordTick(stats_, MEMTABLE_HIT);
       } else if ((s.ok() || s.IsMergeInProgress()) &&
@@ -1891,8 +2064,9 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
   if (!done) {
     PERF_TIMER_GUARD(get_from_output_files_time);
     sv->current->Get(
-        read_options, lkey, get_impl_options.value, timestamp, &s,
-        &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
+        read_options, lkey, get_impl_options.value, get_impl_options.columns,
+        timestamp, &s, &merge_context, &max_covering_tombstone_seq,
+        &pinned_iters_mgr,
         get_impl_options.get_value ? get_impl_options.value_found : nullptr,
         nullptr, nullptr,
         get_impl_options.get_value ? get_impl_options.callback : nullptr,
@@ -1908,7 +2082,11 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
     size_t size = 0;
     if (s.ok()) {
       if (get_impl_options.get_value) {
-        size = get_impl_options.value->size();
+        if (get_impl_options.value) {
+          size = get_impl_options.value->size();
+        } else if (get_impl_options.columns) {
+          size = get_impl_options.columns->serialized_size();
+        }
       } else {
         // Return all merge operands for get_impl_options.key
         *get_impl_options.number_of_operands =
@@ -1919,19 +2097,68 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
           s = Status::Incomplete(
               Status::SubCode::KMergeOperandsInsufficientCapacity);
         } else {
-          for (const Slice& sl : merge_context.GetOperands()) {
-            size += sl.size();
-            get_impl_options.merge_operands->PinSelf(sl);
-            get_impl_options.merge_operands++;
+          // Each operand depends on one of the following resources: `sv`,
+          // `pinned_iters_mgr`, or `merge_context`. It would be crazy expensive
+          // to reference `sv` for each operand relying on it because `sv` is
+          // (un)ref'd in all threads using the DB. Furthermore, we do not track
+          // on which resource each operand depends.
+          //
+          // To solve this, we bundle the resources in a `GetMergeOperandsState`
+          // and manage them with a `SharedCleanablePtr` shared among the
+          // `PinnableSlice`s we return. This bundle includes one `sv` reference
+          // and ownership of the `merge_context` and `pinned_iters_mgr`
+          // objects.
+          bool ref_sv = ShouldReferenceSuperVersion(merge_context);
+          if (ref_sv) {
+            assert(!merge_context.GetOperands().empty());
+            SharedCleanablePtr shared_cleanable;
+            GetMergeOperandsState* state = nullptr;
+            state = new GetMergeOperandsState();
+            state->merge_context = std::move(merge_context);
+            state->pinned_iters_mgr = std::move(pinned_iters_mgr);
+
+            sv->Ref();
+
+            state->sv_handle = new SuperVersionHandle(
+                this, &mutex_, sv,
+                immutable_db_options_.avoid_unnecessary_blocking_io);
+
+            shared_cleanable.Allocate();
+            shared_cleanable->RegisterCleanup(CleanupGetMergeOperandsState,
+                                              state /* arg1 */,
+                                              nullptr /* arg2 */);
+            for (size_t i = 0; i < state->merge_context.GetOperands().size();
+                 ++i) {
+              const Slice& sl = state->merge_context.GetOperands()[i];
+              size += sl.size();
+
+              get_impl_options.merge_operands->PinSlice(
+                  sl, nullptr /* cleanable */);
+              if (i == state->merge_context.GetOperands().size() - 1) {
+                shared_cleanable.MoveAsCleanupTo(
+                    get_impl_options.merge_operands);
+              } else {
+                shared_cleanable.RegisterCopyWith(
+                    get_impl_options.merge_operands);
+              }
+              get_impl_options.merge_operands++;
+            }
+          } else {
+            for (const Slice& sl : merge_context.GetOperands()) {
+              size += sl.size();
+              get_impl_options.merge_operands->PinSelf(sl);
+              get_impl_options.merge_operands++;
+            }
           }
         }
       }
       RecordTick(stats_, BYTES_READ, size);
       PERF_COUNTER_ADD(get_read_bytes, size);
     }
-    RecordInHistogram(stats_, BYTES_PER_READ, size);
 
     ReturnAndCleanupSuperVersion(cfd, sv);
+
+    RecordInHistogram(stats_, BYTES_PER_READ, size);
   }
   return s;
 }
@@ -2068,13 +2295,14 @@ std::vector<Status> DBImpl::MultiGet(
          has_unpersisted_data_.load(std::memory_order_relaxed));
     bool done = false;
     if (!skip_memtable) {
-      if (super_version->mem->Get(lkey, value, timestamp, &s, &merge_context,
-                                  &max_covering_tombstone_seq, read_options,
-                                  read_callback)) {
+      if (super_version->mem->Get(
+              lkey, value, /*columns=*/nullptr, timestamp, &s, &merge_context,
+              &max_covering_tombstone_seq, read_options,
+              false /* immutable_memtable */, read_callback)) {
         done = true;
         RecordTick(stats_, MEMTABLE_HIT);
-      } else if (super_version->imm->Get(lkey, value, timestamp, &s,
-                                         &merge_context,
+      } else if (super_version->imm->Get(lkey, value, /*columns=*/nullptr,
+                                         timestamp, &s, &merge_context,
                                          &max_covering_tombstone_seq,
                                          read_options, read_callback)) {
         done = true;
@@ -2085,9 +2313,9 @@ std::vector<Status> DBImpl::MultiGet(
       PinnableSlice pinnable_val;
       PERF_TIMER_GUARD(get_from_output_files_time);
       PinnedIteratorsManager pinned_iters_mgr;
-      super_version->current->Get(read_options, lkey, &pinnable_val, timestamp,
-                                  &s, &merge_context,
-                                  &max_covering_tombstone_seq,
+      super_version->current->Get(read_options, lkey, &pinnable_val,
+                                  /*columns=*/nullptr, timestamp, &s,
+                                  &merge_context, &max_covering_tombstone_seq,
                                   &pinned_iters_mgr, /*value_found=*/nullptr,
                                   /*key_exists=*/nullptr,
                                   /*seq=*/nullptr, read_callback);
@@ -2606,7 +2834,8 @@ Status DBImpl::MultiGetImpl(
         (read_options.read_tier == kPersistedTier &&
          has_unpersisted_data_.load(std::memory_order_relaxed));
     if (!skip_memtable) {
-      super_version->mem->MultiGet(read_options, &range, callback);
+      super_version->mem->MultiGet(read_options, &range, callback,
+                                   false /* immutable_memtable */);
       if (!range.empty()) {
         super_version->imm->MultiGet(read_options, &range, callback);
       }
@@ -2805,6 +3034,14 @@ Status DBImpl::CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options,
     }
   }  // InstrumentedMutexLock l(&mutex_)
 
+  if (cf_options.preclude_last_level_data_seconds > 0) {
+    // TODO(zjay): Fix the timer issue and re-enable this.
+    ROCKS_LOG_ERROR(
+        immutable_db_options_.info_log,
+        "Creating column family with `preclude_last_level_data_seconds` needs "
+        "to restart DB to take effect");
+    //    s = RegisterRecordSeqnoTimeWorker();
+  }
   sv_context.Clean();
   // this is outside the mutex
   if (s.ok()) {
@@ -2893,6 +3130,10 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {
     bg_cv_.SignalAll();
   }
 
+  if (cfd->ioptions()->preclude_last_level_data_seconds > 0) {
+    s = RegisterRecordSeqnoTimeWorker();
+  }
+
   if (s.ok()) {
     // Note that here we erase the associated cf_info of the to-be-dropped
     // cfd before its ref-count goes to zero to avoid having to erase cf_info
@@ -4663,9 +4904,10 @@ Status DBImpl::GetLatestSequenceForKey(
   *found_record_for_key = false;
 
   // Check if there is a record for this key in the latest memtable
-  sv->mem->Get(lkey, /*value=*/nullptr, timestamp, &s, &merge_context,
-               &max_covering_tombstone_seq, seq, read_options,
-               nullptr /*read_callback*/, is_blob_index);
+  sv->mem->Get(lkey, /*value=*/nullptr, /*columns=*/nullptr, timestamp, &s,
+               &merge_context, &max_covering_tombstone_seq, seq, read_options,
+               false /* immutable_memtable */, nullptr /*read_callback*/,
+               is_blob_index);
 
   if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
     // unexpected error reading memtable.
@@ -4696,8 +4938,8 @@ Status DBImpl::GetLatestSequenceForKey(
   }
 
   // Check if there is a record for this key in the immutable memtables
-  sv->imm->Get(lkey, /*value=*/nullptr, timestamp, &s, &merge_context,
-               &max_covering_tombstone_seq, seq, read_options,
+  sv->imm->Get(lkey, /*value=*/nullptr, /*columns=*/nullptr, timestamp, &s,
+               &merge_context, &max_covering_tombstone_seq, seq, read_options,
                nullptr /*read_callback*/, is_blob_index);
 
   if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
@@ -4728,9 +4970,10 @@ Status DBImpl::GetLatestSequenceForKey(
   }
 
   // Check if there is a record for this key in the immutable memtables
-  sv->imm->GetFromHistory(lkey, /*value=*/nullptr, timestamp, &s,
-                          &merge_context, &max_covering_tombstone_seq, seq,
-                          read_options, is_blob_index);
+  sv->imm->GetFromHistory(lkey, /*value=*/nullptr, /*columns=*/nullptr,
+                          timestamp, &s, &merge_context,
+                          &max_covering_tombstone_seq, seq, read_options,
+                          is_blob_index);
 
   if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
     // unexpected error reading memtable.
@@ -4763,8 +5006,8 @@ Status DBImpl::GetLatestSequenceForKey(
   if (!cache_only) {
     // Check tables
     PinnedIteratorsManager pinned_iters_mgr;
-    sv->current->Get(read_options, lkey, /*value=*/nullptr, timestamp, &s,
-                     &merge_context, &max_covering_tombstone_seq,
+    sv->current->Get(read_options, lkey, /*value=*/nullptr, /*columns=*/nullptr,
+                     timestamp, &s, &merge_context, &max_covering_tombstone_seq,
                      &pinned_iters_mgr, nullptr /* value_found */,
                      found_record_for_key, seq, nullptr /*read_callback*/,
                      is_blob_index);
@@ -5536,6 +5779,26 @@ Status DBImpl::GetCreationTimeOfOldestFile(uint64_t* creation_time) {
     return Status::NotSupported("This API only works if max_open_files = -1");
   }
 }
+
+void DBImpl::RecordSeqnoToTimeMapping() {
+  // Get time first then sequence number, so the actual time of seqno is <=
+  // unix_time recorded
+  int64_t unix_time = 0;
+  immutable_db_options_.clock->GetCurrentTime(&unix_time)
+      .PermitUncheckedError();  // Ignore error
+  SequenceNumber seqno = GetLatestSequenceNumber();
+  bool appended = false;
+  {
+    InstrumentedMutexLock l(&mutex_);
+    appended = seqno_time_mapping_.Append(seqno, unix_time);
+  }
+  if (!appended) {
+    ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                   "Failed to insert sequence number to time entry: %" PRIu64
+                   " -> %" PRIu64,
+                   seqno, unix_time);
+  }
+}
 #endif  // ROCKSDB_LITE
 
 }  // namespace ROCKSDB_NAMESPACE

db/db_impl/db_impl.h

@@ -36,6 +36,7 @@
 #include "db/pre_release_callback.h"
 #include "db/range_del_aggregator.h"
 #include "db/read_callback.h"
+#include "db/seqno_to_time_mapping.h"
 #include "db/snapshot_checker.h"
 #include "db/snapshot_impl.h"
 #include "db/trim_history_scheduler.h"
@@ -117,7 +118,6 @@ class Directories {
   IOStatus Close(const IOOptions& options, IODebugContext* dbg) {
     // close all directories for all database paths
     IOStatus s = IOStatus::OK();
-    IOStatus temp_s = IOStatus::OK();
 
     // The default implementation for Close() in Directory/FSDirectory class
     // "NotSupported" status, the upper level interface should be able to
@@ -126,53 +126,35 @@ class Directories {
     // `FSDirectory::Close()` yet
 
     if (db_dir_) {
-      temp_s = db_dir_->Close(options, dbg);
-      if (!temp_s.ok()) {
-        if (temp_s.IsNotSupported()) {
-          temp_s.PermitUncheckedError();
-        } else {
-          s = temp_s;
-        }
+      IOStatus temp_s = db_dir_->Close(options, dbg);
+      if (!temp_s.ok() && !temp_s.IsNotSupported() && s.ok()) {
+        s = std::move(temp_s);
       }
     }
 
-    if (!s.ok()) {
-      return s;
-    }
+    // Attempt to close everything even if one fails
+    s.PermitUncheckedError();
 
     if (wal_dir_) {
-      s = wal_dir_->Close(options, dbg);
-      if (!temp_s.ok()) {
-        if (temp_s.IsNotSupported()) {
-          temp_s.PermitUncheckedError();
-        } else {
-          s = temp_s;
+      IOStatus temp_s = wal_dir_->Close(options, dbg);
+      if (!temp_s.ok() && !temp_s.IsNotSupported() && s.ok()) {
+        s = std::move(temp_s);
+      }
+    }
+
+    s.PermitUncheckedError();
+
+    for (auto& data_dir_ptr : data_dirs_) {
+      if (data_dir_ptr) {
+        IOStatus temp_s = data_dir_ptr->Close(options, dbg);
+        if (!temp_s.ok() && !temp_s.IsNotSupported() && s.ok()) {
+          s = std::move(temp_s);
         }
       }
     }
 
-    if (!s.ok()) {
-      return s;
-    }
-
-    if (data_dirs_.size() > 0 && s.ok()) {
-      for (auto& data_dir_ptr : data_dirs_) {
-        if (data_dir_ptr) {
-          temp_s = data_dir_ptr->Close(options, dbg);
-          if (!temp_s.ok()) {
-            if (temp_s.IsNotSupported()) {
-              temp_s.PermitUncheckedError();
-            } else {
-              return temp_s;
-            }
-          }
-        }
-      }
-    }
-
-    // Mark temp_s as checked when temp_s is still the initial status
-    // (IOStatus::OK(), not checked yet)
-    temp_s.PermitUncheckedError();
+    // Ready for caller
+    s.MustCheck();
     return s;
   }
 
@@ -256,6 +238,11 @@ class DBImpl : public DB {
                      ColumnFamilyHandle* column_family, const Slice& key,
                      PinnableSlice* value, std::string* timestamp) override;
 
+  using DB::GetEntity;
+  Status GetEntity(const ReadOptions& options,
+                   ColumnFamilyHandle* column_family, const Slice& key,
+                   PinnableWideColumns* columns) override;
+
   using DB::GetMergeOperands;
   Status GetMergeOperands(const ReadOptions& options,
                           ColumnFamilyHandle* column_family, const Slice& key,
@@ -610,6 +597,7 @@ class DBImpl : public DB {
   struct GetImplOptions {
     ColumnFamilyHandle* column_family = nullptr;
     PinnableSlice* value = nullptr;
+    PinnableWideColumns* columns = nullptr;
     std::string* timestamp = nullptr;
     bool* value_found = nullptr;
     ReadCallback* callback = nullptr;
@@ -997,6 +985,7 @@ class DBImpl : public DB {
   }
 
   void AddToLogsToFreeQueue(log::Writer* log_writer) {
+    mutex_.AssertHeld();
     logs_to_free_queue_.push_back(log_writer);
   }
 
@@ -1158,7 +1147,8 @@ class DBImpl : public DB {
   int TEST_BGCompactionsAllowed() const;
   int TEST_BGFlushesAllowed() const;
   size_t TEST_GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;
-  void TEST_WaitForStatsDumpRun(std::function<void()> callback) const;
+  void TEST_WaitForPeridicWorkerRun(std::function<void()> callback) const;
+  SeqnoToTimeMapping TEST_GetSeqnoToTimeMapping() const;
   size_t TEST_EstimateInMemoryStatsHistorySize() const;
 
   uint64_t TEST_GetCurrentLogNumber() const {
@@ -1186,6 +1176,9 @@ class DBImpl : public DB {
   // flush LOG out of application buffer
   void FlushInfoLog();
 
+  // record current sequence number to time mapping
+  void RecordSeqnoToTimeMapping();
+
   // Interface to block and signal the DB in case of stalling writes by
   // WriteBufferManager. Each DBImpl object contains ptr to WBMStallInterface.
   // When DB needs to be blocked or signalled by WriteBufferManager,
@@ -1293,7 +1286,7 @@ class DBImpl : public DB {
 
   // only used for dynamically adjusting max_total_wal_size. it is a sum of
   // [write_buffer_size * max_write_buffer_number] over all column families
-  uint64_t max_total_in_memory_state_;
+  std::atomic<uint64_t> max_total_in_memory_state_;
 
   // The options to access storage files
   const FileOptions file_options_;
@@ -1643,6 +1636,15 @@ class DBImpl : public DB {
     uint64_t pre_sync_size = 0;
   };
 
+  struct LogContext {
+    explicit LogContext(bool need_sync = false)
+        : need_log_sync(need_sync), need_log_dir_sync(need_sync) {}
+    bool need_log_sync = false;
+    bool need_log_dir_sync = false;
+    log::Writer* writer = nullptr;
+    LogFileNumberSize* log_file_number_size = nullptr;
+  };
+
   // PurgeFileInfo is a structure to hold information of files to be deleted in
   // purge_files_
   struct PurgeFileInfo {
@@ -1796,7 +1798,7 @@ class DBImpl : public DB {
   void ReleaseFileNumberFromPendingOutputs(
       std::unique_ptr<std::list<uint64_t>::iterator>& v);
 
-  IOStatus SyncClosedLogs(JobContext* job_context);
+  IOStatus SyncClosedLogs(JobContext* job_context, VersionEdit* synced_wals);
 
   // Flush the in-memory write buffer to storage.  Switches to a new
   // log-file/memtable and writes a new descriptor iff successful. Then
@@ -1956,8 +1958,8 @@ class DBImpl : public DB {
   Status HandleWriteBufferManagerFlush(WriteContext* write_context);
 
   // REQUIRES: mutex locked
-  Status PreprocessWrite(const WriteOptions& write_options, bool* need_log_sync,
-                         WriteContext* write_context);
+  Status PreprocessWrite(const WriteOptions& write_options,
+                         LogContext* log_context, WriteContext* write_context);
 
   // Merge write batches in the write group into merged_batch.
   // Returns OK if merge is successful.
@@ -2069,6 +2071,8 @@ class DBImpl : public DB {
   // Schedule background tasks
   Status StartPeriodicWorkScheduler();
 
+  Status RegisterRecordSeqnoTimeWorker();
+
   void PrintStatistics();
 
   size_t EstimateInMemoryStatsHistorySize() const;
@@ -2094,7 +2098,8 @@ class DBImpl : public DB {
       std::unique_ptr<TaskLimiterToken>* token, LogBuffer* log_buffer);
 
   // helper function to call after some of the logs_ were synced
-  Status MarkLogsSynced(uint64_t up_to, bool synced_dir);
+  void MarkLogsSynced(uint64_t up_to, bool synced_dir, VersionEdit* edit);
+  Status ApplyWALToManifest(VersionEdit* edit);
   // WALs with log number up to up_to are not synced successfully.
   void MarkLogsNotSynced(uint64_t up_to);
 
@@ -2261,6 +2266,8 @@ class DBImpl : public DB {
   Status IncreaseFullHistoryTsLowImpl(ColumnFamilyData* cfd,
                                       std::string ts_low);
 
+  bool ShouldReferenceSuperVersion(const MergeContext& merge_context);
+
   // Lock over the persistent DB state.  Non-nullptr iff successfully acquired.
   FileLock* db_lock_;
 
@@ -2300,8 +2307,9 @@ class DBImpl : public DB {
   // logfile_number_ is currently updated only in write_thread_, it can be read
   // from the same write_thread_ without any locks.
   uint64_t logfile_number_;
-  std::deque<uint64_t>
-      log_recycle_files_;  // a list of log files that we can recycle
+  // Log files that we can recycle. Must be protected by db mutex_.
+  std::deque<uint64_t> log_recycle_files_;
+  // Protected by log_write_mutex_.
   bool log_dir_synced_;
   // Without two_write_queues, read and writes to log_empty_ are protected by
   // mutex_. Since it is currently updated/read only in write_thread_, it can be
@@ -2315,26 +2323,93 @@ class DBImpl : public DB {
 
   bool persistent_stats_cfd_exists_ = true;
 
-  // Without two_write_queues, read and writes to alive_log_files_ are
-  // protected by mutex_. With two_write_queues_, writes
-  // are protected by locking both mutex_ and log_write_mutex_, and reads must
-  // be under either mutex_ or log_write_mutex_.
+  // alive_log_files_ is protected by mutex_ and log_write_mutex_ with details
+  // as follows:
+  // 1. read by FindObsoleteFiles() which can be called in either application
+  //    thread or RocksDB bg threads, both mutex_ and log_write_mutex_ are
+  //    held.
+  // 2. pop_front() by FindObsoleteFiles(), both mutex_ and log_write_mutex_
+  //    are held.
+  // 3. push_back() by DBImpl::Open() and DBImpl::RestoreAliveLogFiles()
+  //    (actually called by Open()), only mutex_ is held because at this point,
+  //    the DB::Open() call has not returned success to application, and the
+  //    only other thread(s) that can conflict are bg threads calling
+  //    FindObsoleteFiles() which ensure that both mutex_ and log_write_mutex_
+  //    are held when accessing alive_log_files_.
+  // 4. read by DBImpl::Open() is protected by mutex_.
+  // 5. push_back() by SwitchMemtable(). Both mutex_ and log_write_mutex_ are
+  //    held. This is done by the write group leader. Note that in the case of
+  //    two-write-queues, another WAL-only write thread can be writing to the
+  //    WAL concurrently. See 9.
+  // 6. read by SwitchWAL() with both mutex_ and log_write_mutex_ held. This is
+  //    done by write group leader.
+  // 7. read by ConcurrentWriteToWAL() by the write group leader in the case of
+  //    two-write-queues. Only log_write_mutex_ is held to protect concurrent
+  //    pop_front() by FindObsoleteFiles().
+  // 8. read by PreprocessWrite() by the write group leader. log_write_mutex_
+  //    is held to protect the data structure from concurrent pop_front() by
+  //    FindObsoleteFiles().
+  // 9. read by ConcurrentWriteToWAL() by a WAL-only write thread in the case
+  //    of two-write-queues. Only log_write_mutex_ is held. This suffices to
+  //    protect the data structure from concurrent push_back() by current
+  //    write group leader as well as pop_front() by FindObsoleteFiles().
   std::deque<LogFileNumberSize> alive_log_files_;
 
   // Log files that aren't fully synced, and the current log file.
   // Synchronization:
-  //  - push_back() is done from write_thread_ with locked mutex_ and
-  //  log_write_mutex_
-  //  - pop_front() is done from any thread with locked mutex_ and
-  //  log_write_mutex_
-  //  - reads are done with either locked mutex_ or log_write_mutex_
+  // 1. read by FindObsoleteFiles() which can be called either in application
+  //    thread or RocksDB bg threads. log_write_mutex_ is always held, while
+  //    some reads are performed without mutex_.
+  // 2. pop_front() by FindObsoleteFiles() with only log_write_mutex_ held.
+  // 3. read by DBImpl::Open() with both mutex_ and log_write_mutex_.
+  // 4. emplace_back() by DBImpl::Open() with both mutex_ and log_write_mutex.
+  //    Note that at this point, DB::Open() has not returned success to
+  //    application, thus the only other thread(s) that can conflict are bg
+  //    threads calling FindObsoleteFiles(). See 1.
+  // 5. iteration and clear() from CloseHelper() always hold log_write_mutex
+  //    and mutex_.
+  // 6. back() called by APIs FlushWAL() and LockWAL() are protected by only
+  //    log_write_mutex_. These two can be called by application threads after
+  //    DB::Open() returns success to applications.
+  // 7. read by SyncWAL(), another API, protected by only log_write_mutex_.
+  // 8. read by MarkLogsNotSynced() and MarkLogsSynced() are protected by
+  //    log_write_mutex_.
+  // 9. erase() by MarkLogsSynced() protected by log_write_mutex_.
+  // 10. read by SyncClosedLogs() protected by only log_write_mutex_. This can
+  //     happen in bg flush threads after DB::Open() returns success to
+  //     applications.
+  // 11. reads, e.g. front(), iteration, and back() called by PreprocessWrite()
+  //     holds only the log_write_mutex_. This is done by the write group
+  //     leader. A bg thread calling FindObsoleteFiles() or MarkLogsSynced()
+  //     can happen concurrently. This is fine because log_write_mutex_ is used
+  //     by all parties. See 2, 5, 9.
+  // 12. reads, empty(), back() called by SwitchMemtable() hold both mutex_ and
+  //     log_write_mutex_. This happens in the write group leader.
+  // 13. emplace_back() by SwitchMemtable() hold both mutex_ and
+  //     log_write_mutex_. This happens in the write group leader. Can conflict
+  //     with bg threads calling FindObsoleteFiles(), MarkLogsSynced(),
+  //     SyncClosedLogs(), etc. as well as application threads calling
+  //     FlushWAL(), SyncWAL(), LockWAL(). This is fine because all parties
+  //     require at least log_write_mutex_.
+  // 14. iteration called in WriteToWAL(write_group) protected by
+  //     log_write_mutex_. This is done by write group leader when
+  //     two-write-queues is disabled and write needs to sync logs.
+  // 15. back() called in ConcurrentWriteToWAL() protected by log_write_mutex_.
+  //     This can be done by the write group leader if two-write-queues is
+  //     enabled. It can also be done by another WAL-only write thread.
+  //
+  // Other observations:
   //  - back() and items with getting_synced=true are not popped,
   //  - The same thread that sets getting_synced=true will reset it.
   //  - it follows that the object referred by back() can be safely read from
-  //  the write_thread_ without using mutex
+  //  the write_thread_ without using mutex. Note that calling back() without
+  //  mutex may be unsafe because different implementations of deque::back() may
+  //  access other member variables of deque, causing undefined behaviors.
+  //  Generally, do not access stl containers without proper synchronization.
   //  - it follows that the items with getting_synced=true can be safely read
   //  from the same thread that has set getting_synced=true
   std::deque<LogWriterNumber> logs_;
+
   // Signaled when getting_synced becomes false for some of the logs_.
   InstrumentedCondVar log_sync_cv_;
   // This is the app-level state that is written to the WAL but will be used
@@ -2349,7 +2424,7 @@ class DBImpl : public DB {
   std::atomic<uint64_t> total_log_size_;
 
   // If this is non-empty, we need to delete these log files in background
-  // threads. Protected by db mutex.
+  // threads. Protected by log_write_mutex_.
   autovector<log::Writer*> logs_to_free_;
 
   bool is_snapshot_supported_;
@@ -2429,10 +2504,13 @@ class DBImpl : public DB {
   // JobContext. Current implementation tracks table and blob files only.
   std::unordered_set<uint64_t> files_grabbed_for_purge_;
 
-  // A queue to store log writers to close
+  // A queue to store log writers to close. Protected by db mutex_.
   std::deque<log::Writer*> logs_to_free_queue_;
+
   std::deque<SuperVersion*> superversions_to_free_queue_;
+
   int unscheduled_flushes_;
+
   int unscheduled_compactions_;
 
   // count how many background compactions are running or have been scheduled in
@@ -2537,6 +2615,10 @@ class DBImpl : public DB {
   // PeriodicWorkScheduler::Default(). Only in unittest, it can be overrided by
   // PeriodicWorkTestScheduler.
   PeriodicWorkScheduler* periodic_work_scheduler_;
+
+  // Current cadence of the periodic worker for recording sequence number to
+  // time.
+  uint64_t record_seqno_time_cadence_ = 0;
 #endif
 
   // When set, we use a separate queue for writes that don't write to memtable.
@@ -2581,11 +2663,16 @@ class DBImpl : public DB {
   InstrumentedCondVar atomic_flush_install_cv_;
 
   bool wal_in_db_path_;
+  std::atomic<uint64_t> max_total_wal_size_;
 
   BlobFileCompletionCallback blob_callback_;
 
   // Pointer to WriteBufferManager stalling interface.
   std::unique_ptr<StallInterface> wbm_stall_;
+
+  // seqno_time_mapping_ stores the sequence number to time mapping, it's not
+  // thread safe, both read and write need db mutex hold.
+  SeqnoToTimeMapping seqno_time_mapping_;
 };
 
 class GetWithTimestampReadCallback : public ReadCallback {

db/db_impl/db_impl_compaction_flush.cc

@@ -82,9 +82,10 @@ bool DBImpl::RequestCompactionToken(ColumnFamilyData* cfd, bool force,
   return false;
 }
 
-IOStatus DBImpl::SyncClosedLogs(JobContext* job_context) {
+IOStatus DBImpl::SyncClosedLogs(JobContext* job_context,
+                                VersionEdit* synced_wals) {
   TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Start");
-  mutex_.AssertHeld();
+  InstrumentedMutexLock l(&log_write_mutex_);
   autovector<log::Writer*, 1> logs_to_sync;
   uint64_t current_log_number = logfile_number_;
   while (logs_.front().number < current_log_number &&
@@ -100,7 +101,7 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context) {
 
   IOStatus io_s;
   if (!logs_to_sync.empty()) {
-    mutex_.Unlock();
+    log_write_mutex_.Unlock();
 
     assert(job_context);
 
@@ -108,12 +109,18 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context) {
       ROCKS_LOG_INFO(immutable_db_options_.info_log,
                      "[JOB %d] Syncing log #%" PRIu64, job_context->job_id,
                      log->get_log_number());
+      if (error_handler_.IsRecoveryInProgress()) {
+        log->file()->reset_seen_error();
+      }
       io_s = log->file()->Sync(immutable_db_options_.use_fsync);
       if (!io_s.ok()) {
         break;
       }
 
       if (immutable_db_options_.recycle_log_file_num > 0) {
+        if (error_handler_.IsRecoveryInProgress()) {
+          log->file()->reset_seen_error();
+        }
         io_s = log->Close();
         if (!io_s.ok()) {
           break;
@@ -128,12 +135,12 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context) {
 
     TEST_SYNC_POINT_CALLBACK("DBImpl::SyncClosedLogs:BeforeReLock",
                              /*arg=*/nullptr);
-    mutex_.Lock();
+    log_write_mutex_.Lock();
 
     // "number <= current_log_number - 1" is equivalent to
     // "number < current_log_number".
     if (io_s.ok()) {
-      io_s = status_to_io_status(MarkLogsSynced(current_log_number - 1, true));
+      MarkLogsSynced(current_log_number - 1, true, synced_wals);
     } else {
       MarkLogsNotSynced(current_log_number - 1);
     }
@@ -212,16 +219,26 @@ Status DBImpl::FlushMemTableToOutputFile(
       GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
       &event_logger_, mutable_cf_options.report_bg_io_stats,
       true /* sync_output_directory */, true /* write_manifest */, thread_pri,
-      io_tracer_, db_id_, db_session_id_, cfd->GetFullHistoryTsLow(),
-      &blob_callback_);
+      io_tracer_, seqno_time_mapping_, db_id_, db_session_id_,
+      cfd->GetFullHistoryTsLow(), &blob_callback_);
   FileMetaData file_meta;
 
   Status s;
   bool need_cancel = false;
   IOStatus log_io_s = IOStatus::OK();
   if (needs_to_sync_closed_wals) {
-    // SyncClosedLogs() may unlock and re-lock the db_mutex.
-    log_io_s = SyncClosedLogs(job_context);
+    // SyncClosedLogs() may unlock and re-lock the log_write_mutex multiple
+    // times.
+    VersionEdit synced_wals;
+    mutex_.Unlock();
+    log_io_s = SyncClosedLogs(job_context, &synced_wals);
+    mutex_.Lock();
+    if (log_io_s.ok() && synced_wals.IsWalAddition()) {
+      log_io_s = status_to_io_status(ApplyWALToManifest(&synced_wals));
+      TEST_SYNC_POINT_CALLBACK("DBImpl::FlushMemTableToOutputFile:CommitWal:1",
+                               nullptr);
+    }
+
     if (!log_io_s.ok() && !log_io_s.IsShutdownInProgress() &&
         !log_io_s.IsColumnFamilyDropped()) {
       error_handler_.SetBGError(log_io_s, BackgroundErrorReason::kFlush);
@@ -450,7 +467,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
         data_dir, GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
         stats_, &event_logger_, mutable_cf_options.report_bg_io_stats,
         false /* sync_output_directory */, false /* write_manifest */,
-        thread_pri, io_tracer_, db_id_, db_session_id_,
+        thread_pri, io_tracer_, seqno_time_mapping_, db_id_, db_session_id_,
         cfd->GetFullHistoryTsLow(), &blob_callback_));
   }
 
@@ -474,7 +491,14 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
   if (logfile_number_ > 0) {
     // TODO (yanqin) investigate whether we should sync the closed logs for
     // single column family case.
-    log_io_s = SyncClosedLogs(job_context);
+    VersionEdit synced_wals;
+    mutex_.Unlock();
+    log_io_s = SyncClosedLogs(job_context, &synced_wals);
+    mutex_.Lock();
+    if (log_io_s.ok() && synced_wals.IsWalAddition()) {
+      log_io_s = status_to_io_status(ApplyWALToManifest(&synced_wals));
+    }
+
     if (!log_io_s.ok() && !log_io_s.IsShutdownInProgress() &&
         !log_io_s.IsColumnFamilyDropped()) {
       if (total_log_size_ > 0) {
@@ -1392,7 +1416,8 @@ Status DBImpl::CompactFilesImpl(
       &compaction_job_stats, Env::Priority::USER, io_tracer_,
       kManualCompactionCanceledFalse_, db_id_, db_session_id_,
       c->column_family_data()->GetFullHistoryTsLow(), c->trim_ts(),
-      &blob_callback_);
+      &blob_callback_, &bg_compaction_scheduled_,
+      &bg_bottom_compaction_scheduled_);
 
   // Creating a compaction influences the compaction score because the score
   // takes running compactions into account (by skipping files that are already
@@ -1669,8 +1694,7 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
           f->smallest, f->largest, f->fd.smallest_seqno, f->fd.largest_seqno,
           f->marked_for_compaction, f->temperature, f->oldest_blob_file_number,
           f->oldest_ancester_time, f->file_creation_time, f->file_checksum,
-          f->file_checksum_func_name, f->min_timestamp, f->max_timestamp,
-          f->unique_id);
+          f->file_checksum_func_name, f->unique_id);
     }
     ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                     "[%s] Apply version edit:\n%s", cfd->GetName().c_str(),
@@ -1836,8 +1860,7 @@ Status DBImpl::RunManualCompaction(
   // jobs drops to zero. This used to be needed to ensure that this manual
   // compaction can compact any range of keys/files. Now it is optional
   // (see `CompactRangeOptions::exclusive_manual_compaction`). The use case for
-  // `exclusive_manual_compaction=true` (the default) is unclear beyond not
-  // trusting the new code.
+  // `exclusive_manual_compaction=true` is unclear beyond not trusting the code.
   //
   // HasPendingManualCompaction() is true when at least one thread is inside
   // RunManualCompaction(), i.e. during that time no other compaction will
@@ -3297,7 +3320,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
             f->fd.largest_seqno, f->marked_for_compaction, f->temperature,
             f->oldest_blob_file_number, f->oldest_ancester_time,
             f->file_creation_time, f->file_checksum, f->file_checksum_func_name,
-            f->min_timestamp, f->max_timestamp, f->unique_id);
+            f->unique_id);
 
         ROCKS_LOG_BUFFER(
             log_buffer,
@@ -3314,7 +3337,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
       if (start_level > 0) {
         auto vstorage = c->input_version()->storage_info();
         c->edit()->AddCompactCursor(
-            start_level, vstorage->GetNextCompactCursor(start_level));
+            start_level,
+            vstorage->GetNextCompactCursor(start_level, c->num_input_files(0)));
       }
     }
     status = versions_->LogAndApply(c->column_family_data(),
@@ -3399,7 +3423,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
         is_manual ? manual_compaction->canceled
                   : kManualCompactionCanceledFalse_,
         db_id_, db_session_id_, c->column_family_data()->GetFullHistoryTsLow(),
-        c->trim_ts(), &blob_callback_);
+        c->trim_ts(), &blob_callback_, &bg_compaction_scheduled_,
+        &bg_bottom_compaction_scheduled_);
     compaction_job.Prepare();
 
     NotifyOnCompactionBegin(c->column_family_data(), c.get(), status,
@@ -3802,11 +3827,6 @@ void DBImpl::GetSnapshotContext(
 
 Status DBImpl::WaitForCompact(bool wait_unscheduled) {
   // Wait until the compaction completes
-
-  // TODO: a bug here. This function actually does not necessarily
-  // wait for compact. It actually waits for scheduled compaction
-  // OR flush to finish.
-
   InstrumentedMutexLock l(&mutex_);
   while ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
           bg_flush_scheduled_ ||

db/db_impl/db_impl_debug.cc

@@ -223,7 +223,7 @@ void DBImpl::TEST_EndWrite(void* w) {
 }
 
 size_t DBImpl::TEST_LogsToFreeSize() {
-  InstrumentedMutexLock l(&mutex_);
+  InstrumentedMutexLock l(&log_write_mutex_);
   return logs_to_free_.size();
 }
 
@@ -302,7 +302,8 @@ size_t DBImpl::TEST_GetWalPreallocateBlockSize(
 }
 
 #ifndef ROCKSDB_LITE
-void DBImpl::TEST_WaitForStatsDumpRun(std::function<void()> callback) const {
+void DBImpl::TEST_WaitForPeridicWorkerRun(
+    std::function<void()> callback) const {
   if (periodic_work_scheduler_ != nullptr) {
     static_cast<PeriodicWorkTestScheduler*>(periodic_work_scheduler_)
         ->TEST_WaitForRun(callback);
@@ -312,6 +313,12 @@ void DBImpl::TEST_WaitForStatsDumpRun(std::function<void()> callback) const {
 PeriodicWorkTestScheduler* DBImpl::TEST_GetPeriodicWorkScheduler() const {
   return static_cast<PeriodicWorkTestScheduler*>(periodic_work_scheduler_);
 }
+
+SeqnoToTimeMapping DBImpl::TEST_GetSeqnoToTimeMapping() const {
+  InstrumentedMutexLock l(&mutex_);
+  return seqno_time_mapping_;
+}
+
 #endif  // !ROCKSDB_LITE
 
 size_t DBImpl::TEST_EstimateInMemoryStatsHistorySize() const {

db/db_impl/db_impl_experimental.cc

@@ -131,13 +131,13 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
     edit.SetColumnFamily(cfd->GetID());
     for (const auto& f : l0_files) {
       edit.DeleteFile(0, f->fd.GetNumber());
-      edit.AddFile(
-          target_level, f->fd.GetNumber(), f->fd.GetPathId(),
-          f->fd.GetFileSize(), f->smallest, f->largest, f->fd.smallest_seqno,
-          f->fd.largest_seqno, f->marked_for_compaction, f->temperature,
-          f->oldest_blob_file_number, f->oldest_ancester_time,
-          f->file_creation_time, f->file_checksum, f->file_checksum_func_name,
-          f->min_timestamp, f->max_timestamp, f->unique_id);
+      edit.AddFile(target_level, f->fd.GetNumber(), f->fd.GetPathId(),
+                   f->fd.GetFileSize(), f->smallest, f->largest,
+                   f->fd.smallest_seqno, f->fd.largest_seqno,
+                   f->marked_for_compaction, f->temperature,
+                   f->oldest_blob_file_number, f->oldest_ancester_time,
+                   f->file_creation_time, f->file_checksum,
+                   f->file_checksum_func_name, f->unique_id);
     }
 
     status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),

db/db_impl/db_impl_files.cc

@@ -271,6 +271,15 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
 
   // logs_ is empty when called during recovery, in which case there can't yet
   // be any tracked obsolete logs
+  log_write_mutex_.Lock();
+
+  if (alive_log_files_.empty() || logs_.empty()) {
+    mutex_.AssertHeld();
+    // We may reach here if the db is DBImplSecondary
+    log_write_mutex_.Unlock();
+    return;
+  }
+
   if (!alive_log_files_.empty() && !logs_.empty()) {
     uint64_t min_log_number = job_context->log_number;
     size_t num_alive_log_files = alive_log_files_.size();
@@ -292,17 +301,15 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
       }
       job_context->size_log_to_delete += earliest.size;
       total_log_size_ -= earliest.size;
-      if (two_write_queues_) {
-        log_write_mutex_.Lock();
-      }
       alive_log_files_.pop_front();
-      if (two_write_queues_) {
-        log_write_mutex_.Unlock();
-      }
+
       // Current log should always stay alive since it can't have
       // number < MinLogNumber().
       assert(alive_log_files_.size());
     }
+    log_write_mutex_.Unlock();
+    mutex_.Unlock();
+    log_write_mutex_.Lock();
     while (!logs_.empty() && logs_.front().number < min_log_number) {
       auto& log = logs_.front();
       if (log.IsSyncing()) {
@@ -311,10 +318,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
         continue;
       }
       logs_to_free_.push_back(log.ReleaseWriter());
-      {
-        InstrumentedMutexLock wl(&log_write_mutex_);
-        logs_.pop_front();
-      }
+      logs_.pop_front();
     }
     // Current log cannot be obsolete.
     assert(!logs_.empty());
@@ -323,24 +327,14 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
   // We're just cleaning up for DB::Write().
   assert(job_context->logs_to_free.empty());
   job_context->logs_to_free = logs_to_free_;
+
+  logs_to_free_.clear();
+  log_write_mutex_.Unlock();
+  mutex_.Lock();
   job_context->log_recycle_files.assign(log_recycle_files_.begin(),
                                         log_recycle_files_.end());
-  logs_to_free_.clear();
 }
 
-namespace {
-bool CompareCandidateFile(const JobContext::CandidateFileInfo& first,
-                          const JobContext::CandidateFileInfo& second) {
-  if (first.file_name > second.file_name) {
-    return true;
-  } else if (first.file_name < second.file_name) {
-    return false;
-  } else {
-    return (first.file_path > second.file_path);
-  }
-}
-}  // namespace
-
 // Delete obsolete files and log status and information of file deletion
 void DBImpl::DeleteObsoleteFileImpl(int job_id, const std::string& fname,
                                     const std::string& path_to_sync,
@@ -445,7 +439,16 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
   // dedup state.candidate_files so we don't try to delete the same
   // file twice
   std::sort(candidate_files.begin(), candidate_files.end(),
-            CompareCandidateFile);
+            [](const JobContext::CandidateFileInfo& lhs,
+               const JobContext::CandidateFileInfo& rhs) {
+              if (lhs.file_name > rhs.file_name) {
+                return true;
+              } else if (lhs.file_name < rhs.file_name) {
+                return false;
+              } else {
+                return (lhs.file_path > rhs.file_path);
+              }
+            });
   candidate_files.erase(
       std::unique(candidate_files.begin(), candidate_files.end()),
       candidate_files.end());

db/db_impl/db_impl_open.cc

@@ -441,11 +441,14 @@ Status DBImpl::Recover(
         uint64_t number = 0;
         FileType type = kWalFile;  // initialize
         if (ParseFileName(file, &number, &type) && type == kDescriptorFile) {
-          // Found MANIFEST (descriptor log), thus best-efforts recovery does
-          // not have to treat the db as empty.
-          s = Status::OK();
-          manifest_path = dbname_ + "/" + file;
-          break;
+          uint64_t bytes;
+          s = env_->GetFileSize(DescriptorFileName(dbname_, number), &bytes);
+          if (s.ok() && bytes != 0) {
+            // Found non-empty MANIFEST (descriptor log), thus best-efforts
+            // recovery does not have to treat the db as empty.
+            manifest_path = dbname_ + "/" + file;
+            break;
+          }
         }
       }
     }
@@ -538,6 +541,7 @@ Status DBImpl::Recover(
     s = CheckConsistency();
   }
   if (s.ok() && !read_only) {
+    // TODO: share file descriptors (FSDirectory) with SetDirectories above
     std::map<std::string, std::shared_ptr<FSDirectory>> created_dirs;
     for (auto cfd : *versions_->GetColumnFamilySet()) {
       s = cfd->AddDirectories(&created_dirs);
@@ -718,6 +722,12 @@ Status DBImpl::VerifySstUniqueIdInManifest() {
       status = version->VerifySstUniqueIds();
       mutex_.Lock();
       version->Unref();
+      if (!status.ok()) {
+        ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                       "SST unique id mismatch in column family \"%s\": %s",
+                       cfd->GetName().c_str(), status.ToString().c_str());
+        return status;
+      }
     }
   }
   return status;
@@ -1453,9 +1463,6 @@ Status DBImpl::RestoreAliveLogFiles(const std::vector<uint64_t>& wal_numbers) {
   Status s;
   mutex_.AssertHeld();
   assert(immutable_db_options_.avoid_flush_during_recovery);
-  if (two_write_queues_) {
-    log_write_mutex_.Lock();
-  }
   // Mark these as alive so they'll be considered for deletion later by
   // FindObsoleteFiles()
   total_log_size_ = 0;
@@ -1480,9 +1487,6 @@ Status DBImpl::RestoreAliveLogFiles(const std::vector<uint64_t>& wal_numbers) {
     total_log_size_ += log.size;
     alive_log_files_.push_back(log);
   }
-  if (two_write_queues_) {
-    log_write_mutex_.Unlock();
-  }
   return s;
 }
 
@@ -1542,7 +1546,11 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
       std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
           range_del_iters;
       auto range_del_iter =
-          mem->NewRangeTombstoneIterator(ro, kMaxSequenceNumber);
+          // This is called during recovery, where a live memtable is flushed
+          // directly. In this case, no fragmented tombstone list is cached in
+          // this memtable yet.
+          mem->NewRangeTombstoneIterator(ro, kMaxSequenceNumber,
+                                         false /* immutable_memtable */);
       if (range_del_iter != nullptr) {
         range_del_iters.emplace_back(range_del_iter);
       }
@@ -1554,17 +1562,19 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
           GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
           mutable_cf_options.compression_opts, cfd->GetID(), cfd->GetName(),
           0 /* level */, false /* is_bottommost */,
-          TableFileCreationReason::kRecovery, current_time,
-          0 /* oldest_key_time */, 0 /* file_creation_time */, db_id_,
-          db_session_id_, 0 /* target_file_size */, meta.fd.GetNumber());
+          TableFileCreationReason::kRecovery, 0 /* oldest_key_time */,
+          0 /* file_creation_time */, db_id_, db_session_id_,
+          0 /* target_file_size */, meta.fd.GetNumber());
+      SeqnoToTimeMapping empty_seqno_time_mapping;
       s = BuildTable(
           dbname_, versions_.get(), immutable_db_options_, tboptions,
           file_options_for_compaction_, cfd->table_cache(), iter.get(),
           std::move(range_del_iters), &meta, &blob_file_additions,
           snapshot_seqs, earliest_write_conflict_snapshot, kMaxSequenceNumber,
           snapshot_checker, paranoid_file_checks, cfd->internal_stats(), &io_s,
-          io_tracer_, BlobFileCreationReason::kRecovery, &event_logger_, job_id,
-          Env::IO_HIGH, nullptr /* table_properties */, write_hint,
+          io_tracer_, BlobFileCreationReason::kRecovery,
+          empty_seqno_time_mapping, &event_logger_, job_id, Env::IO_HIGH,
+          nullptr /* table_properties */, write_hint,
           nullptr /*full_history_ts_low*/, &blob_callback_);
       LogFlush(immutable_db_options_.info_log);
       ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
@@ -1595,8 +1605,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
                   meta.marked_for_compaction, meta.temperature,
                   meta.oldest_blob_file_number, meta.oldest_ancester_time,
                   meta.file_creation_time, meta.file_checksum,
-                  meta.file_checksum_func_name, meta.min_timestamp,
-                  meta.max_timestamp, meta.unique_id);
+                  meta.file_checksum_func_name, meta.unique_id);
 
     for (const auto& blob : blob_file_additions) {
       edit->AddBlobFile(blob);
@@ -1863,16 +1872,8 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
     }
 
     if (s.ok()) {
-      if (impl->two_write_queues_) {
-        impl->log_write_mutex_.Lock();
-      }
       impl->alive_log_files_.push_back(
           DBImpl::LogFileNumberSize(impl->logfile_number_));
-      if (impl->two_write_queues_) {
-        impl->log_write_mutex_.Unlock();
-      }
-    }
-    if (s.ok()) {
       // In WritePrepared there could be gap in sequence numbers. This breaks
       // the trick we use in kPointInTimeRecovery which assumes the first seq in
       // the log right after the corrupted log is one larger than the last seq
@@ -2106,6 +2107,10 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
   if (s.ok()) {
     s = impl->StartPeriodicWorkScheduler();
   }
+
+  if (s.ok()) {
+    s = impl->RegisterRecordSeqnoTimeWorker();
+  }
   if (!s.ok()) {
     for (auto* h : *handles) {
       delete h;

db/db_impl/db_impl_readonly.cc

@@ -85,17 +85,18 @@ Status DBImplReadOnly::Get(const ReadOptions& read_options,
   SequenceNumber max_covering_tombstone_seq = 0;
   LookupKey lkey(key, snapshot, read_options.timestamp);
   PERF_TIMER_STOP(get_snapshot_time);
-  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), ts, &s,
-                              &merge_context, &max_covering_tombstone_seq,
-                              read_options, &read_cb)) {
+  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(),
+                              /*columns=*/nullptr, ts, &s, &merge_context,
+                              &max_covering_tombstone_seq, read_options,
+                              false /* immutable_memtable */, &read_cb)) {
     pinnable_val->PinSelf();
     RecordTick(stats_, MEMTABLE_HIT);
   } else {
     PERF_TIMER_GUARD(get_from_output_files_time);
     PinnedIteratorsManager pinned_iters_mgr;
     super_version->current->Get(
-        read_options, lkey, pinnable_val, ts, &s, &merge_context,
-        &max_covering_tombstone_seq, &pinned_iters_mgr,
+        read_options, lkey, pinnable_val, /*columns=*/nullptr, ts, &s,
+        &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
         /*value_found*/ nullptr,
         /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb,
         /*is_blob*/ nullptr,

db/db_impl/db_impl_secondary.cc

@@ -261,6 +261,7 @@ Status DBImplSecondary::RecoverLogFiles(
             MemTable* new_mem =
                 cfd->ConstructNewMemtable(mutable_cf_options, seq_of_batch);
             cfd->mem()->SetNextLogNumber(log_number);
+            cfd->mem()->ConstructFragmentedRangeTombstones();
             cfd->imm()->Add(cfd->mem(), &job_context->memtables_to_free);
             new_mem->Ref();
             cfd->SetMemtable(new_mem);
@@ -389,16 +390,18 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
   const Comparator* ucmp = column_family->GetComparator();
   assert(ucmp);
   std::string* ts = ucmp->timestamp_size() > 0 ? timestamp : nullptr;
-  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), ts, &s,
-                              &merge_context, &max_covering_tombstone_seq,
-                              read_options, &read_cb)) {
+  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(),
+                              /*columns=*/nullptr, ts, &s, &merge_context,
+                              &max_covering_tombstone_seq, read_options,
+                              false /* immutable_memtable */, &read_cb)) {
     done = true;
     pinnable_val->PinSelf();
     RecordTick(stats_, MEMTABLE_HIT);
   } else if ((s.ok() || s.IsMergeInProgress()) &&
              super_version->imm->Get(
-                 lkey, pinnable_val->GetSelf(), ts, &s, &merge_context,
-                 &max_covering_tombstone_seq, read_options, &read_cb)) {
+                 lkey, pinnable_val->GetSelf(), /*columns=*/nullptr, ts, &s,
+                 &merge_context, &max_covering_tombstone_seq, read_options,
+                 &read_cb)) {
     done = true;
     pinnable_val->PinSelf();
     RecordTick(stats_, MEMTABLE_HIT);
@@ -411,8 +414,9 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
     PERF_TIMER_GUARD(get_from_output_files_time);
     PinnedIteratorsManager pinned_iters_mgr;
     super_version->current->Get(
-        read_options, lkey, pinnable_val, ts, &s, &merge_context,
-        &max_covering_tombstone_seq, &pinned_iters_mgr, /*value_found*/ nullptr,
+        read_options, lkey, pinnable_val, /*columns=*/nullptr, ts, &s,
+        &merge_context, &max_covering_tombstone_seq, &pinned_iters_mgr,
+        /*value_found*/ nullptr,
         /*key_exists*/ nullptr, /*seq*/ nullptr, &read_cb, /*is_blob*/ nullptr,
         /*do_merge*/ true);
     RecordTick(stats_, MEMTABLE_MISS);

db/db_impl/db_impl_write.cc

@@ -349,14 +349,11 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   // when it finds suitable, and finish them in the same write batch.
   // This is how a write job could be done by the other writer.
   WriteContext write_context;
+  LogContext log_context(write_options.sync);
   WriteThread::WriteGroup write_group;
   bool in_parallel_group = false;
   uint64_t last_sequence = kMaxSequenceNumber;
 
-  mutex_.Lock();
-
-  bool need_log_sync = write_options.sync;
-  bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
   assert(!two_write_queues_ || !disable_memtable);
   {
     // With concurrent writes we do preprocess only in the write thread that
@@ -366,7 +363,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     // PreprocessWrite does its own perf timing.
     PERF_TIMER_STOP(write_pre_and_post_process_time);
 
-    status = PreprocessWrite(write_options, &need_log_sync, &write_context);
+    status = PreprocessWrite(write_options, &log_context, &write_context);
     if (!two_write_queues_) {
       // Assign it after ::PreprocessWrite since the sequence might advance
       // inside it by WriteRecoverableState
@@ -376,13 +373,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     PERF_TIMER_START(write_pre_and_post_process_time);
   }
 
-  log::Writer* log_writer = logs_.back().writer;
-  LogFileNumberSize& log_file_number_size = alive_log_files_.back();
-
-  assert(log_writer->get_log_number() == log_file_number_size.number);
-
-  mutex_.Unlock();
-
   // Add to log and apply to memtable.  We can release the lock
   // during this phase since &w is currently responsible for logging
   // and protects against concurrent loggers and concurrent writes
@@ -477,10 +467,14 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 
     if (!two_write_queues_) {
       if (status.ok() && !write_options.disableWAL) {
+        assert(log_context.log_file_number_size);
+        LogFileNumberSize& log_file_number_size =
+            *(log_context.log_file_number_size);
         PERF_TIMER_GUARD(write_wal_time);
-        io_s = WriteToWAL(write_group, log_writer, log_used, need_log_sync,
-                          need_log_dir_sync, last_sequence + 1,
-                          log_file_number_size);
+        io_s =
+            WriteToWAL(write_group, log_context.writer, log_used,
+                       log_context.need_log_sync, log_context.need_log_dir_sync,
+                       last_sequence + 1, log_file_number_size);
       }
     } else {
       if (status.ok() && !write_options.disableWAL) {
@@ -582,14 +576,21 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     assert(pre_release_cb_status.ok());
   }
 
-  if (need_log_sync) {
-    mutex_.Lock();
+  if (log_context.need_log_sync) {
+    VersionEdit synced_wals;
+    log_write_mutex_.Lock();
     if (status.ok()) {
-      status = MarkLogsSynced(logfile_number_, need_log_dir_sync);
+      MarkLogsSynced(logfile_number_, log_context.need_log_dir_sync,
+                     &synced_wals);
     } else {
       MarkLogsNotSynced(logfile_number_);
     }
-    mutex_.Unlock();
+    log_write_mutex_.Unlock();
+    if (status.ok() && synced_wals.IsWalAddition()) {
+      InstrumentedMutexLock l(&mutex_);
+      status = ApplyWALToManifest(&synced_wals);
+    }
+
     // Requesting sync with two_write_queues_ is expected to be very rare. We
     // hence provide a simple implementation that is not necessarily efficient.
     if (two_write_queues_) {
@@ -652,19 +653,11 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
     if (w.callback && !w.callback->AllowWriteBatching()) {
       write_thread_.WaitForMemTableWriters();
     }
-    mutex_.Lock();
-    bool need_log_sync = !write_options.disableWAL && write_options.sync;
-    bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
+    LogContext log_context(!write_options.disableWAL && write_options.sync);
     // PreprocessWrite does its own perf timing.
     PERF_TIMER_STOP(write_pre_and_post_process_time);
-    w.status = PreprocessWrite(write_options, &need_log_sync, &write_context);
+    w.status = PreprocessWrite(write_options, &log_context, &write_context);
     PERF_TIMER_START(write_pre_and_post_process_time);
-    log::Writer* log_writer = logs_.back().writer;
-    LogFileNumberSize& log_file_number_size = alive_log_files_.back();
-
-    assert(log_writer->get_log_number() == log_file_number_size.number);
-
-    mutex_.Unlock();
 
     // This can set non-OK status if callback fail.
     last_batch_group_size_ =
@@ -727,9 +720,13 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
                           wal_write_group.size - 1);
         RecordTick(stats_, WRITE_DONE_BY_OTHER, wal_write_group.size - 1);
       }
+      assert(log_context.log_file_number_size);
+      LogFileNumberSize& log_file_number_size =
+          *(log_context.log_file_number_size);
       io_s =
-          WriteToWAL(wal_write_group, log_writer, log_used, need_log_sync,
-                     need_log_dir_sync, current_sequence, log_file_number_size);
+          WriteToWAL(wal_write_group, log_context.writer, log_used,
+                     log_context.need_log_sync, log_context.need_log_dir_sync,
+                     current_sequence, log_file_number_size);
       w.status = io_s;
     }
 
@@ -740,16 +737,20 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
       WriteStatusCheck(w.status);
     }
 
-    if (need_log_sync) {
-      mutex_.Lock();
+    VersionEdit synced_wals;
+    if (log_context.need_log_sync) {
+      InstrumentedMutexLock l(&log_write_mutex_);
       if (w.status.ok()) {
-        w.status = MarkLogsSynced(logfile_number_, need_log_dir_sync);
+        MarkLogsSynced(logfile_number_, log_context.need_log_dir_sync,
+                       &synced_wals);
       } else {
         MarkLogsNotSynced(logfile_number_);
       }
-      mutex_.Unlock();
     }
-
+    if (w.status.ok() && synced_wals.IsWalAddition()) {
+      InstrumentedMutexLock l(&mutex_);
+      w.status = ApplyWALToManifest(&synced_wals);
+    }
     write_thread_.ExitAsBatchGroupLeader(wal_write_group, w.status);
   }
 
@@ -893,9 +894,8 @@ Status DBImpl::WriteImplWALOnly(
     // TODO(myabandeh): Make preliminary checks thread-safe so we could do them
     // without paying the cost of obtaining the mutex.
     if (status.ok()) {
-      InstrumentedMutexLock l(&mutex_);
-      bool need_log_sync = false;
-      status = PreprocessWrite(write_options, &need_log_sync, &write_context);
+      LogContext log_context;
+      status = PreprocessWrite(write_options, &log_context, &write_context);
       WriteStatusCheckOnLocked(status);
     }
     if (!status.ok()) {
@@ -1057,9 +1057,8 @@ Status DBImpl::WriteImplWALOnly(
 void DBImpl::WriteStatusCheckOnLocked(const Status& status) {
   // Is setting bg_error_ enough here?  This will at least stop
   // compaction and fail any further writes.
-  // Caller must hold mutex_.
+  InstrumentedMutexLock l(&mutex_);
   assert(!status.IsIOFenced() || !error_handler_.GetBGError().ok());
-  mutex_.AssertHeld();
   if (immutable_db_options_.paranoid_checks && !status.ok() &&
       !status.IsBusy() && !status.IsIncomplete()) {
     // Maybe change the return status to void?
@@ -1090,6 +1089,9 @@ void DBImpl::IOStatusCheck(const IOStatus& io_status) {
     // Maybe change the return status to void?
     error_handler_.SetBGError(io_status, BackgroundErrorReason::kWriteCallback);
     mutex_.Unlock();
+  } else {
+    // Force writable file to be continue writable.
+    logs_.back().writer->file()->reset_seen_error();
   }
 }
 
@@ -1110,13 +1112,13 @@ void DBImpl::MemTableInsertStatusCheck(const Status& status) {
 }
 
 Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
-                               bool* need_log_sync,
+                               LogContext* log_context,
                                WriteContext* write_context) {
-  mutex_.AssertHeld();
-  assert(write_context != nullptr && need_log_sync != nullptr);
+  assert(write_context != nullptr && log_context != nullptr);
   Status status;
 
   if (error_handler_.IsDBStopped()) {
+    InstrumentedMutexLock l(&mutex_);
     status = error_handler_.GetBGError();
   }
 
@@ -1124,11 +1126,11 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
 
   if (UNLIKELY(status.ok() && total_log_size_ > GetMaxTotalWalSize())) {
     assert(versions_);
+    InstrumentedMutexLock l(&mutex_);
     const ColumnFamilySet* const column_families =
         versions_->GetColumnFamilySet();
     assert(column_families);
     size_t num_cfs = column_families->NumberOfColumnFamilies();
-
     assert(num_cfs >= 1);
     if (num_cfs > 1) {
       WaitForPendingWrites();
@@ -1142,15 +1144,18 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
     // thread is writing to another DB with the same write buffer, they may also
     // be flushed. We may end up with flushing much more DBs than needed. It's
     // suboptimal but still correct.
+    InstrumentedMutexLock l(&mutex_);
     WaitForPendingWrites();
     status = HandleWriteBufferManagerFlush(write_context);
   }
 
   if (UNLIKELY(status.ok() && !trim_history_scheduler_.Empty())) {
+    InstrumentedMutexLock l(&mutex_);
     status = TrimMemtableHistory(write_context);
   }
 
   if (UNLIKELY(status.ok() && !flush_scheduler_.Empty())) {
+    InstrumentedMutexLock l(&mutex_);
     WaitForPendingWrites();
     status = ScheduleFlushes(write_context);
   }
@@ -1166,6 +1171,7 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
     // for previous one. It might create a fairness issue that expiration
     // might happen for smaller writes but larger writes can go through.
     // Can optimize it if it is an issue.
+    InstrumentedMutexLock l(&mutex_);
     status = DelayWrite(last_batch_group_size_, write_options);
     PERF_TIMER_START(write_pre_and_post_process_time);
   }
@@ -1180,11 +1186,12 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
     if (write_options.no_slowdown) {
       status = Status::Incomplete("Write stall");
     } else {
+      InstrumentedMutexLock l(&mutex_);
       WriteBufferManagerStallWrites();
     }
   }
-
-  if (status.ok() && *need_log_sync) {
+  InstrumentedMutexLock l(&log_write_mutex_);
+  if (status.ok() && log_context->need_log_sync) {
     // Wait until the parallel syncs are finished. Any sync process has to sync
     // the front log too so it is enough to check the status of front()
     // We do a while loop since log_sync_cv_ is signalled when any sync is
@@ -1204,8 +1211,12 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
       log.PrepareForSync();
     }
   } else {
-    *need_log_sync = false;
+    log_context->need_log_sync = false;
   }
+  log_context->writer = logs_.back().writer;
+  log_context->need_log_dir_sync =
+      log_context->need_log_dir_sync && !log_dir_synced_;
+  log_context->log_file_number_size = std::addressof(alive_log_files_.back());
 
   return status;
 }
@@ -1637,12 +1648,6 @@ Status DBImpl::HandleWriteBufferManagerFlush(WriteContext* write_context) {
   // thread is writing to another DB with the same write buffer, they may also
   // be flushed. We may end up with flushing much more DBs than needed. It's
   // suboptimal but still correct.
-  ROCKS_LOG_INFO(
-      immutable_db_options_.info_log,
-      "Flushing column family with oldest memtable entry. Write buffers are "
-      "using %" ROCKSDB_PRIszt " bytes out of a total of %" ROCKSDB_PRIszt ".",
-      write_buffer_manager_->memory_usage(),
-      write_buffer_manager_->buffer_size());
   // no need to refcount because drop is happening in write thread, so can't
   // happen while we're in the write thread
   autovector<ColumnFamilyData*> cfds;
@@ -1656,9 +1661,11 @@ Status DBImpl::HandleWriteBufferManagerFlush(WriteContext* write_context) {
       if (cfd->IsDropped()) {
         continue;
       }
-      if (!cfd->mem()->IsEmpty()) {
-        // We only consider active mem table, hoping immutable memtable is
-        // already in the process of flushing.
+      if (!cfd->mem()->IsEmpty() && !cfd->imm()->IsFlushPendingOrRunning()) {
+        // We only consider flush on CFs with bytes in the mutable memtable,
+        // and no immutable memtables for which flush has yet to finish. If
+        // we triggered flush on CFs already trying to flush, we would risk
+        // creating too many immutable memtables leading to write stalls.
         uint64_t seq = cfd->mem()->GetCreationSeq();
         if (cfd_picked == nullptr || seq < seq_num_for_cf_picked) {
           cfd_picked = cfd;
@@ -1671,6 +1678,15 @@ Status DBImpl::HandleWriteBufferManagerFlush(WriteContext* write_context) {
     }
     MaybeFlushStatsCF(&cfds);
   }
+  if (!cfds.empty()) {
+    ROCKS_LOG_INFO(
+        immutable_db_options_.info_log,
+        "Flushing triggered to alleviate write buffer memory usage. Write "
+        "buffer is using %" ROCKSDB_PRIszt
+        " bytes out of a total of %" ROCKSDB_PRIszt ".",
+        write_buffer_manager_->memory_usage(),
+        write_buffer_manager_->buffer_size());
+  }
 
   WriteThread::Writer nonmem_w;
   if (two_write_queues_) {
@@ -1714,10 +1730,12 @@ Status DBImpl::HandleWriteBufferManagerFlush(WriteContext* write_context) {
 }
 
 uint64_t DBImpl::GetMaxTotalWalSize() const {
-  mutex_.AssertHeld();
-  return mutable_db_options_.max_total_wal_size == 0
-             ? 4 * max_total_in_memory_state_
-             : mutable_db_options_.max_total_wal_size;
+  uint64_t max_total_wal_size =
+      max_total_wal_size_.load(std::memory_order_acquire);
+  if (max_total_wal_size > 0) {
+    return max_total_wal_size;
+  }
+  return 4 * max_total_in_memory_state_.load(std::memory_order_acquire);
 }
 
 // REQUIRES: mutex_ is held
@@ -2055,6 +2073,10 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
                  "[%s] New memtable created with log file: #%" PRIu64
                  ". Immutable memtables: %d.\n",
                  cfd->GetName().c_str(), new_log_number, num_imm_unflushed);
+  // There should be no concurrent write as the thread is at the front of
+  // writer queue
+  cfd->mem()->ConstructFragmentedRangeTombstones();
+
   mutex_.Lock();
   if (recycle_log_number != 0) {
     // Since renaming the file is done outside DB mutex, we need to ensure
@@ -2065,11 +2087,15 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
     log_recycle_files_.pop_front();
   }
   if (s.ok() && creating_new_log) {
-    log_write_mutex_.Lock();
+    InstrumentedMutexLock l(&log_write_mutex_);
     assert(new_log != nullptr);
     if (!logs_.empty()) {
       // Alway flush the buffer of the last log before switching to a new one
       log::Writer* cur_log_writer = logs_.back().writer;
+      if (error_handler_.IsRecoveryInProgress()) {
+        // In recovery path, we force another try of writing WAL buffer.
+        cur_log_writer->file()->reset_seen_error();
+      }
       io_s = cur_log_writer->WriteBuffer();
       if (s.ok()) {
         s = io_s;
@@ -2089,7 +2115,6 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
       logs_.emplace_back(logfile_number_, new_log);
       alive_log_files_.push_back(LogFileNumberSize(logfile_number_));
     }
-    log_write_mutex_.Unlock();
   }
 
   if (!s.ok()) {

db/db_iter.cc

@@ -17,6 +17,7 @@
 #include "db/merge_context.h"
 #include "db/merge_helper.h"
 #include "db/pinned_iterators_manager.h"
+#include "db/wide/wide_column_serialization.h"
 #include "file/filename.h"
 #include "logging/logging.h"
 #include "memory/arena.h"
@@ -71,9 +72,11 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options,
                                      read_options.total_order_seek ||
                                      read_options.auto_prefix_mode),
       read_tier_(read_options.read_tier),
+      fill_cache_(read_options.fill_cache),
       verify_checksums_(read_options.verify_checksums),
       expose_blob_index_(expose_blob_index),
       is_blob_(false),
+      is_wide_(false),
       arena_mode_(arena_mode),
       range_del_agg_(&ioptions.internal_comparator, s),
       db_impl_(db_impl),
@@ -131,6 +134,8 @@ void DBIter::Next() {
   PERF_CPU_TIMER_GUARD(iter_next_cpu_nanos, clock_);
   // Release temporarily pinned blocks from last operation
   ReleaseTempPinnedData();
+  ResetBlobValue();
+  ResetWideColumnValue();
   local_stats_.skip_count_ += num_internal_keys_skipped_;
   local_stats_.skip_count_--;
   num_internal_keys_skipped_ = 0;
@@ -173,6 +178,9 @@ void DBIter::Next() {
 bool DBIter::SetBlobValueIfNeeded(const Slice& user_key,
                                   const Slice& blob_index) {
   assert(!is_blob_);
+  assert(blob_value_.empty());
+  assert(!is_wide_);
+  assert(value_of_default_column_.empty());
 
   if (expose_blob_index_) {  // Stacked BlobDB implementation
     is_blob_ = true;
@@ -189,6 +197,7 @@ bool DBIter::SetBlobValueIfNeeded(const Slice& user_key,
   // avoid having to copy options back and forth.
   ReadOptions read_options;
   read_options.read_tier = read_tier_;
+  read_options.fill_cache = fill_cache_;
   read_options.verify_checksums = verify_checksums_;
 
   constexpr FilePrefetchBuffer* prefetch_buffer = nullptr;
@@ -207,13 +216,25 @@ bool DBIter::SetBlobValueIfNeeded(const Slice& user_key,
   return true;
 }
 
-bool DBIter::SetWideColumnValueIfNeeded(const Slice& /* wide_columns_slice */) {
+bool DBIter::SetWideColumnValueIfNeeded(const Slice& wide_columns_slice) {
   assert(!is_blob_);
+  assert(blob_value_.empty());
+  assert(!is_wide_);
+  assert(value_of_default_column_.empty());
 
-  // TODO: support wide-column entities
-  status_ = Status::NotSupported("Encountered unexpected wide-column entity");
-  valid_ = false;
-  return false;
+  Slice wide_columns_copy = wide_columns_slice;
+
+  const Status s = WideColumnSerialization::GetValueOfDefaultColumn(
+      wide_columns_copy, value_of_default_column_);
+
+  if (!s.ok()) {
+    status_ = s;
+    valid_ = false;
+    return false;
+  }
+
+  is_wide_ = true;
+  return true;
 }
 
 // PRE: saved_key_ has the current user key if skipping_saved_key
@@ -262,7 +283,10 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
   // to one.
   bool reseek_done = false;
 
-  is_blob_ = false;
+  assert(!is_blob_);
+  assert(blob_value_.empty());
+  assert(!is_wide_);
+  assert(value_of_default_column_.empty());
 
   do {
     // Will update is_key_seqnum_zero_ as soon as we parsed the current key
@@ -590,7 +614,11 @@ bool DBIter::MergeValuesNewToOld() {
       if (!s.ok()) {
         return false;
       }
-      is_blob_ = false;
+
+      ResetBlobValue();
+      assert(!is_wide_);
+      assert(value_of_default_column_.empty());
+
       // iter_ is positioned after put
       iter_.Next();
       if (!iter_.status().ok()) {
@@ -636,6 +664,8 @@ void DBIter::Prev() {
 
   PERF_CPU_TIMER_GUARD(iter_prev_cpu_nanos, clock_);
   ReleaseTempPinnedData();
+  ResetBlobValue();
+  ResetWideColumnValue();
   ResetInternalKeysSkippedCounter();
   bool ok = true;
   if (direction_ == kForward) {
@@ -965,7 +995,12 @@ bool DBIter::FindValueForCurrentKey() {
 
   Status s;
   s.PermitUncheckedError();
-  is_blob_ = false;
+
+  assert(!is_blob_);
+  assert(blob_value_.empty());
+  assert(!is_wide_);
+  assert(value_of_default_column_.empty());
+
   switch (last_key_entry_type) {
     case kTypeDeletion:
     case kTypeDeletionWithTimestamp:
@@ -1004,7 +1039,11 @@ bool DBIter::FindValueForCurrentKey() {
         if (!s.ok()) {
           return false;
         }
-        is_blob_ = false;
+
+        ResetBlobValue();
+        assert(!is_wide_);
+        assert(value_of_default_column_.empty());
+
         return true;
       } else if (last_not_merge_type == kTypeWideColumnEntity) {
         // TODO: support wide-column entities
@@ -1079,7 +1118,12 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   // In case read_callback presents, the value we seek to may not be visible.
   // Find the next value that's visible.
   ParsedInternalKey ikey;
-  is_blob_ = false;
+
+  assert(!is_blob_);
+  assert(blob_value_.empty());
+  assert(!is_wide_);
+  assert(value_of_default_column_.empty());
+
   while (true) {
     if (!iter_.Valid()) {
       valid_ = false;
@@ -1214,7 +1258,11 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
       if (!s.ok()) {
         return false;
       }
-      is_blob_ = false;
+
+      ResetBlobValue();
+      assert(!is_wide_);
+      assert(value_of_default_column_.empty());
+
       return true;
     } else if (ikey.type == kTypeWideColumnEntity) {
       // TODO: support wide-column entities
@@ -1439,6 +1487,8 @@ void DBIter::Seek(const Slice& target) {
 
   status_ = Status::OK();
   ReleaseTempPinnedData();
+  ResetBlobValue();
+  ResetWideColumnValue();
   ResetInternalKeysSkippedCounter();
 
   // Seek the inner iterator based on the target key.
@@ -1515,6 +1565,8 @@ void DBIter::SeekForPrev(const Slice& target) {
 
   status_ = Status::OK();
   ReleaseTempPinnedData();
+  ResetBlobValue();
+  ResetWideColumnValue();
   ResetInternalKeysSkippedCounter();
 
   // Seek the inner iterator based on the target key.
@@ -1572,6 +1624,8 @@ void DBIter::SeekToFirst() {
   status_ = Status::OK();
   direction_ = kForward;
   ReleaseTempPinnedData();
+  ResetBlobValue();
+  ResetWideColumnValue();
   ResetInternalKeysSkippedCounter();
   ClearSavedValue();
   is_key_seqnum_zero_ = false;
@@ -1619,6 +1673,8 @@ void DBIter::SeekToLast() {
                                *iterate_upper_bound_, /*a_has_ts=*/false, k,
                                /*b_has_ts=*/false)) {
       ReleaseTempPinnedData();
+      ResetBlobValue();
+      ResetWideColumnValue();
       PrevInternal(nullptr);
 
       k = key();
@@ -1638,6 +1694,8 @@ void DBIter::SeekToLast() {
   status_ = Status::OK();
   direction_ = kReverse;
   ReleaseTempPinnedData();
+  ResetBlobValue();
+  ResetWideColumnValue();
   ResetInternalKeysSkippedCounter();
   ClearSavedValue();
   is_key_seqnum_zero_ = false;

db/db_iter.h

@@ -160,9 +160,12 @@ class DBIter final : public Iterator {
   }
   Slice value() const override {
     assert(valid_);
+    assert(!is_blob_ || !is_wide_);
 
     if (!expose_blob_index_ && is_blob_) {
       return blob_value_;
+    } else if (is_wide_) {
+      return value_of_default_column_;
     } else if (current_entry_is_merged_) {
       // If pinned_value_ is set then the result of merge operator is one of
       // the merge operands and we should return it.
@@ -300,8 +303,18 @@ class DBIter final : public Iterator {
   // index when using the integrated BlobDB implementation.
   bool SetBlobValueIfNeeded(const Slice& user_key, const Slice& blob_index);
 
+  void ResetBlobValue() {
+    is_blob_ = false;
+    blob_value_.Reset();
+  }
+
   bool SetWideColumnValueIfNeeded(const Slice& wide_columns_slice);
 
+  void ResetWideColumnValue() {
+    is_wide_ = false;
+    value_of_default_column_.clear();
+  }
+
   Status Merge(const Slice* val, const Slice& user_key);
 
   const SliceTransform* prefix_extractor_;
@@ -326,6 +339,7 @@ class DBIter final : public Iterator {
   Slice pinned_value_;
   // for prefix seek mode to support prev()
   PinnableSlice blob_value_;
+  Slice value_of_default_column_;
   Statistics* statistics_;
   uint64_t max_skip_;
   uint64_t max_skippable_internal_keys_;
@@ -356,11 +370,13 @@ class DBIter final : public Iterator {
   // prefix_extractor_ must be non-NULL if the value is false.
   const bool expect_total_order_inner_iter_;
   ReadTier read_tier_;
+  bool fill_cache_;
   bool verify_checksums_;
   // Whether the iterator is allowed to expose blob references. Set to true when
   // the stacked BlobDB implementation is used, false otherwise.
   bool expose_blob_index_;
   bool is_blob_;
+  bool is_wide_;
   bool arena_mode_;
   // List of operands for merge operator.
   MergeContext merge_context_;

db/db_kv_checksum_test.cc

@@ -13,9 +13,9 @@ enum class WriteBatchOpType {
   kPut = 0,
   kDelete,
   kSingleDelete,
-  kDeleteRange,
   kMerge,
   kPutEntity,
+  kDeleteRange,
   kNum,
 };
 
@@ -26,11 +26,14 @@ WriteBatchOpType operator+(WriteBatchOpType lhs, const int rhs) {
 }
 
 enum class WriteMode {
+  // `Write()` a `WriteBatch` constructed with `protection_bytes_per_key = 0`
+  // and `WriteOptions::protection_bytes_per_key = 0`
+  kWriteUnprotectedBatch = 0,
   // `Write()` a `WriteBatch` constructed with `protection_bytes_per_key > 0`.
-  kWriteProtectedBatch = 0,
+  kWriteProtectedBatch,
   // `Write()` a `WriteBatch` constructed with `protection_bytes_per_key == 0`.
   // Protection is enabled via `WriteOptions::protection_bytes_per_key > 0`.
-  kWriteUnprotectedBatch,
+  kWriteOptionProtectedBatch,
   // TODO(ajkr): add a mode that uses `Write()` wrappers, e.g., `Put()`.
   kNum,
 };
@@ -89,19 +92,30 @@ class DbKvChecksumTestBase : public DBTestBase {
   }
 };
 
-class DbKvChecksumTest : public DbKvChecksumTestBase,
-                         public ::testing::WithParamInterface<
-                             std::tuple<WriteBatchOpType, char, WriteMode>> {
+class DbKvChecksumTest
+    : public DbKvChecksumTestBase,
+      public ::testing::WithParamInterface<
+          std::tuple<WriteBatchOpType, char, WriteMode,
+                     uint32_t /* memtable_protection_bytes_per_key */>> {
  public:
   DbKvChecksumTest()
       : DbKvChecksumTestBase("db_kv_checksum_test", /*env_do_fsync=*/false) {
     op_type_ = std::get<0>(GetParam());
     corrupt_byte_addend_ = std::get<1>(GetParam());
     write_mode_ = std::get<2>(GetParam());
+    memtable_protection_bytes_per_key_ = std::get<3>(GetParam());
   }
 
   Status ExecuteWrite(ColumnFamilyHandle* cf_handle) {
     switch (write_mode_) {
+      case WriteMode::kWriteUnprotectedBatch: {
+        auto batch_and_status =
+            GetWriteBatch(GetCFHandleToUse(cf_handle, op_type_),
+                          0 /* protection_bytes_per_key */, op_type_);
+        assert(batch_and_status.second.ok());
+        // Default write option has protection_bytes_per_key = 0
+        return db_->Write(WriteOptions(), &batch_and_status.first);
+      }
       case WriteMode::kWriteProtectedBatch: {
         auto batch_and_status =
             GetWriteBatch(GetCFHandleToUse(cf_handle, op_type_),
@@ -109,7 +123,7 @@ class DbKvChecksumTest : public DbKvChecksumTestBase,
         assert(batch_and_status.second.ok());
         return db_->Write(WriteOptions(), &batch_and_status.first);
       }
-      case WriteMode::kWriteUnprotectedBatch: {
+      case WriteMode::kWriteOptionProtectedBatch: {
         auto batch_and_status =
             GetWriteBatch(GetCFHandleToUse(cf_handle, op_type_),
                           0 /* protection_bytes_per_key */, op_type_);
@@ -131,8 +145,6 @@ class DbKvChecksumTest : public DbKvChecksumTestBase,
       // We learn the entry size on the first attempt
       entry_len_ = encoded.size();
     }
-    // All entries should be the same size
-    assert(entry_len_ == encoded.size());
     char* buf = const_cast<char*>(encoded.data());
     buf[corrupt_byte_offset_] += corrupt_byte_addend_;
     ++corrupt_byte_offset_;
@@ -144,6 +156,7 @@ class DbKvChecksumTest : public DbKvChecksumTestBase,
   WriteBatchOpType op_type_;
   char corrupt_byte_addend_;
   WriteMode write_mode_;
+  uint32_t memtable_protection_bytes_per_key_;
   size_t corrupt_byte_offset_ = 0;
   size_t entry_len_ = std::numeric_limits<size_t>::max();
 };
@@ -169,29 +182,36 @@ std::string GetOpTypeString(const WriteBatchOpType& op_type) {
   return "";
 }
 
+std::string GetWriteModeString(const WriteMode& mode) {
+  switch (mode) {
+    case WriteMode::kWriteUnprotectedBatch:
+      return "WriteUnprotectedBatch";
+    case WriteMode::kWriteProtectedBatch:
+      return "WriteProtectedBatch";
+    case WriteMode::kWriteOptionProtectedBatch:
+      return "kWriteOptionProtectedBatch";
+    case WriteMode::kNum:
+      assert(false);
+  }
+  return "";
+}
+
 INSTANTIATE_TEST_CASE_P(
     DbKvChecksumTest, DbKvChecksumTest,
     ::testing::Combine(::testing::Range(static_cast<WriteBatchOpType>(0),
                                         WriteBatchOpType::kNum),
                        ::testing::Values(2, 103, 251),
-                       ::testing::Range(static_cast<WriteMode>(0),
-                                        WriteMode::kNum)),
+                       ::testing::Range(WriteMode::kWriteProtectedBatch,
+                                        WriteMode::kNum),
+                       ::testing::Values(0)),
     [](const testing::TestParamInfo<
-        std::tuple<WriteBatchOpType, char, WriteMode>>& args) {
+        std::tuple<WriteBatchOpType, char, WriteMode, uint32_t>>& args) {
       std::ostringstream oss;
       oss << GetOpTypeString(std::get<0>(args.param)) << "Add"
           << static_cast<int>(
-                 static_cast<unsigned char>(std::get<1>(args.param)));
-      switch (std::get<2>(args.param)) {
-        case WriteMode::kWriteProtectedBatch:
-          oss << "WriteProtectedBatch";
-          break;
-        case WriteMode::kWriteUnprotectedBatch:
-          oss << "WriteUnprotectedBatch";
-          break;
-        case WriteMode::kNum:
-          assert(false);
-      }
+                 static_cast<unsigned char>(std::get<1>(args.param)))
+          << GetWriteModeString(std::get<2>(args.param))
+          << static_cast<uint32_t>(std::get<3>(args.param));
       return oss.str();
     });
 
@@ -660,6 +680,202 @@ TEST_F(DbKVChecksumWALToWriteBatchTest, WriteBatchChecksumHandoff) {
   SyncPoint::GetInstance()->DisableProcessing();
 };
 
+// TODO (cbi): add DeleteRange coverage once it is implemented
+class DbMemtableKVChecksumTest : public DbKvChecksumTest {
+ public:
+  DbMemtableKVChecksumTest() : DbKvChecksumTest() {}
+
+ protected:
+  // Indices in the memtable entry that we will not corrupt.
+  // For memtable entry format, see comments in MemTable::Add().
+  // We do not corrupt key length and value length fields in this test
+  // case since it causes segfault and ASAN will complain.
+  // For this test case, key and value are all of length 3, so
+  // key length field is at index 0 and value length field is at index 12.
+  const std::set<size_t> index_not_to_corrupt{0, 12};
+
+  void SkipNotToCorruptEntry() {
+    if (index_not_to_corrupt.find(corrupt_byte_offset_) !=
+        index_not_to_corrupt.end()) {
+      corrupt_byte_offset_++;
+    }
+  }
+};
+
+INSTANTIATE_TEST_CASE_P(
+    DbMemtableKVChecksumTest, DbMemtableKVChecksumTest,
+    ::testing::Combine(::testing::Range(static_cast<WriteBatchOpType>(0),
+                                        WriteBatchOpType::kDeleteRange),
+                       ::testing::Values(2, 103, 251),
+                       ::testing::Range(static_cast<WriteMode>(0),
+                                        WriteMode::kWriteOptionProtectedBatch),
+                       // skip 1 byte checksum as it makes test flaky
+                       ::testing::Values(2, 4, 8)),
+    [](const testing::TestParamInfo<
+        std::tuple<WriteBatchOpType, char, WriteMode, uint32_t>>& args) {
+      std::ostringstream oss;
+      oss << GetOpTypeString(std::get<0>(args.param)) << "Add"
+          << static_cast<int>(
+                 static_cast<unsigned char>(std::get<1>(args.param)))
+          << GetWriteModeString(std::get<2>(args.param))
+          << static_cast<uint32_t>(std::get<3>(args.param));
+      return oss.str();
+    });
+
+TEST_P(DbMemtableKVChecksumTest, GetWithCorruptAfterMemtableInsert) {
+  // Record memtable entry size.
+  // Not corrupting memtable entry here since it will segfault
+  // or fail some asserts inside memtablerep implementation
+  // e.g., when key_len is corrupted.
+  SyncPoint::GetInstance()->SetCallBack(
+      "MemTable::Add:BeforeReturn:Encoded", [&](void* arg) {
+        Slice encoded = *static_cast<Slice*>(arg);
+        entry_len_ = encoded.size();
+      });
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "Memtable::SaveValue:Begin:entry", [&](void* entry) {
+        char* buf = *static_cast<char**>(entry);
+        buf[corrupt_byte_offset_] += corrupt_byte_addend_;
+        ++corrupt_byte_offset_;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  Options options = CurrentOptions();
+  options.memtable_protection_bytes_per_key =
+      memtable_protection_bytes_per_key_;
+  if (op_type_ == WriteBatchOpType::kMerge) {
+    options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  }
+
+  SkipNotToCorruptEntry();
+  while (MoreBytesToCorrupt()) {
+    Reopen(options);
+    ASSERT_OK(ExecuteWrite(nullptr));
+    std::string val;
+    ASSERT_TRUE(db_->Get(ReadOptions(), "key", &val).IsCorruption());
+    Destroy(options);
+    SkipNotToCorruptEntry();
+  }
+}
+
+TEST_P(DbMemtableKVChecksumTest,
+       GetWithColumnFamilyCorruptAfterMemtableInsert) {
+  // Record memtable entry size.
+  // Not corrupting memtable entry here since it will segfault
+  // or fail some asserts inside memtablerep implementation
+  // e.g., when key_len is corrupted.
+  SyncPoint::GetInstance()->SetCallBack(
+      "MemTable::Add:BeforeReturn:Encoded", [&](void* arg) {
+        Slice encoded = *static_cast<Slice*>(arg);
+        entry_len_ = encoded.size();
+      });
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "Memtable::SaveValue:Begin:entry", [&](void* entry) {
+        char* buf = *static_cast<char**>(entry);
+        buf[corrupt_byte_offset_] += corrupt_byte_addend_;
+        ++corrupt_byte_offset_;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  Options options = CurrentOptions();
+  options.memtable_protection_bytes_per_key =
+      memtable_protection_bytes_per_key_;
+  if (op_type_ == WriteBatchOpType::kMerge) {
+    options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  }
+
+  SkipNotToCorruptEntry();
+  while (MoreBytesToCorrupt()) {
+    Reopen(options);
+    CreateAndReopenWithCF({"pikachu"}, options);
+    ASSERT_OK(ExecuteWrite(handles_[1]));
+    std::string val;
+    ASSERT_TRUE(
+        db_->Get(ReadOptions(), handles_[1], "key", &val).IsCorruption());
+    Destroy(options);
+    SkipNotToCorruptEntry();
+  }
+}
+
+TEST_P(DbMemtableKVChecksumTest, IteratorWithCorruptAfterMemtableInsert) {
+  SyncPoint::GetInstance()->SetCallBack(
+      "MemTable::Add:BeforeReturn:Encoded",
+      std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this,
+                std::placeholders::_1));
+  SyncPoint::GetInstance()->EnableProcessing();
+  Options options = CurrentOptions();
+  options.memtable_protection_bytes_per_key =
+      memtable_protection_bytes_per_key_;
+  if (op_type_ == WriteBatchOpType::kMerge) {
+    options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  }
+
+  SkipNotToCorruptEntry();
+  while (MoreBytesToCorrupt()) {
+    Reopen(options);
+    ASSERT_OK(ExecuteWrite(nullptr));
+    Iterator* it = db_->NewIterator(ReadOptions());
+    it->SeekToFirst();
+    ASSERT_FALSE(it->Valid());
+    ASSERT_TRUE(it->status().IsCorruption());
+    delete it;
+    Destroy(options);
+    SkipNotToCorruptEntry();
+  }
+}
+
+TEST_P(DbMemtableKVChecksumTest,
+       IteratorWithColumnFamilyCorruptAfterMemtableInsert) {
+  SyncPoint::GetInstance()->SetCallBack(
+      "MemTable::Add:BeforeReturn:Encoded",
+      std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this,
+                std::placeholders::_1));
+  SyncPoint::GetInstance()->EnableProcessing();
+  Options options = CurrentOptions();
+  options.memtable_protection_bytes_per_key =
+      memtable_protection_bytes_per_key_;
+  if (op_type_ == WriteBatchOpType::kMerge) {
+    options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  }
+
+  SkipNotToCorruptEntry();
+  while (MoreBytesToCorrupt()) {
+    Reopen(options);
+    CreateAndReopenWithCF({"pikachu"}, options);
+    ASSERT_OK(ExecuteWrite(handles_[1]));
+    Iterator* it = db_->NewIterator(ReadOptions(), handles_[1]);
+    it->SeekToFirst();
+    ASSERT_FALSE(it->Valid());
+    ASSERT_TRUE(it->status().IsCorruption());
+    delete it;
+    Destroy(options);
+    SkipNotToCorruptEntry();
+  }
+}
+
+TEST_P(DbMemtableKVChecksumTest, FlushWithCorruptAfterMemtableInsert) {
+  SyncPoint::GetInstance()->SetCallBack(
+      "MemTable::Add:BeforeReturn:Encoded",
+      std::bind(&DbKvChecksumTest::CorruptNextByteCallBack, this,
+                std::placeholders::_1));
+  SyncPoint::GetInstance()->EnableProcessing();
+  Options options = CurrentOptions();
+  options.memtable_protection_bytes_per_key =
+      memtable_protection_bytes_per_key_;
+  if (op_type_ == WriteBatchOpType::kMerge) {
+    options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  }
+
+  SkipNotToCorruptEntry();
+  // Not corruping each byte like other tests since Flush() is relatively slow.
+  Reopen(options);
+  ASSERT_OK(ExecuteWrite(nullptr));
+  ASSERT_TRUE(Flush().IsCorruption());
+  // DB enters read-only state when flush reads corrupted data
+  ASSERT_TRUE(dbfull()->TEST_GetBGError().IsCorruption());
+  Destroy(options);
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_memtable_test.cc

@@ -262,8 +262,9 @@ TEST_F(DBMemTableTest, ConcurrentMergeWrite) {
   ReadOptions roptions;
   SequenceNumber max_covering_tombstone_seq = 0;
   LookupKey lkey("key", kMaxSequenceNumber);
-  bool res = mem->Get(lkey, &value, /*timestamp=*/nullptr, &status,
-                      &merge_context, &max_covering_tombstone_seq, roptions);
+  bool res = mem->Get(lkey, &value, /*columns=*/nullptr, /*timestamp=*/nullptr,
+                      &status, &merge_context, &max_covering_tombstone_seq,
+                      roptions, false /* immutable_memtable */);
   ASSERT_OK(status);
   ASSERT_TRUE(res);
   uint64_t ivalue = DecodeFixed64(Slice(value).data());

db/db_merge_operand_test.cc

@@ -397,6 +397,48 @@ TEST_F(DBMergeOperandTest, BlobDBGetMergeOperandsBasic) {
   ASSERT_EQ(values[3], "ed");
 }
 
+TEST_F(DBMergeOperandTest, GetMergeOperandsLargeResultOptimization) {
+  // These constants are chosen to trigger the large result optimization
+  // (pinning a bundle of `DBImpl` resources).
+  const int kNumOperands = 1024;
+  const int kOperandLen = 1024;
+
+  Options options;
+  options.create_if_missing = true;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  std::vector<std::string> expected_merge_operands;
+  expected_merge_operands.reserve(kNumOperands);
+  for (int i = 0; i < kNumOperands; ++i) {
+    expected_merge_operands.emplace_back(rnd.RandomString(kOperandLen));
+    ASSERT_OK(Merge("key", expected_merge_operands.back()));
+  }
+
+  std::vector<PinnableSlice> merge_operands(kNumOperands);
+  GetMergeOperandsOptions merge_operands_info;
+  merge_operands_info.expected_max_number_of_operands = kNumOperands;
+  int num_merge_operands = 0;
+  ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                  "key", merge_operands.data(),
+                                  &merge_operands_info, &num_merge_operands));
+  ASSERT_EQ(num_merge_operands, kNumOperands);
+
+  // Ensures the large result optimization was used.
+  for (int i = 0; i < kNumOperands; ++i) {
+    ASSERT_TRUE(merge_operands[i].IsPinned());
+  }
+
+  // Add a Flush() to change the `SuperVersion` to challenge the resource
+  // pinning.
+  ASSERT_OK(Flush());
+
+  for (int i = 0; i < kNumOperands; ++i) {
+    ASSERT_EQ(expected_merge_operands[i], merge_operands[i]);
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_test.cc

@@ -492,7 +492,7 @@ TEST_F(DBTest, PutSingleDeleteGet) {
     ASSERT_EQ("v2", Get(1, "foo2"));
     ASSERT_OK(SingleDelete(1, "foo"));
     ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
-    // Skip FIFO and universal compaction beccause they do not apply to the test
+    // Skip FIFO and universal compaction because they do not apply to the test
     // case. Skip MergePut because single delete does not get removed when it
     // encounters a merge.
   } while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction |
@@ -645,7 +645,7 @@ TEST_F(DBTest, SingleDeleteFlush) {
 
     ASSERT_EQ("NOT_FOUND", Get(1, "bar"));
     ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
-    // Skip FIFO and universal compaction beccause they do not apply to the test
+    // Skip FIFO and universal compaction beccaus they do not apply to the test
     // case. Skip MergePut because single delete does not get removed when it
     // encounters a merge.
   } while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction |
@@ -668,7 +668,7 @@ TEST_F(DBTest, SingleDeletePutFlush) {
     ASSERT_OK(Flush(1));
 
     ASSERT_EQ("[ ]", AllEntriesFor("a", 1));
-    // Skip FIFO and universal compaction beccause they do not apply to the test
+    // Skip FIFO and universal compaction beccaus they do not apply to the test
     // case. Skip MergePut because single delete does not get removed when it
     // encounters a merge.
   } while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction |
@@ -1993,7 +1993,7 @@ TEST_F(DBTest, UnremovableSingleDelete) {
     ASSERT_EQ("first", Get(1, "foo", snapshot));
     ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
     db_->ReleaseSnapshot(snapshot);
-    // Skip FIFO and universal compaction beccause they do not apply to the test
+    // Skip FIFO and universal compaction because they do not apply to the test
     // case. Skip MergePut because single delete does not get removed when it
     // encounters a merge.
   } while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction |
@@ -2773,7 +2773,7 @@ INSTANTIATE_TEST_CASE_P(
 #endif  // ROCKSDB_LITE
 
 // Group commit test:
-#if !defined(TRAVIS) && !defined(OS_WIN)
+#if !defined(OS_WIN)
 // Disable this test temporarily on Travis and appveyor as it fails
 // intermittently. Github issue: #4151
 namespace {
@@ -2850,7 +2850,7 @@ TEST_F(DBTest, GroupCommitTest) {
     ASSERT_GT(hist_data.average, 0.0);
   } while (ChangeOptions(kSkipNoSeekToLast));
 }
-#endif  // TRAVIS
+#endif  // OS_WIN
 
 namespace {
 using KVMap = std::map<std::string, std::string>;
@@ -4101,9 +4101,6 @@ class MockedRateLimiterWithNoOptionalAPIImpl : public RateLimiter {
 
   ~MockedRateLimiterWithNoOptionalAPIImpl() override {}
 
-  const char* Name() const override {
-    return "MockedRateLimiterWithNoOptionalAPI";
-  }
   void SetBytesPerSecond(int64_t bytes_per_second) override {
     (void)bytes_per_second;
   }
@@ -5971,7 +5968,7 @@ TEST_F(DBTest, EmptyCompactedDB) {
 #endif  // ROCKSDB_LITE
 
 #ifndef ROCKSDB_LITE
-TEST_F(DBTest, DISABLED_SuggestCompactRangeTest) {
+TEST_F(DBTest, SuggestCompactRangeTest) {
   class CompactionFilterFactoryGetContext : public CompactionFilterFactory {
    public:
     std::unique_ptr<CompactionFilter> CreateCompactionFilter(
@@ -6010,41 +6007,17 @@ TEST_F(DBTest, DISABLED_SuggestCompactRangeTest) {
 
   Random rnd(301);
 
-  for (int num = 0; num < 3; num++) {
+  for (int num = 0; num < 10; num++) {
     GenerateNewRandomFile(&rnd);
   }
 
-  GenerateNewRandomFile(&rnd);
-  ASSERT_EQ("0,4", FilesPerLevel(0));
   ASSERT_TRUE(!CompactionFilterFactoryGetContext::IsManual(
       options.compaction_filter_factory.get()));
 
-  GenerateNewRandomFile(&rnd);
-  ASSERT_EQ("1,4", FilesPerLevel(0));
-
-  GenerateNewRandomFile(&rnd);
-  ASSERT_EQ("2,4", FilesPerLevel(0));
-
-  GenerateNewRandomFile(&rnd);
-  ASSERT_EQ("3,4", FilesPerLevel(0));
-
-  GenerateNewRandomFile(&rnd);
-  ASSERT_EQ("0,4,4", FilesPerLevel(0));
-
-  GenerateNewRandomFile(&rnd);
-  ASSERT_EQ("1,4,4", FilesPerLevel(0));
-
-  GenerateNewRandomFile(&rnd);
-  ASSERT_EQ("2,4,4", FilesPerLevel(0));
-
-  GenerateNewRandomFile(&rnd);
-  ASSERT_EQ("3,4,4", FilesPerLevel(0));
-
-  GenerateNewRandomFile(&rnd);
-  ASSERT_EQ("0,4,8", FilesPerLevel(0));
-
-  GenerateNewRandomFile(&rnd);
-  ASSERT_EQ("1,4,8", FilesPerLevel(0));
+  // make sure either L0 or L1 has file
+  while (NumTableFilesAtLevel(0) == 0 && NumTableFilesAtLevel(1) == 0) {
+    GenerateNewRandomFile(&rnd);
+  }
 
   // compact it three times
   for (int i = 0; i < 3; ++i) {
@@ -6071,12 +6044,69 @@ TEST_F(DBTest, DISABLED_SuggestCompactRangeTest) {
   end = Slice("m");
   ASSERT_OK(experimental::SuggestCompactRange(db_, &start, &end));
   ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_TRUE(CompactionFilterFactoryGetContext::IsManual(
+  // SuggestCompactRange() is not going to be reported as manual compaction
+  ASSERT_TRUE(!CompactionFilterFactoryGetContext::IsManual(
       options.compaction_filter_factory.get()));
 
   // now it should compact the level 0 file
+  // as it's a trivial move to L1, it triggers another one to compact to L2
   ASSERT_EQ(0, NumTableFilesAtLevel(0));
-  ASSERT_EQ(1, NumTableFilesAtLevel(1));
+  ASSERT_EQ(0, NumTableFilesAtLevel(1));
+}
+
+TEST_F(DBTest, SuggestCompactRangeUniversal) {
+  Options options = CurrentOptions();
+  options.memtable_factory.reset(test::NewSpecialSkipListFactory(
+      DBTestBase::kNumKeysByGenerateNewRandomFile));
+  options.compaction_style = kCompactionStyleUniversal;
+  options.write_buffer_size = 200 << 10;
+  options.arena_block_size = 4 << 10;
+  options.level0_file_num_compaction_trigger = 4;
+  options.num_levels = 4;
+  options.compression = kNoCompression;
+  options.max_bytes_for_level_base = 450 << 10;
+  options.target_file_size_base = 98 << 10;
+  options.max_compaction_bytes = static_cast<uint64_t>(1) << 60;  // inf
+
+  Reopen(options);
+
+  Random rnd(301);
+
+  for (int num = 0; num < 10; num++) {
+    GenerateNewRandomFile(&rnd);
+  }
+
+  ASSERT_EQ("1,2,3,4", FilesPerLevel());
+  for (int i = 0; i < 3; i++) {
+    ASSERT_OK(
+        db_->SuggestCompactRange(db_->DefaultColumnFamily(), nullptr, nullptr));
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  }
+
+  // All files are compacted
+  ASSERT_EQ(0, NumTableFilesAtLevel(0));
+  ASSERT_EQ(0, NumTableFilesAtLevel(1));
+  ASSERT_EQ(0, NumTableFilesAtLevel(2));
+
+  GenerateNewRandomFile(&rnd);
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+
+  // nonoverlapping with the file on level 0
+  Slice start("a"), end("b");
+  ASSERT_OK(experimental::SuggestCompactRange(db_, &start, &end));
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // should not compact the level 0 file
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+
+  start = Slice("j");
+  end = Slice("m");
+  ASSERT_OK(experimental::SuggestCompactRange(db_, &start, &end));
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // now it should compact the level 0 file to the last level
+  ASSERT_EQ(0, NumTableFilesAtLevel(0));
+  ASSERT_EQ(0, NumTableFilesAtLevel(1));
 }
 
 TEST_F(DBTest, PromoteL0) {

db/db_test2.cc

@@ -33,18 +33,6 @@ namespace ROCKSDB_NAMESPACE {
 class DBTest2 : public DBTestBase {
  public:
   DBTest2() : DBTestBase("db_test2", /*env_do_fsync=*/true) {}
-
- protected:
-#ifndef ROCKSDB_LITE
-  uint64_t GetSstSizeHelper(Temperature temperature) {
-    std::string prop;
-    EXPECT_TRUE(dbfull()->GetProperty(
-        DB::Properties::kLiveSstFilesSizeAtTemperature +
-            std::to_string(static_cast<uint8_t>(temperature)),
-        &prop));
-    return static_cast<uint64_t>(std::atoi(prop.c_str()));
-  }
-#endif  // ROCKSDB_LITE
 };
 
 #ifndef ROCKSDB_LITE
@@ -663,8 +651,12 @@ TEST_F(DBTest2, SharedWriteBufferLimitAcrossDB) {
 TEST_F(DBTest2, TestWriteBufferNoLimitWithCache) {
   Options options = CurrentOptions();
   options.arena_block_size = 4096;
-  std::shared_ptr<Cache> cache =
-      NewLRUCache(LRUCacheOptions(10000000, 1, false, 0.0));
+  std::shared_ptr<Cache> cache = NewLRUCache(LRUCacheOptions(
+      10000000 /* capacity */, 1 /* num_shard_bits */,
+      false /* strict_capacity_limit */, 0.0 /* high_pri_pool_ratio */,
+      nullptr /* memory_allocator */, kDefaultToAdaptiveMutex,
+      kDontChargeCacheMetadata));
+
   options.write_buffer_size = 50000;  // this is never hit
   // Use a write buffer total size so that the soft limit is about
   // 105000.
@@ -6688,7 +6680,7 @@ TEST_P(RenameCurrentTest, Compaction) {
   ASSERT_EQ("d_value", Get("d"));
 }
 
-TEST_F(DBTest2, BottommostTemperature) {
+TEST_F(DBTest2, LastLevelTemperature) {
   class TestListener : public EventListener {
    public:
     void OnFileReadFinish(const FileOperationInfo& info) override {
@@ -6742,11 +6734,16 @@ TEST_F(DBTest2, BottommostTemperature) {
     port::Mutex mutex_;
   };
 
+  const int kNumLevels = 7;
+  const int kLastLevel = kNumLevels - 1;
+
   auto* listener = new TestListener();
 
   Options options = CurrentOptions();
   options.bottommost_temperature = Temperature::kWarm;
   options.level0_file_num_compaction_trigger = 2;
+  options.level_compaction_dynamic_level_bytes = true;
+  options.num_levels = kNumLevels;
   options.statistics = CreateDBStatistics();
   options.listeners.emplace_back(listener);
   Reopen(options);
@@ -6772,7 +6769,7 @@ TEST_F(DBTest2, BottommostTemperature) {
   ColumnFamilyMetaData metadata;
   db_->GetColumnFamilyMetaData(&metadata);
   ASSERT_EQ(1, metadata.file_count);
-  SstFileMetaData meta = metadata.levels[1].files[0];
+  SstFileMetaData meta = metadata.levels[kLastLevel].files[0];
   ASSERT_EQ(Temperature::kWarm, meta.temperature);
   uint64_t number;
   FileType type;
@@ -6830,7 +6827,7 @@ TEST_F(DBTest2, BottommostTemperature) {
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
 
-  meta = metadata.levels[1].files[0];
+  meta = metadata.levels[kLastLevel].files[0];
   ASSERT_EQ(Temperature::kWarm, meta.temperature);
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
@@ -6849,7 +6846,7 @@ TEST_F(DBTest2, BottommostTemperature) {
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
 
-  meta = metadata.levels[1].files[0];
+  meta = metadata.levels[kLastLevel].files[0];
   ASSERT_EQ(Temperature::kWarm, meta.temperature);
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
@@ -6877,13 +6874,13 @@ TEST_F(DBTest2, BottommostTemperature) {
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
 
-  meta = metadata.levels[1].files[0];
+  meta = metadata.levels[kLastLevel].files[0];
   ASSERT_EQ(Temperature::kWarm, meta.temperature);
   ASSERT_TRUE(ParseFileName(meta.name, &number, &type));
   ASSERT_EQ(listener->file_temperatures.at(number), meta.temperature);
 }
 
-TEST_F(DBTest2, BottommostTemperatureUniversal) {
+TEST_F(DBTest2, LastLevelTemperatureUniversal) {
   const int kTriggerNum = 3;
   const int kNumLevels = 5;
   const int kBottommostLevel = kNumLevels - 1;
@@ -7009,7 +7006,7 @@ TEST_F(DBTest2, BottommostTemperatureUniversal) {
   ASSERT_EQ(std::atoi(prop.c_str()), 0);
 
   // Update bottommost temperature dynamically with SetOptions
-  auto s = db_->SetOptions({{"bottommost_temperature", "kCold"}});
+  auto s = db_->SetOptions({{"last_level_temperature", "kCold"}});
   ASSERT_OK(s);
   ASSERT_EQ(db_->GetOptions().bottommost_temperature, Temperature::kCold);
   db_->GetColumnFamilyMetaData(&metadata);
@@ -7109,6 +7106,9 @@ TEST_F(DBTest2, CheckpointFileTemperature) {
   std::unique_ptr<Env> env(new CompositeEnvWrapper(env_, test_fs));
   Options options = CurrentOptions();
   options.bottommost_temperature = Temperature::kWarm;
+  // set dynamic_level to true so the compaction would compact the data to the
+  // last level directly which will have the last_level_temperature
+  options.level_compaction_dynamic_level_bytes = true;
   options.level0_file_num_compaction_trigger = 2;
   options.env = env.get();
   Reopen(options);
@@ -7165,6 +7165,9 @@ TEST_F(DBTest2, FileTemperatureManifestFixup) {
   std::unique_ptr<Env> env(new CompositeEnvWrapper(env_, test_fs));
   Options options = CurrentOptions();
   options.bottommost_temperature = Temperature::kWarm;
+  // set dynamic_level to true so the compaction would compact the data to the
+  // last level directly which will have the last_level_temperature
+  options.level_compaction_dynamic_level_bytes = true;
   options.level0_file_num_compaction_trigger = 2;
   options.env = env.get();
   std::vector<std::string> cfs = {/*"default",*/ "test1", "test2"};
@@ -7436,6 +7439,45 @@ TEST_F(DBTest2, SstUniqueIdVerify) {
   ASSERT_TRUE(s.IsCorruption());
 }
 
+TEST_F(DBTest2, SstUniqueIdVerifyMultiCFs) {
+  const int kNumSst = 3;
+  const int kLevel0Trigger = 4;
+  auto options = CurrentOptions();
+  options.level0_file_num_compaction_trigger = kLevel0Trigger;
+
+  CreateAndReopenWithCF({"one", "two"}, options);
+
+  // generate good SSTs
+  for (int cf_num : {0, 2}) {
+    for (int i = 0; i < kNumSst; i++) {
+      for (int j = 0; j < 100; j++) {
+        ASSERT_OK(Put(cf_num, Key(i * 10 + j), "value"));
+      }
+      ASSERT_OK(Flush(cf_num));
+    }
+  }
+
+  // generate SSTs with bad unique id
+  SyncPoint::GetInstance()->SetCallBack(
+      "PropertyBlockBuilder::AddTableProperty:Start", [&](void* props_vs) {
+        auto props = static_cast<TableProperties*>(props_vs);
+        // update table property session_id to a different one
+        props->db_session_id = DBImpl::GenerateDbSessionId(nullptr);
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  for (int i = 0; i < kNumSst; i++) {
+    for (int j = 0; j < 100; j++) {
+      ASSERT_OK(Put(1, Key(i * 10 + j), "value"));
+    }
+    ASSERT_OK(Flush(1));
+  }
+
+  // Reopen with verification should report corruption
+  options.verify_sst_unique_id_in_manifest = true;
+  auto s = TryReopenWithColumnFamilies({"default", "one", "two"}, options);
+  ASSERT_TRUE(s.IsCorruption());
+}
+
 #ifndef ROCKSDB_LITE
 TEST_F(DBTest2, GetLatestSeqAndTsForKey) {
   Destroy(last_options_);

db/db_test_util.cc

@@ -1676,6 +1676,15 @@ uint64_t DBTestBase::GetNumberOfSstFilesForColumnFamily(
   }
   return result;
 }
+
+uint64_t DBTestBase::GetSstSizeHelper(Temperature temperature) {
+  std::string prop;
+  EXPECT_TRUE(dbfull()->GetProperty(
+      DB::Properties::kLiveSstFilesSizeAtTemperature +
+          std::to_string(static_cast<uint8_t>(temperature)),
+      &prop));
+  return static_cast<uint64_t>(std::atoi(prop.c_str()));
+}
 #endif  // ROCKSDB_LITE
 
 void VerifySstUniqueIds(const TablePropertiesCollection& props) {

db/db_test_util.h

@@ -1345,6 +1345,8 @@ class DBTestBase : public testing::Test {
 #ifndef ROCKSDB_LITE
   uint64_t GetNumberOfSstFilesForColumnFamily(DB* db,
                                               std::string column_family_name);
+
+  uint64_t GetSstSizeHelper(Temperature temperature);
 #endif  // ROCKSDB_LITE
 
   uint64_t TestGetTickerCount(const Options& options, Tickers ticker_type) {

db/db_wal_test.cc

@@ -1523,8 +1523,25 @@ TEST_F(DBWALTest, RaceInstallFlushResultsWithWalObsoletion) {
       /*wait=*/false, /*allow_write_stall=*/true, handles_[0]));
 
   bool called = false;
+  std::atomic<int> bg_flush_threads{0};
+  std::atomic<bool> wal_synced{false};
   SyncPoint::GetInstance()->DisableProcessing();
   SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::BackgroundCallFlush:start", [&](void* /*arg*/) {
+        int cur = bg_flush_threads.load();
+        int desired = cur + 1;
+        if (cur > 0 ||
+            !bg_flush_threads.compare_exchange_strong(cur, desired)) {
+          while (!wal_synced.load()) {
+            // Wait until the other bg flush thread finishes committing WAL sync
+            // operation to the MANIFEST.
+          }
+        }
+      });
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::FlushMemTableToOutputFile:CommitWal:1",
+      [&](void* /*arg*/) { wal_synced.store(true); });
   // This callback will be called when the first bg flush thread reaches the
   // point before entering the MANIFEST write queue after flushing the SST
   // file.

db/db_with_timestamp_compaction_test.cc

@@ -109,6 +109,67 @@ TEST_F(TimestampCompatibleCompactionTest, UserKeyCrossFileBoundary) {
   SyncPoint::GetInstance()->DisableProcessing();
 }
 
+TEST_F(TimestampCompatibleCompactionTest, MultipleSubCompactions) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.compaction_style = kCompactionStyleUniversal;
+  options.comparator = test::BytewiseComparatorWithU64TsWrapper();
+  options.level0_file_num_compaction_trigger = 3;
+  options.max_subcompactions = 3;
+  options.target_file_size_base = 1024;
+  options.statistics = CreateDBStatistics();
+  DestroyAndReopen(options);
+
+  uint64_t ts = 100;
+  uint64_t key = 0;
+  WriteOptions write_opts;
+
+  // Write keys 0, 1, ..., 499 with ts from 100 to 599.
+  {
+    for (; key <= 499; ++key, ++ts) {
+      std::string ts_str = Timestamp(ts);
+      ASSERT_OK(db_->Put(write_opts, Key1(key), ts_str,
+                         "foo_" + std::to_string(key)));
+    }
+  }
+
+  // Write keys 500, ..., 999 with ts from 600 to 1099.
+  {
+    for (; key <= 999; ++key, ++ts) {
+      std::string ts_str = Timestamp(ts);
+      ASSERT_OK(db_->Put(write_opts, Key1(key), ts_str,
+                         "foo_" + std::to_string(key)));
+    }
+    ASSERT_OK(Flush());
+  }
+
+  // Wait for compaction to finish
+  {
+    ASSERT_OK(dbfull()->RunManualCompaction(
+        static_cast_with_check<ColumnFamilyHandleImpl>(
+            db_->DefaultColumnFamily())
+            ->cfd(),
+        0 /* input_level */, 1 /* output_level */, CompactRangeOptions(),
+        nullptr /* begin */, nullptr /* end */, true /* exclusive */,
+        true /* disallow_trivial_move */,
+        std::numeric_limits<uint64_t>::max() /* max_file_num_to_ignore */,
+        "" /*trim_ts*/));
+  }
+
+  // Check stats to make sure multiple subcompactions were scheduled for
+  // boundaries not to be nullptr.
+  {
+    HistogramData num_sub_compactions;
+    options.statistics->histogramData(NUM_SUBCOMPACTIONS_SCHEDULED,
+                                      &num_sub_compactions);
+    ASSERT_GT(num_sub_compactions.sum, 1);
+  }
+
+  for (key = 0; key <= 999; ++key) {
+    ASSERT_EQ("foo_" + std::to_string(key), Get(Key1(key), ts));
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_write_buffer_manager_test.cc

@@ -780,6 +780,75 @@ TEST_P(DBWriteBufferManagerTest, MixedSlowDownOptionsMultipleDB) {
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
 
+#ifndef ROCKSDB_LITE
+
+// Tests a `WriteBufferManager` constructed with `allow_stall == false` does not
+// thrash memtable switching when full and a CF receives multiple writes.
+// Instead, we expect to switch a CF's memtable for flush only when that CF does
+// not have any pending or running flush.
+//
+// This test uses multiple DBs each with a single CF instead of a single DB
+// with multiple CFs. That way we can control which CF is considered for switch
+// by writing to that CF's DB.
+//
+// Not supported in LITE mode due to `GetProperty()` unavailable.
+TEST_P(DBWriteBufferManagerTest, StopSwitchingMemTablesOnceFlushing) {
+  Options options = CurrentOptions();
+  options.arena_block_size = 4 << 10;   // 4KB
+  options.write_buffer_size = 1 << 20;  // 1MB
+  std::shared_ptr<Cache> cache =
+      NewLRUCache(4 << 20 /* capacity (4MB) */, 2 /* num_shard_bits */);
+  ASSERT_LT(cache->GetUsage(), 256 << 10 /* 256KB */);
+  cost_cache_ = GetParam();
+  if (cost_cache_) {
+    options.write_buffer_manager.reset(new WriteBufferManager(
+        512 << 10 /* buffer_size (512KB) */, cache, false /* allow_stall */));
+  } else {
+    options.write_buffer_manager.reset(
+        new WriteBufferManager(512 << 10 /* buffer_size (512KB) */,
+                               nullptr /* cache */, false /* allow_stall */));
+  }
+
+  Reopen(options);
+  std::string dbname = test::PerThreadDBPath("db_shared_wbm_db");
+  DB* shared_wbm_db = nullptr;
+
+  ASSERT_OK(DestroyDB(dbname, options));
+  ASSERT_OK(DB::Open(options, dbname, &shared_wbm_db));
+
+  // The last write will make WBM need flush, but it won't flush yet.
+  ASSERT_OK(Put(Key(1), DummyString(256 << 10 /* 256KB */), WriteOptions()));
+  ASSERT_FALSE(options.write_buffer_manager->ShouldFlush());
+  ASSERT_OK(Put(Key(1), DummyString(256 << 10 /* 256KB */), WriteOptions()));
+  ASSERT_TRUE(options.write_buffer_manager->ShouldFlush());
+
+  // Flushes will be pending, not running because flush threads are blocked.
+  test::SleepingBackgroundTask sleeping_task_high;
+  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                 &sleeping_task_high, Env::Priority::HIGH);
+
+  for (int i = 0; i < 3; ++i) {
+    ASSERT_OK(
+        shared_wbm_db->Put(WriteOptions(), Key(1), DummyString(1 /* len */)));
+    std::string prop;
+    ASSERT_TRUE(
+        shared_wbm_db->GetProperty("rocksdb.num-immutable-mem-table", &prop));
+    ASSERT_EQ(std::to_string(i > 0 ? 1 : 0), prop);
+    ASSERT_TRUE(
+        shared_wbm_db->GetProperty("rocksdb.mem-table-flush-pending", &prop));
+    ASSERT_EQ(std::to_string(i > 0 ? 1 : 0), prop);
+  }
+
+  // Clean up DBs.
+  sleeping_task_high.WakeUp();
+  sleeping_task_high.WaitUntilDone();
+  ASSERT_OK(shared_wbm_db->Close());
+  ASSERT_OK(DestroyDB(dbname, options));
+  delete shared_wbm_db;
+}
+
+#endif  // ROCKSDB_LITE
+
 INSTANTIATE_TEST_CASE_P(DBWriteBufferManagerTest, DBWriteBufferManagerTest,
                         testing::Bool());
 

db/db_write_test.cc

@@ -31,6 +31,12 @@ class DBWriteTest : public DBTestBase, public testing::WithParamInterface<int> {
   void Open() { DBTestBase::Reopen(GetOptions()); }
 };
 
+class DBWriteTestUnparameterized : public DBTestBase {
+ public:
+  explicit DBWriteTestUnparameterized()
+      : DBTestBase("pipelined_write_test", /*env_do_fsync=*/false) {}
+};
+
 // It is invalid to do sync write while disabling WAL.
 TEST_P(DBWriteTest, SyncAndDisableWAL) {
   WriteOptions write_options;
@@ -318,6 +324,130 @@ TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) {
   Close();
 }
 
+TEST_F(DBWriteTestUnparameterized, PipelinedWriteRace) {
+  // This test was written to trigger a race in ExitAsBatchGroupLeader in case
+  // enable_pipelined_write_ was true.
+  // Writers for which ShouldWriteToMemtable() evaluates to false are removed
+  // from the write_group via CompleteFollower/ CompleteLeader. Writers in the
+  // middle of the group are fully unlinked, but if that writers is the
+  // last_writer, then we did not update the predecessor's link_older, i.e.,
+  // this writer was still reachable via newest_writer_.
+  //
+  // But the problem was, that CompleteFollower already wakes up the thread
+  // owning that writer before the writer has been removed. This resulted in a
+  // race - if the leader thread was fast enough, then everything was fine.
+  // However, if the woken up thread finished the current write operation and
+  // then performed yet another write, then a new writer instance was added
+  // to newest_writer_. It is possible that the new writer is located on the
+  // same address on stack, and if this happened, then we had a problem,
+  // because the old code tried to find the last_writer in the list to unlink
+  // it, which in this case produced a cycle in the list.
+  // Whether two invocations of PipelinedWriteImpl() by the same thread actually
+  // allocate the writer on the same address depends on the OS and/or compiler,
+  // so it is rather hard to create a deterministic test for this.
+
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.enable_pipelined_write = true;
+  std::vector<port::Thread> threads;
+
+  std::atomic<int> write_counter{0};
+  std::atomic<int> active_writers{0};
+  std::atomic<bool> second_write_starting{false};
+  std::atomic<bool> second_write_in_progress{false};
+  std::atomic<WriteThread::Writer*> leader{nullptr};
+  std::atomic<bool> finished_WAL_write{false};
+
+  DestroyAndReopen(options);
+
+  auto write_one_doc = [&]() {
+    int a = write_counter.fetch_add(1);
+    std::string key = "foo" + std::to_string(a);
+    WriteOptions wo;
+    ASSERT_OK(dbfull()->Put(wo, key, "bar"));
+    --active_writers;
+  };
+
+  auto write_two_docs = [&]() {
+    write_one_doc();
+    second_write_starting = true;
+    write_one_doc();
+  };
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "WriteThread::JoinBatchGroup:Wait", [&](void* arg) {
+        if (second_write_starting.load()) {
+          second_write_in_progress = true;
+          return;
+        }
+        auto* w = reinterpret_cast<WriteThread::Writer*>(arg);
+        if (w->state == WriteThread::STATE_GROUP_LEADER) {
+          active_writers++;
+          if (leader.load() == nullptr) {
+            leader.store(w);
+            while (active_writers.load() < 2) {
+              // wait for another thread to join the write_group
+            }
+          }
+        } else {
+          // we disable the memtable for all followers so that they they are
+          // removed from the write_group before enqueuing it for the memtable
+          // write
+          w->disable_memtable = true;
+          active_writers++;
+        }
+      });
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "WriteThread::ExitAsBatchGroupLeader:Start", [&](void* arg) {
+        auto* wg = reinterpret_cast<WriteThread::WriteGroup*>(arg);
+        if (wg->leader == leader && !finished_WAL_write) {
+          finished_WAL_write = true;
+          while (active_writers.load() < 3) {
+            // wait for the new writer to be enqueued
+          }
+        }
+      });
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "WriteThread::ExitAsBatchGroupLeader:AfterCompleteWriters",
+      [&](void* arg) {
+        auto* wg = reinterpret_cast<WriteThread::WriteGroup*>(arg);
+        if (wg->leader == leader) {
+          while (!second_write_in_progress.load()) {
+            // wait for the old follower thread to start the next write
+          }
+        }
+      });
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  // start leader + one follower
+  threads.emplace_back(write_one_doc);
+  while (leader.load() == nullptr) {
+    // wait for leader
+  }
+
+  // we perform two writes in the follower, so that for the second write
+  // the thread reinserts a Writer with the same address
+  threads.emplace_back(write_two_docs);
+
+  // wait for the leader to enter ExitAsBatchGroupLeader
+  while (!finished_WAL_write.load()) {
+    // wait for write_group to have finished the WAL writes
+  }
+
+  // start another writer thread to be enqueued before the leader can
+  // complete the writers from its write_group
+  threads.emplace_back(write_one_doc);
+
+  for (auto& t : threads) {
+    t.join();
+  }
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 TEST_P(DBWriteTest, ManualWalFlushInEffect) {
   Options options = GetOptions();
   Reopen(options);

db/dbformat.h

@@ -68,7 +68,9 @@ enum ValueType : unsigned char {
   kTypeCommitXIDAndTimestamp = 0x15,  // WAL only
   kTypeWideColumnEntity = 0x16,
   kTypeColumnFamilyWideColumnEntity = 0x17,  // WAL only
-  kMaxValue = 0x7F                           // Not used for storing records.
+  kTypeMaxValid,    // Should be after the last valid type, only used for
+                    // validation
+  kMaxValue = 0x7F  // Not used for storing records.
 };
 
 // Defined in dbformat.cc

db/error_handler.cc

@@ -358,6 +358,9 @@ const Status& ErrorHandler::HandleKnownErrors(const Status& bg_err,
       RecoverFromNoSpace();
     }
   }
+  if (bg_error_.severity() >= Status::Severity::kHardError) {
+    is_db_stopped_.store(true, std::memory_order_release);
+  }
   return bg_error_;
 }
 
@@ -736,6 +739,7 @@ void ErrorHandler::RecoverFromRetryableBGIOError() {
         // the bg_error and notify user.
         TEST_SYNC_POINT("RecoverFromRetryableBGIOError:RecoverSuccess");
         Status old_bg_error = bg_error_;
+        is_db_stopped_.store(false, std::memory_order_release);
         bg_error_ = Status::OK();
         bg_error_.PermitUncheckedError();
         EventHelpers::NotifyOnErrorRecoveryEnd(
@@ -792,6 +796,9 @@ void ErrorHandler::CheckAndSetRecoveryAndBGError(const Status& bg_err) {
   if (bg_err.severity() > bg_error_.severity()) {
     bg_error_ = bg_err;
   }
+  if (bg_error_.severity() >= Status::Severity::kHardError) {
+    is_db_stopped_.store(true, std::memory_order_release);
+  }
   return;
 }