Fix test

Summary:
Clang-analyze complains about potential nullptr dereference.
Fix by adding an assertion to make clang happy.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9682

Test Plan: USE_CLANG=1 make -j20 analyze_incremental

Reviewed By: ltamasi

Differential Revision: D34755210

Pulled By: riversand963

fbshipit-source-id: 948e1899846ee1aa05a1b500a11ff43b0b412e0a

.circleci/config.yml

@@ -87,7 +87,6 @@ commands:
       - run:
           name: Install Clang 13
           command: |
-            wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
             echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
             echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main" | sudo tee -a /etc/apt/sources.list
             echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
@@ -171,19 +170,42 @@ jobs:
       - increase-max-open-files-on-macos
       - install-gflags-on-macos
       - pre-steps-macos
-      - run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 check
+      - run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 all
       - post-steps
 
   build-macos-cmake:
     macos:
       xcode: 12.5.1
     resource_class: large
+    parameters:
+      run_even_tests:
+        description: run even or odd tests, used to split tests to 2 groups
+        type: boolean
+        default: true
     steps:
       - increase-max-open-files-on-macos
       - install-cmake-on-macos
       - install-gflags-on-macos
       - pre-steps-macos
-      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32 && ctest -j10)
+      - run:
+          name: "cmake generate project file"
+          command: ulimit -S -n 1048576 && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
+      - run:
+          name: "Build tests"
+          command: cd build && make V=1 -j32
+      - when:
+          condition: << parameters.run_even_tests >>
+          steps:
+            - run:
+                name: "Run even tests"
+                command: ulimit -S -n 1048576 && cd build && ctest -j32 -I 0,,2
+      - when:
+          condition:
+            not: << parameters.run_even_tests >>
+          steps:
+            - run:
+                name: "Run odd tests"
+                command: ulimit -S -n 1048576 && cd build && ctest -j32 -I 1,,2
       - post-steps
 
   build-linux:
@@ -219,38 +241,38 @@ jobs:
   build-linux-release:
     machine:
       image: ubuntu-2004:202111-02
-    resource_class: large
+    resource_class: 2xlarge
     steps:
       - checkout # check out the code in the project directory
-      - run: make V=1 -j8 release
+      - run: make V=1 -j32 release
       - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
       - install-gflags
-      - run: make V=1 -j8 release
+      - run: make V=1 -j32 release
       - run: ./db_stress --version # ensure with gflags
       - post-steps
 
   build-linux-release-rtti:
     machine:
       image: ubuntu-2004:202111-02
-    resource_class: large
+    resource_class: xlarge
     steps:
       - checkout # check out the code in the project directory
       - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench
+      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
       - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
       - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
       - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench
+      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
       - run: ./db_stress --version # ensure with gflags
 
   build-linux-lite:
     machine:
       image: ubuntu-2004:202111-02
-    resource_class: 2xlarge
+    resource_class: large
     steps:
       - pre-steps
       - install-gflags
-      - run: LITE=1 make V=1 J=32 -j32 check
+      - run: LITE=1 make V=1 J=8 -j8 check
       - post-steps
 
   build-linux-lite-release:
@@ -291,11 +313,30 @@ jobs:
     machine:
       image: ubuntu-2004:202111-02
     resource_class: 2xlarge
+    # find test list by `make list_all_tests`
+    parameters:
+      start_test:
+        default: ""
+        type: string
+      end_test:
+        default: ""
+        type: string
     steps:
       - pre-steps
       - install-gflags
       - install-clang-10
-      - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out.
+      - install-gtest-parallel
+      - run:
+          name: "Build unit tests"
+          command: |
+            echo "env: $(env)"
+            ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 --output-sync=target build_subset_tests
+      - run:
+          name: "Run unit tests in parallel"
+          command: |
+            sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
+            cat /tmp/test_list
+            gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat  # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.
       - post-steps
 
   build-linux-clang10-ubsan:
@@ -367,11 +408,11 @@ jobs:
   build-linux-gcc-8-no_test_run:
     machine:
       image: ubuntu-2004:202111-02
-    resource_class: large
+    resource_class: xlarge
     steps:
       - pre-steps
       - run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get update -y && sudo apt-get install gcc-8 g++-8 libgflags-dev
-      - run: CC=gcc-8 CXX=g++-8 V=1 make -j8 all
+      - run: CC=gcc-8 CXX=g++-8 V=1 make -j16 all
       - post-steps
 
   build-linux-gcc-10-cxx20-no_test_run:
@@ -381,7 +422,7 @@ jobs:
     steps:
       - pre-steps
       - run: sudo apt-get update -y && sudo apt-get install gcc-10 g++-10 libgflags-dev
-      - run: CC=gcc-10 CXX=g++-10 V=1 SKIP_LINK=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all # Linking broken because libgflags compiled with newer ABI
+      - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all
       - post-steps
 
   build-linux-gcc-11-no_test_run:
@@ -390,8 +431,9 @@ jobs:
     resource_class: xlarge
     steps:
       - pre-steps
+      - install-benchmark
       - run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get update -y && sudo apt-get install gcc-11 g++-11 libgflags-dev
-      - run: CC=gcc-11 CXX=g++-11 V=1 SKIP_LINK=1 make -j16 all # Linking broken because libgflags compiled with newer ABI
+      - run: CC=gcc-11 CXX=g++-11 V=1 make -j16 all microbench
       - post-steps
 
   build-linux-clang-13-no_test_run:
@@ -401,18 +443,19 @@ jobs:
     steps:
       - pre-steps
       - install-clang-13
-      - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j16 all
+      - install-benchmark
+      - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j16 all microbench
       - post-steps
 
   # This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing.
-  build-linux-microbench:
+  build-linux-run-microbench:
     machine:
       image: ubuntu-2004:202111-02
-    resource_class: xlarge
+    resource_class: 2xlarge
     steps:
       - pre-steps
       - install-benchmark
-      - run: DEBUG_LEVEL=0 make microbench
+      - run: DEBUG_LEVEL=0 make -j32 run_microbench
       - post-steps
 
   build-windows:
@@ -476,7 +519,7 @@ jobs:
             ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 << parameters.extra_cmake_opt >> ..
             cd ..
             echo "Building with VS version: ${CMAKE_GENERATOR}"
-            msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+            msbuild.exe build/rocksdb.sln -maxCpuCount:64 -property:Configuration=Debug -property:Platform=x64
       - run:
           name: "Test RocksDB"
           shell: powershell.exe
@@ -531,7 +574,7 @@ jobs:
   build-macos-java:
     macos:
       xcode: 12.5.1
-    resource_class: medium
+    resource_class: large
     environment:
       JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
       ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc causes java 8 crash
@@ -549,16 +592,16 @@ jobs:
             which javac && javac -version
       - run:
           name: "Build RocksDBJava Shared Library"
-          command: make V=1 J=8 -j8 rocksdbjava
+          command: make V=1 J=16 -j16 rocksdbjava
       - run:
           name: "Test RocksDBJava"
-          command: make V=1 J=8 -j8 jtest
+          command: make V=1 J=16 -j16 jtest
       - post-steps
 
   build-macos-java-static:
     macos:
       xcode: 12.5.1
-    resource_class: medium
+    resource_class: large
     environment:
       JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
     steps:
@@ -576,13 +619,13 @@ jobs:
             which javac && javac -version
       - run:
           name: "Build RocksDBJava x86 and ARM Static Libraries"
-          command: make V=1 J=8 -j8 rocksdbjavastaticosx
+          command: make V=1 J=16 -j16 rocksdbjavastaticosx
       - post-steps
 
   build-macos-java-static-universal:
     macos:
       xcode: 12.5.1
-    resource_class: medium
+    resource_class: large
     environment:
       JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
     steps:
@@ -600,7 +643,7 @@ jobs:
             which javac && javac -version
       - run:
           name: "Build RocksDBJava Universal Binary Static Library"
-          command: make V=1 J=8 -j8 rocksdbjavastaticosx_ub
+          command: make V=1 J=16 -j16 rocksdbjavastaticosx_ub
       - post-steps
 
   build-examples:
@@ -619,7 +662,7 @@ jobs:
   build-cmake-mingw:
     machine:
       image: ubuntu-2004:202111-02
-    resource_class: 2xlarge
+    resource_class: large
     steps:
       - pre-steps
       - install-gflags
@@ -643,30 +686,12 @@ jobs:
     machine:
       image: ubuntu-2004:202111-02
     resource_class: 2xlarge
-    parameters:
-      start_test:
-        default: ""
-        type: string
-      end_test:
-        default: ""
-        type: string
+    environment:
+      TEST_TMPDIR: /tmp/rocksdb_test_tmp
     steps:
       - pre-steps
       - install-gflags
-      - install-gtest-parallel
-      - run:
-          name: "Build unit tests"
-          command: |
-            echo "env: $(env)"
-            echo "** done env"
-            ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list make V=1 -j32 --output-sync=target build_subset_tests
-      - run:
-          name: "Run unit tests in parallel"
-          command: |
-            sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
-            cat /tmp/test_list
-            export TEST_TMPDIR=/tmp/rocksdb_test_tmp
-            gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat  # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.
+      - run: make V=1 -j32 check
       - post-steps
 
   build-linux-arm-test-full:
@@ -788,7 +813,12 @@ workflows:
       - build-linux-clang10-asan
   build-linux-clang10-mini-tsan:
     jobs:
-      - build-linux-clang10-mini-tsan
+      - build-linux-clang10-mini-tsan:
+          start_test: ""
+          end_test: "env_test"
+      - build-linux-clang10-mini-tsan:
+          start_test: "env_test"
+          end_test: ""
   build-linux-clang10-ubsan:
     jobs:
       - build-linux-clang10-ubsan
@@ -823,20 +853,6 @@ workflows:
   build-examples:
     jobs:
       - build-examples
-  build-linux-non-shm:
-    jobs:
-      - build-linux-non-shm:
-          start_test: ""
-          end_test: "db_options_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "db_options_test" # make sure unique in src.mk
-          end_test: "filename_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "filename_test" # make sure unique in src.mk
-          end_test: "statistics_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "statistics_test" # make sure unique in src.mk
-          end_test: ""
   build-linux-compilers-no_test_run:
     jobs:
       - build-linux-clang-no_test_run
@@ -849,18 +865,16 @@ workflows:
   build-macos:
     jobs:
       - build-macos
-  build-macos-cmake:
-    jobs:
-      - build-macos-cmake
+      - build-macos-cmake:
+          run_even_tests: true
+      - build-macos-cmake:
+          run_even_tests: false
   build-cmake-mingw:
     jobs:
       - build-cmake-mingw
   build-linux-arm:
     jobs:
       - build-linux-arm
-  build-microbench:
-    jobs:
-      - build-linux-microbench
   build-fuzzers:
     jobs:
       - build-fuzzers
@@ -875,3 +889,5 @@ workflows:
     jobs:
       - build-format-compatible
       - build-linux-arm-test-full
+      - build-linux-run-microbench
+      - build-linux-non-shm

.github/workflows/sanity_check.yml

@@ -32,7 +32,7 @@ jobs:
     - name: Download clang-format-diff.py
       uses: wei/wget@v1
       with:
-        args: https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py
+        args: https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py
 
     - name: Check format
       run: VERBOSE_CHECK=1 make check-format

.travis.yml

@@ -43,7 +43,6 @@ env:
   - JOB_NAME=cmake-gcc9 # 3-5 minutes
   - JOB_NAME=cmake-gcc9-c++20 # 3-5 minutes
   - JOB_NAME=cmake-mingw # 3 minutes
-  - JOB_NAME=status_checked
 
 matrix:
   exclude:
@@ -58,10 +57,6 @@ matrix:
     env: JOB_NAME=cmake-mingw
   - os: linux
     compiler: clang
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: s390x # Temporary while working through gcc-7 + assembler issue
-    env: TEST_GROUP=platform_dependent
   - if: type = pull_request AND commit_message !~ /FULL_CI/
     os: linux
     arch: arm64
@@ -190,18 +185,6 @@ matrix:
     os: linux
     arch: s390x
     env: JOB_NAME=cmake-gcc9-c++20
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=status_checked
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=status_checked
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: s390x
-    env: JOB_NAME=status_checked
 
 install:
   -  CC=gcc-7 && CXX=g++-7
@@ -247,21 +230,25 @@ before_script:
 script:
   - date; ${CXX} --version
   - if [ `command -v ccache` ]; then ccache -C; fi
+  - export MK_PARALLEL=4;
+    if [[ "$TRAVIS_CPU_ARCH" == s390x ]]; then
+      export MK_PARALLEL=1;
+    fi
   - case $TEST_GROUP in
     platform_dependent)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=only make -j4 all_but_some_tests check_some
+      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=only make -j$MK_PARALLEL all_but_some_tests check_some
       ;;
     1)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_END=backupable_db_test make -j4 check_some
+      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_END=backupable_db_test make -j$MK_PARALLEL check_some
       ;;
     2)
-      OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 make -j4 tools && OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=backupable_db_test ROCKSDBTESTS_END=db_universal_compaction_test make -j4 check_some
+      OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 make -j$MK_PARALLEL tools && OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=backupable_db_test ROCKSDBTESTS_END=db_universal_compaction_test make -j$MK_PARALLEL check_some
       ;;
     3)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=db_universal_compaction_test ROCKSDBTESTS_END=table_properties_collector_test make -j4 check_some
+      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=db_universal_compaction_test ROCKSDBTESTS_END=table_properties_collector_test make -j$MK_PARALLEL check_some
       ;;
     4)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=table_properties_collector_test make -j4 check_some
+      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=table_properties_collector_test make -j$MK_PARALLEL check_some
       ;;
     esac
   - case $JOB_NAME in
@@ -269,10 +256,10 @@ script:
       OPT=-DTRAVIS LIB_MODE=shared V=1 make rocksdbjava jtest
       ;;
     lite_build)
-      OPT='-DTRAVIS -DROCKSDB_LITE' LIB_MODE=shared V=1 make -j4 all
+      OPT='-DTRAVIS -DROCKSDB_LITE' LIB_MODE=shared V=1 make -j$MK_PARALLEL all
       ;;
     examples)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 make -j4 static_lib && cd examples && make -j4
+      OPT=-DTRAVIS LIB_MODE=shared V=1 make -j$MK_PARALLEL static_lib && cd examples && make -j$MK_PARALLEL
       ;;
     cmake-mingw)
       sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix;
@@ -285,10 +272,7 @@ script:
           ;;
       esac
 
-      mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=0 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. && make -j4 && cd  .. && rm -rf build && mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release $OPT && make -j4 rocksdb rocksdbjni
-      ;;
-    status_checked)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ASSERT_STATUS_CHECKED=1 make -j4 check_some
+      mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=0 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. && make -j$MK_PARALLEL && cd  .. && rm -rf build && mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release $OPT && make -j$MK_PARALLEL rocksdb rocksdbjni
       ;;
     esac
 notifications:

CMakeLists.txt

@@ -627,6 +627,7 @@ set(SOURCES
         cache/cache_reservation_manager.cc
         cache/clock_cache.cc
         cache/lru_cache.cc
+        cache/lru_secondary_cache.cc
         cache/sharded_cache.cc
         db/arena_wrapped_db_iter.cc
         db/blob/blob_fetcher.cc
@@ -829,6 +830,7 @@ set(SOURCES
         util/coding.cc
         util/compaction_job_stats_impl.cc
         util/comparator.cc
+        util/compression.cc
         util/compression_context_cache.cc
         util/concurrent_task_limiter_impl.cc
         util/crc32c.cc
@@ -1154,6 +1156,7 @@ if(WITH_TESTS)
         cache/cache_reservation_manager_test.cc
         cache/cache_test.cc
         cache/lru_cache_test.cc
+        cache/lru_secondary_cache_test.cc
         db/blob/blob_counting_iterator_test.cc
         db/blob/blob_file_addition_test.cc
         db/blob/blob_file_builder_test.cc
@@ -1331,6 +1334,7 @@ if(WITH_TESTS)
         utilities/transactions/optimistic_transaction_test.cc
         utilities/transactions/transaction_test.cc
         utilities/transactions/lock/point/point_lock_manager_test.cc
+        utilities/transactions/write_committed_transaction_ts_test.cc
         utilities/transactions/write_prepared_transaction_test.cc
         utilities/transactions/write_unprepared_transaction_test.cc
         utilities/transactions/lock/range/range_locking_test.cc

HISTORY.md

@@ -1,5 +1,25 @@
 # Rocksdb Change Log
 ## Unreleased
+### New Features
+* Allow WriteBatchWithIndex to index a WriteBatch that includes keys with user-defined timestamps. The index itself does not have timestamp.
+* Add support for user-defined timestamps to write-committed transaction without API change. The `TransactionDB` layer APIs do not allow timestamps because we require that all user-defined-timestamps-aware operations go through the `Transaction` APIs.
+* Added BlobDB options to `ldb`
+* `BlockBasedTableOptions::detect_filter_construct_corruption` can now be dynamically configured using `DB::SetOptions`.
+
+### Bug Fixes
+* Fixed a data race on `versions_` between `DBImpl::ResumeImpl()` and threads waiting for recovery to complete (#9496)
+* Fixed a bug caused by race among flush, incoming writes and taking snapshots. Queries to snapshots created with these race condition can return incorrect result, e.g. resurfacing deleted data.
+* Fixed a bug that DB flush uses `options.compression` even `options.compression_per_level` is set.
+
+### Public API changes
+* Remove BlockBasedTableOptions.hash_index_allow_collision which already takes no effect.
+* `options.compression_per_level` is dynamically changeable with `SetOptions()`.
+* Added `WriteOptions::rate_limiter_priority`. When set to something other than `Env::IO_TOTAL`, the internal rate limiter (`DBOptions::rate_limiter`) will be charged at the specified priority for writes associated with the API to which the `WriteOptions` was provided. Currently the support covers automatic WAL flushes, which happen during live updates (`Put()`, `Write()`, `Delete()`, etc.) when `WriteOptions::disableWAL == false` and `DBOptions::manual_wal_flush == false`.
+
+### Bug Fixes
+* Fix a race condition when cancel manual compaction with `DisableManualCompaction`. Also DB close can cancel the manual compaction thread.
+
+## 7.0.0 (02/20/2022)
 ### Bug Fixes
 * Fixed a major bug in which batched MultiGet could return old values for keys deleted by DeleteRange when memtable Bloom filter is enabled (memtable_prefix_bloom_size_ratio > 0). (The fix includes a substantial MultiGet performance improvement in the unusual case of both memtable_whole_key_filtering and prefix_extractor.)
 * Fixed more cases of EventListener::OnTableFileCreated called with OK status, file_size==0, and no SST file kept. Now the status is Aborted.

Makefile

@@ -329,8 +329,8 @@ ifneq ($(MACHINE), arm64)
 # linking with jemalloc (as it won't be arm64-compatible) and remove some other options
 # set during platform detection
 DISABLE_JEMALLOC=1
-PLATFORM_CFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CFLAGS))
-PLATFORM_CXXFLAGS := $(filter-out -march=native -DHAVE_SSE42, $(PLATFORM_CXXFLAGS))
+PLATFORM_CFLAGS := $(filter-out -march=native -DHAVE_SSE42 -DHAVE_AVX2, $(PLATFORM_CFLAGS))
+PLATFORM_CXXFLAGS := $(filter-out -march=native -DHAVE_SSE42 -DHAVE_AVX2, $(PLATFORM_CXXFLAGS))
 endif
 endif
 endif
@@ -815,6 +815,8 @@ test_libs: $(TEST_LIBS)
 benchmarks: $(BENCHMARKS)
 
 microbench: $(MICROBENCHS)
+
+run_microbench: $(MICROBENCHS)
 	for t in $(MICROBENCHS); do echo "===== Running benchmark $$t (`date`)"; ./$$t || exit 1; done;
 
 dbg: $(LIBRARY) $(BENCHMARKS) tools $(TESTS)
@@ -1864,6 +1866,9 @@ point_lock_manager_test: utilities/transactions/lock/point/point_lock_manager_te
 transaction_test: $(OBJ_DIR)/utilities/transactions/transaction_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+write_committed_transaction_ts_test: $(OBJ_DIR)/utilities/transactions/write_committed_transaction_ts_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 write_prepared_transaction_test: $(OBJ_DIR)/utilities/transactions/write_prepared_transaction_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -1897,6 +1902,9 @@ statistics_test: $(OBJ_DIR)/monitoring/statistics_test.o $(TEST_LIBRARY) $(LIBRA
 stats_history_test: $(OBJ_DIR)/monitoring/stats_history_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+lru_secondary_cache_test: $(OBJ_DIR)/cache/lru_secondary_cache_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 lru_cache_test: $(OBJ_DIR)/cache/lru_cache_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -2116,7 +2124,9 @@ CURL_SSL_OPTS ?= --tlsv1
 ifeq ($(PLATFORM), OS_MACOSX)
 ifeq (,$(findstring librocksdbjni-osx,$(ROCKSDBJNILIB)))
 ifeq ($(MACHINE),arm64)
-	ROCKSDBJNILIB = librocksdbjni-osx-aarch64.jnilib
+	ROCKSDBJNILIB = librocksdbjni-osx-arm64.jnilib
+else ifeq ($(MACHINE),x86_64)
+	ROCKSDBJNILIB = librocksdbjni-osx-x86_64.jnilib
 else
 	ROCKSDBJNILIB = librocksdbjni-osx.jnilib
 endif
@@ -2247,15 +2257,20 @@ endif
 	$(MAKE) rocksdbjavastatic_deps
 	$(MAKE) rocksdbjavastatic_libobjects
 	$(MAKE) rocksdbjavastatic_javalib
-	$(MAKE) rocksdbjavastatic_jar
+	$(MAKE) rocksdbjava_jar
 
 rocksdbjavastaticosx: rocksdbjavastaticosx_archs
-	mv java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx.jnilib
-	mv java/target/librocksdbjni-osx-arm64.jnilib java/target/librocksdbjni-osx-aarch64.jnilib
+	cd java; $(JAR_CMD)  -cf target/$(ROCKSDB_JAR) HISTORY*.md
+	cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib
+	cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
+	openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1
 
 rocksdbjavastaticosx_ub: rocksdbjavastaticosx_archs
-	lipo -create -output ./java/target/$(ROCKSDBJNILIB) java/target/librocksdbjni-osx-x86_64.jnilib java/target/librocksdbjni-osx-arm64.jnilib
-	$(MAKE) rocksdbjavastatic_jar
+	cd java/target; lipo -create -output librocksdbjni-osx.jnilib librocksdbjni-osx-x86_64.jnilib librocksdbjni-osx-arm64.jnilib
+	cd java; $(JAR_CMD)  -cf target/$(ROCKSDB_JAR) HISTORY*.md
+	cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) librocksdbjni-osx.jnilib
+	cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
+	openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1
 
 rocksdbjavastaticosx_archs:
 	$(MAKE) rocksdbjavastaticosx_arch_x86_64
@@ -2289,28 +2304,32 @@ rocksdbjavastatic_javalib:
 		strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \
 	fi
 
-rocksdbjavastatic_jar:
+rocksdbjava_jar:
 	cd java; $(JAR_CMD)  -cf target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
 	cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
-	cd java/target/apidocs; $(JAR_CMD) -cf ../$(ROCKSDB_JAVADOCS_JAR) *
-	cd java/src/main/java; $(JAR_CMD) -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org
 	openssl sha1 java/target/$(ROCKSDB_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR).sha1
+
+rocksdbjava_javadocs_jar:
+	cd java/target/apidocs; $(JAR_CMD) -cf ../$(ROCKSDB_JAVADOCS_JAR) *
 	openssl sha1 java/target/$(ROCKSDB_JAVADOCS_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAVADOCS_JAR).sha1
+
+rocksdbjava_sources_jar:
+	cd java/src/main/java; $(JAR_CMD) -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org
 	openssl sha1 java/target/$(ROCKSDB_SOURCES_JAR) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_SOURCES_JAR).sha1
 
 rocksdbjavastatic_deps: $(JAVA_COMPRESSIONS)
 
 rocksdbjavastatic_libobjects: $(LIB_OBJECTS)
 
-rocksdbjavastaticrelease: rocksdbjavastaticosx
+rocksdbjavastaticrelease: rocksdbjavastaticosx rocksdbjava_javadocs_jar rocksdbjava_sources_jar
 	cd java/crossbuild && (vagrant destroy -f || true) && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64 && vagrant up linux64-musl && vagrant halt linux64-musl
 	cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
 	cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
 	cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
 	openssl sha1 java/target/$(ROCKSDB_JAR_ALL) | sed 's/.*= \([0-9a-f]*\)/\1/' > java/target/$(ROCKSDB_JAR_ALL).sha1
 
-rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl
+rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl rocksdbjava_javadocs_jar rocksdbjava_sources_jar
 	cd java; $(JAR_CMD) -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
 	cd java/target; $(JAR_CMD) -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
 	cd java/target/classes; $(JAR_CMD) -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
@@ -2513,6 +2532,9 @@ endif
 build_subset_tests: $(ROCKSDBTESTS_SUBSET)
 	$(AM_V_GEN)if [ -n "$${ROCKSDBTESTS_SUBSET_TESTS_TO_FILE}" ]; then echo "$(ROCKSDBTESTS_SUBSET)" > "$${ROCKSDBTESTS_SUBSET_TESTS_TO_FILE}"; else echo "$(ROCKSDBTESTS_SUBSET)"; fi
 
+list_all_tests:
+	echo "$(ROCKSDBTESTS_SUBSET)"
+
 # Remove the rules for which dependencies should not be generated and see if any are left.
 #If so, include the dependencies; if not, do not include the dependency files
 ROCKS_DEP_RULES=$(filter-out clean format check-format check-buck-targets check-headers check-sources jclean jtest package analyze tags rocksdbjavastatic% unity.% unity_test, $(MAKECMDGOALS))

buckifier/buckify_rocksdb.py

@@ -210,14 +210,34 @@ def generate_targets(repo_path, deps_map):
         with open(f"{repo_path}/buckifier/bench.json") as json_file:
             fast_fancy_bench_config_list = json.load(json_file)
             for config_dict in fast_fancy_bench_config_list:
-                TARGETS.add_fancy_bench_config(config_dict['name'],config_dict['benchmarks'], False, config_dict['expected_runtime'])
+                clean_benchmarks = {}
+                benchmarks = config_dict['benchmarks']
+                for binary, benchmark_dict in benchmarks.items():
+                    clean_benchmarks[binary] = {}
+                    for benchmark, overloaded_metric_list in benchmark_dict.items():
+                        clean_benchmarks[binary][benchmark] = []
+                        for metric in overloaded_metric_list:
+                            if not isinstance(metric, dict):
+                                clean_benchmarks[binary][benchmark].append(metric)
+                TARGETS.add_fancy_bench_config(config_dict['name'], clean_benchmarks, False, config_dict['expected_runtime_one_iter'], config_dict['sl_iterations'], config_dict['regression_threshold'])
 
         with open(f"{repo_path}/buckifier/bench-slow.json") as json_file:
             slow_fancy_bench_config_list = json.load(json_file)
             for config_dict in slow_fancy_bench_config_list:
-                TARGETS.add_fancy_bench_config(config_dict['name']+"_slow",config_dict['benchmarks'], True, config_dict['expected_runtime'])
-
-    except (FileNotFoundError, KeyError):
+                clean_benchmarks = {}
+                benchmarks = config_dict['benchmarks']
+                for binary, benchmark_dict in benchmarks.items():
+                    clean_benchmarks[binary] = {}
+                    for benchmark, overloaded_metric_list in benchmark_dict.items():
+                        clean_benchmarks[binary][benchmark] = []
+                        for metric in overloaded_metric_list:
+                            if not isinstance(metric, dict):
+                                clean_benchmarks[binary][benchmark].append(metric)
+            for config_dict in slow_fancy_bench_config_list:
+                TARGETS.add_fancy_bench_config(config_dict['name']+"_slow", clean_benchmarks, True, config_dict['expected_runtime_one_iter'], config_dict['sl_iterations'], config_dict['regression_threshold'])
+    # it is better servicelab experiments break
+    # than rocksdb github ci
+    except Exception:
         pass
 
     TARGETS.add_test_header()

buckifier/targets_builder.py

@@ -92,12 +92,14 @@ add_c_test_wrapper()
         # will not be included.
 """)
 
-    def add_fancy_bench_config(self, name, bench_config, slow, expected_runtime):
+    def add_fancy_bench_config(self, name, bench_config, slow, expected_runtime, sl_iterations, regression_threshold):
         self.targets_file.write(targets_cfg.fancy_bench_template.format(
                     name=name,
                     bench_config=pprint.pformat(bench_config),
                     slow=slow,
                     expected_runtime=expected_runtime,
+                    sl_iterations=sl_iterations,
+                    regression_threshold=regression_threshold
                     ).encode("utf-8"))
 
     def register_test(self,

buckifier/targets_cfg.py

@@ -41,6 +41,6 @@ cpp_unittest_wrapper(name="{test_name}",
 """
 
 fancy_bench_template = """
-fancy_bench_wrapper(suite_name="{name}", binary_to_bench_to_metric_list_map={bench_config}, slow={slow}, expected_runtime={expected_runtime})
+fancy_bench_wrapper(suite_name="{name}", binary_to_bench_to_metric_list_map={bench_config}, slow={slow}, expected_runtime={expected_runtime}, sl_iterations={sl_iterations}, regression_threshold={regression_threshold})
 
 """

build_tools/build_detect_platform

@@ -632,12 +632,15 @@ if test "0$PORTABLE" -eq 0; then
     COMMON_FLAGS="$COMMON_FLAGS"
   elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
     if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ \
-      -fsyntax-only -march=native - -o /dev/null 2>/dev/null; then
+      -march=native - -o /dev/null 2>/dev/null; then
       COMMON_FLAGS="$COMMON_FLAGS -march=native "
     else
       COMMON_FLAGS="$COMMON_FLAGS -march=z196 "
     fi
     COMMON_FLAGS="$COMMON_FLAGS"
+  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^riscv64`"; then
+    RISC_ISA=$(cat /proc/cpuinfo | grep isa | head -1 | cut --delimiter=: -f 2 | cut -b 2-)
+    COMMON_FLAGS="$COMMON_FLAGS -march=${RISC_ISA}"
   elif [ "$TARGET_OS" == "IOS" ]; then
     COMMON_FLAGS="$COMMON_FLAGS"
   elif [ "$TARGET_OS" == "AIX" ] || [ "$TARGET_OS" == "SunOS" ]; then
@@ -658,6 +661,11 @@ else
     COMMON_FLAGS="$COMMON_FLAGS -march=z196 "
   fi
 
+  if test -n "`echo $TARGET_ARCHITECTURE | grep ^riscv64`"; then
+    RISC_ISA=$(cat /proc/cpuinfo | grep isa | head -1 | cut --delimiter=: -f 2 | cut -b 2-)
+    COMMON_FLAGS="$COMMON_FLAGS -march=${RISC_ISA}"
+  fi
+
   if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then
     # For portability compile for macOS 10.12 (2016) or newer
     COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.12"

build_tools/rocksdb-lego-determinator

@@ -110,8 +110,9 @@ CLANG="USE_CLANG=1"
 TSAN="COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 ASAN_CRASH="ASAN_OPTIONS=disable_coredump=0"
-TSAN_CRASH="CRASH_TEST_EXT_ARGS=\\\"--compression_type=zstd --log2_keys_per_lock=22\\\""
+TSAN_CRASH="TSAN_OPTIONS=disable_coredump=0 CRASH_TEST_EXT_ARGS=\\\"--compression_type=zstd --log2_keys_per_lock=22\\\""
 NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd"
+UBSAN_CRASH="UBSAN_OPTIONS=disable_coredump=0"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
 SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH"
@@ -760,7 +761,7 @@ UBSAN_CRASH_TEST_COMMANDS="[
             {
                 \"name\":\"Build and run RocksDB debug ubsan_crash_test\",
                 \"timeout\": 86400,
-                \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL\",
+                \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $UBSAN_CRASH $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL\",
                 \"user\":\"root\",
                 $PARSER
             },
@@ -784,7 +785,7 @@ UBSAN_BLACKBOX_CRASH_TEST_COMMANDS="[
             {
                 \"name\":\"Build and run RocksDB debug blackbox ubsan_crash_test\",
                 \"timeout\": 86400,
-                \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM blackbox_ubsan_crash_test || $CONTRUN_NAME=blackbox_ubsan_crash_test $TASK_CREATION_TOOL\",
+                \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $UBSAN_CRASH $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM blackbox_ubsan_crash_test || $CONTRUN_NAME=blackbox_ubsan_crash_test $TASK_CREATION_TOOL\",
                 \"user\":\"root\",
                 $PARSER
             },
@@ -808,7 +809,7 @@ UBSAN_WHITEBOX_CRASH_TEST_COMMANDS="[
             {
                 \"name\":\"Build and run RocksDB debug whitebox ubsan_crash_test\",
                 \"timeout\": 86400,
-                \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM whitebox_ubsan_crash_test || $CONTRUN_NAME=whitebox_ubsan_crash_test $TASK_CREATION_TOOL\",
+                \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $UBSAN_CRASH $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM whitebox_ubsan_crash_test || $CONTRUN_NAME=whitebox_ubsan_crash_test $TASK_CREATION_TOOL\",
                 \"user\":\"root\",
                 $PARSER
             },
@@ -832,7 +833,7 @@ UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
             {
                 \"name\":\"Build and run RocksDB debug ubsan_crash_test_with_atomic_flush\",
                 \"timeout\": 86400,
-                \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL\",
+                \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $UBSAN_CRASH $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL\",
                 \"user\":\"root\",
                 $PARSER
             },
@@ -856,7 +857,7 @@ UBSAN_CRASH_TEST_WITH_TXN_COMMANDS="[
             {
                 \"name\":\"Build and run RocksDB debug ubsan_crash_test_with_txn\",
                 \"timeout\": 86400,
-                \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL\",
+                \"shell\":\"cd $WORKING_DIR; $SHM $DEBUG $UBSAN_CRASH $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL\",
                 \"user\":\"root\",
                 $PARSER
             },

cache/clock_cache.cc

@@ -687,7 +687,7 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
   Status s;
   if (out_handle != nullptr) {
     if (handle == nullptr) {
-      s = Status::Incomplete("Insert failed due to LRU cache being full.");
+      s = Status::Incomplete("Insert failed due to CLOCK cache being full.");
     } else {
       *out_handle = reinterpret_cast<Cache::Handle*>(handle);
     }

cache/lru_secondary_cache.cc

@@ -0,0 +1,168 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "cache/lru_secondary_cache.h"
+
+#include <memory>
+
+#include "memory/memory_allocator.h"
+#include "util/compression.h"
+#include "util/string_util.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+namespace {
+
+void DeletionCallback(const Slice& /*key*/, void* obj) {
+  delete reinterpret_cast<CacheAllocationPtr*>(obj);
+  obj = nullptr;
+}
+
+}  // namespace
+
+LRUSecondaryCache::LRUSecondaryCache(
+    size_t capacity, int num_shard_bits, bool strict_capacity_limit,
+    double high_pri_pool_ratio,
+    std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
+    CacheMetadataChargePolicy metadata_charge_policy,
+    CompressionType compression_type, uint32_t compress_format_version)
+    : cache_options_(capacity, num_shard_bits, strict_capacity_limit,
+                     high_pri_pool_ratio, memory_allocator, use_adaptive_mutex,
+                     metadata_charge_policy, compression_type,
+                     compress_format_version) {
+  cache_ = NewLRUCache(capacity, num_shard_bits, strict_capacity_limit,
+                       high_pri_pool_ratio, memory_allocator,
+                       use_adaptive_mutex, metadata_charge_policy);
+}
+
+LRUSecondaryCache::~LRUSecondaryCache() { cache_.reset(); }
+
+std::unique_ptr<SecondaryCacheResultHandle> LRUSecondaryCache::Lookup(
+    const Slice& key, const Cache::CreateCallback& create_cb, bool /*wait*/) {
+  std::unique_ptr<SecondaryCacheResultHandle> handle;
+  Cache::Handle* lru_handle = cache_->Lookup(key);
+  if (lru_handle == nullptr) {
+    return handle;
+  }
+
+  CacheAllocationPtr* ptr =
+      reinterpret_cast<CacheAllocationPtr*>(cache_->Value(lru_handle));
+  void* value = nullptr;
+  size_t charge = 0;
+  Status s;
+
+  if (cache_options_.compression_type == kNoCompression) {
+    s = create_cb(ptr->get(), cache_->GetCharge(lru_handle), &value, &charge);
+  } else {
+    UncompressionContext uncompression_context(cache_options_.compression_type);
+    UncompressionInfo uncompression_info(uncompression_context,
+                                         UncompressionDict::GetEmptyDict(),
+                                         cache_options_.compression_type);
+
+    size_t uncompressed_size = 0;
+    CacheAllocationPtr uncompressed;
+    uncompressed = UncompressData(
+        uncompression_info, (char*)ptr->get(), cache_->GetCharge(lru_handle),
+        &uncompressed_size, cache_options_.compress_format_version,
+        cache_options_.memory_allocator.get());
+
+    if (!uncompressed) {
+      cache_->Release(lru_handle, true);
+      return handle;
+    }
+    s = create_cb(uncompressed.get(), uncompressed_size, &value, &charge);
+  }
+
+  if (!s.ok()) {
+    cache_->Release(lru_handle, true);
+    return handle;
+  }
+
+  handle.reset(new LRUSecondaryCacheResultHandle(value, charge));
+  cache_->Release(lru_handle);
+  return handle;
+}
+
+Status LRUSecondaryCache::Insert(const Slice& key, void* value,
+                                 const Cache::CacheItemHelper* helper) {
+  size_t size = (*helper->size_cb)(value);
+  CacheAllocationPtr ptr =
+      AllocateBlock(size, cache_options_.memory_allocator.get());
+
+  Status s = (*helper->saveto_cb)(value, 0, size, ptr.get());
+  if (!s.ok()) {
+    return s;
+  }
+  Slice val(ptr.get(), size);
+
+  std::string compressed_val;
+  if (cache_options_.compression_type != kNoCompression) {
+    CompressionOptions compression_opts;
+    CompressionContext compression_context(cache_options_.compression_type);
+    uint64_t sample_for_compression = 0;
+    CompressionInfo compression_info(
+        compression_opts, compression_context, CompressionDict::GetEmptyDict(),
+        cache_options_.compression_type, sample_for_compression);
+
+    bool success =
+        CompressData(val, compression_info,
+                     cache_options_.compress_format_version, &compressed_val);
+
+    if (!success) {
+      return Status::Corruption("Error compressing value.");
+    }
+
+    val = Slice(compressed_val);
+    size = compressed_val.size();
+    ptr = AllocateBlock(size, cache_options_.memory_allocator.get());
+    memcpy(ptr.get(), compressed_val.data(), size);
+  }
+
+  CacheAllocationPtr* buf = new CacheAllocationPtr(std::move(ptr));
+
+  return cache_->Insert(key, buf, size, DeletionCallback);
+}
+
+void LRUSecondaryCache::Erase(const Slice& key) { cache_->Erase(key); }
+
+std::string LRUSecondaryCache::GetPrintableOptions() const {
+  std::string ret;
+  ret.reserve(20000);
+  const int kBufferSize = 200;
+  char buffer[kBufferSize];
+  ret.append(cache_->GetPrintableOptions());
+  snprintf(buffer, kBufferSize, "    compression_type : %s\n",
+           CompressionTypeToString(cache_options_.compression_type).c_str());
+  ret.append(buffer);
+  snprintf(buffer, kBufferSize, "    compression_type : %d\n",
+           cache_options_.compress_format_version);
+  ret.append(buffer);
+  return ret;
+}
+
+std::shared_ptr<SecondaryCache> NewLRUSecondaryCache(
+    size_t capacity, int num_shard_bits, bool strict_capacity_limit,
+    double high_pri_pool_ratio,
+    std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
+    CacheMetadataChargePolicy metadata_charge_policy,
+    CompressionType compression_type, uint32_t compress_format_version) {
+  return std::make_shared<LRUSecondaryCache>(
+      capacity, num_shard_bits, strict_capacity_limit, high_pri_pool_ratio,
+      memory_allocator, use_adaptive_mutex, metadata_charge_policy,
+      compression_type, compress_format_version);
+}
+
+std::shared_ptr<SecondaryCache> NewLRUSecondaryCache(
+    const LRUSecondaryCacheOptions& opts) {
+  // The secondary_cache is disabled for this LRUCache instance.
+  assert(opts.secondary_cache == nullptr);
+  return NewLRUSecondaryCache(
+      opts.capacity, opts.num_shard_bits, opts.strict_capacity_limit,
+      opts.high_pri_pool_ratio, opts.memory_allocator, opts.use_adaptive_mutex,
+      opts.metadata_charge_policy, opts.compression_type,
+      opts.compress_format_version);
+}
+
+}  // namespace ROCKSDB_NAMESPACE

cache/lru_secondary_cache.h

@@ -0,0 +1,85 @@
+// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <memory>
+
+#include "cache/lru_cache.h"
+#include "memory/memory_allocator.h"
+#include "rocksdb/secondary_cache.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
+#include "util/compression.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+class LRUSecondaryCacheResultHandle : public SecondaryCacheResultHandle {
+ public:
+  LRUSecondaryCacheResultHandle(void* value, size_t size)
+      : value_(value), size_(size) {}
+  virtual ~LRUSecondaryCacheResultHandle() override = default;
+
+  LRUSecondaryCacheResultHandle(const LRUSecondaryCacheResultHandle&) = delete;
+  LRUSecondaryCacheResultHandle& operator=(
+      const LRUSecondaryCacheResultHandle&) = delete;
+
+  bool IsReady() override { return true; }
+
+  void Wait() override {}
+
+  void* Value() override { return value_; }
+
+  size_t Size() override { return size_; }
+
+ private:
+  void* value_;
+  size_t size_;
+};
+
+// The LRUSecondaryCache is a concrete implementation of
+// rocksdb::SecondaryCache.
+//
+// Users can also cast a pointer to it and call methods on
+// it directly, especially custom methods that may be added
+// in the future.  For example -
+// std::unique_ptr<rocksdb::SecondaryCache> cache =
+//      NewLRUSecondaryCache(opts);
+// static_cast<LRUSecondaryCache*>(cache.get())->Erase(key);
+
+class LRUSecondaryCache : public SecondaryCache {
+ public:
+  LRUSecondaryCache(
+      size_t capacity, int num_shard_bits, bool strict_capacity_limit,
+      double high_pri_pool_ratio,
+      std::shared_ptr<MemoryAllocator> memory_allocator = nullptr,
+      bool use_adaptive_mutex = kDefaultToAdaptiveMutex,
+      CacheMetadataChargePolicy metadata_charge_policy =
+          kDontChargeCacheMetadata,
+      CompressionType compression_type = CompressionType::kLZ4Compression,
+      uint32_t compress_format_version = 2);
+  virtual ~LRUSecondaryCache() override;
+
+  const char* Name() const override { return "LRUSecondaryCache"; }
+
+  Status Insert(const Slice& key, void* value,
+                const Cache::CacheItemHelper* helper) override;
+
+  std::unique_ptr<SecondaryCacheResultHandle> Lookup(
+      const Slice& key, const Cache::CreateCallback& create_cb,
+      bool /*wait*/) override;
+
+  void Erase(const Slice& key) override;
+
+  void WaitAll(std::vector<SecondaryCacheResultHandle*> /*handles*/) override {}
+
+  std::string GetPrintableOptions() const override;
+
+ private:
+  std::shared_ptr<Cache> cache_;
+  LRUSecondaryCacheOptions cache_options_;
+};
+
+}  // namespace ROCKSDB_NAMESPACE

cache/lru_secondary_cache_test.cc

@@ -0,0 +1,597 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "cache/lru_secondary_cache.h"
+
+#include <algorithm>
+#include <cstdint>
+
+#include "memory/jemalloc_nodump_allocator.h"
+#include "memory/memory_allocator.h"
+#include "test_util/testharness.h"
+#include "test_util/testutil.h"
+#include "util/compression.h"
+#include "util/random.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+class LRUSecondaryCacheTest : public testing::Test {
+ public:
+  LRUSecondaryCacheTest() : fail_create_(false) {}
+  ~LRUSecondaryCacheTest() {}
+
+ protected:
+  class TestItem {
+   public:
+    TestItem(const char* buf, size_t size) : buf_(new char[size]), size_(size) {
+      memcpy(buf_.get(), buf, size);
+    }
+    ~TestItem() {}
+
+    char* Buf() { return buf_.get(); }
+    size_t Size() { return size_; }
+
+   private:
+    std::unique_ptr<char[]> buf_;
+    size_t size_;
+  };
+
+  static size_t SizeCallback(void* obj) {
+    return reinterpret_cast<TestItem*>(obj)->Size();
+  }
+
+  static Status SaveToCallback(void* from_obj, size_t from_offset,
+                               size_t length, void* out) {
+    TestItem* item = reinterpret_cast<TestItem*>(from_obj);
+    const char* buf = item->Buf();
+    EXPECT_EQ(length, item->Size());
+    EXPECT_EQ(from_offset, 0);
+    memcpy(out, buf, length);
+    return Status::OK();
+  }
+
+  static void DeletionCallback(const Slice& /*key*/, void* obj) {
+    delete reinterpret_cast<TestItem*>(obj);
+    obj = nullptr;
+  }
+
+  static Cache::CacheItemHelper helper_;
+
+  static Status SaveToCallbackFail(void* /*obj*/, size_t /*offset*/,
+                                   size_t /*size*/, void* /*out*/) {
+    return Status::NotSupported();
+  }
+
+  static Cache::CacheItemHelper helper_fail_;
+
+  Cache::CreateCallback test_item_creator = [&](const void* buf, size_t size,
+                                                void** out_obj,
+                                                size_t* charge) -> Status {
+    if (fail_create_) {
+      return Status::NotSupported();
+    }
+    *out_obj = reinterpret_cast<void*>(new TestItem((char*)buf, size));
+    *charge = size;
+    return Status::OK();
+  };
+
+  void SetFailCreate(bool fail) { fail_create_ = fail; }
+
+  void BasicTest(bool sec_cache_is_compressed, bool use_jemalloc) {
+    LRUSecondaryCacheOptions opts;
+    opts.capacity = 2048;
+    opts.num_shard_bits = 0;
+    opts.metadata_charge_policy = kDontChargeCacheMetadata;
+
+    if (sec_cache_is_compressed) {
+      if (!LZ4_Supported()) {
+        ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+        opts.compression_type = CompressionType::kNoCompression;
+      }
+    } else {
+      opts.compression_type = CompressionType::kNoCompression;
+    }
+
+    if (use_jemalloc) {
+      JemallocAllocatorOptions jopts;
+      std::shared_ptr<MemoryAllocator> allocator;
+      std::string msg;
+      if (JemallocNodumpAllocator::IsSupported(&msg)) {
+        Status s = NewJemallocNodumpAllocator(jopts, &allocator);
+        if (s.ok()) {
+          opts.memory_allocator = allocator;
+        }
+      } else {
+        ROCKSDB_GTEST_BYPASS("JEMALLOC not supported");
+      }
+    }
+    std::shared_ptr<SecondaryCache> cache = NewLRUSecondaryCache(opts);
+
+    // Lookup an non-existent key.
+    std::unique_ptr<SecondaryCacheResultHandle> handle0 =
+        cache->Lookup("k0", test_item_creator, true);
+    ASSERT_EQ(handle0, nullptr);
+
+    Random rnd(301);
+    // Insert and Lookup the first item.
+    std::string str1;
+    test::CompressibleString(&rnd, 0.25, 1000, &str1);
+    TestItem item1(str1.data(), str1.length());
+    ASSERT_OK(cache->Insert("k1", &item1, &LRUSecondaryCacheTest::helper_));
+    std::unique_ptr<SecondaryCacheResultHandle> handle1 =
+        cache->Lookup("k1", test_item_creator, true);
+    ASSERT_NE(handle1, nullptr);
+    // delete reinterpret_cast<TestItem*>(handle1->Value());
+    std::unique_ptr<TestItem> val1 =
+        std::unique_ptr<TestItem>(static_cast<TestItem*>(handle1->Value()));
+    ASSERT_NE(val1, nullptr);
+    ASSERT_EQ(memcmp(val1->Buf(), item1.Buf(), item1.Size()), 0);
+
+    // Insert and Lookup the second item.
+    std::string str2;
+    test::CompressibleString(&rnd, 0.5, 1000, &str2);
+    TestItem item2(str2.data(), str2.length());
+    ASSERT_OK(cache->Insert("k2", &item2, &LRUSecondaryCacheTest::helper_));
+    std::unique_ptr<SecondaryCacheResultHandle> handle2 =
+        cache->Lookup("k2", test_item_creator, true);
+    ASSERT_NE(handle2, nullptr);
+    std::unique_ptr<TestItem> val2 =
+        std::unique_ptr<TestItem>(static_cast<TestItem*>(handle2->Value()));
+    ASSERT_NE(val2, nullptr);
+    ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0);
+
+    // Lookup the first item again to make sure it is still in the cache.
+    std::unique_ptr<SecondaryCacheResultHandle> handle1_1 =
+        cache->Lookup("k1", test_item_creator, true);
+    ASSERT_NE(handle1_1, nullptr);
+    std::unique_ptr<TestItem> val1_1 =
+        std::unique_ptr<TestItem>(static_cast<TestItem*>(handle1_1->Value()));
+    ASSERT_NE(val1_1, nullptr);
+    ASSERT_EQ(memcmp(val1_1->Buf(), item1.Buf(), item1.Size()), 0);
+
+    std::vector<SecondaryCacheResultHandle*> handles = {handle1.get(),
+                                                        handle2.get()};
+    cache->WaitAll(handles);
+
+    cache->Erase("k1");
+    handle1 = cache->Lookup("k1", test_item_creator, true);
+    ASSERT_EQ(handle1, nullptr);
+
+    cache.reset();
+  }
+
+  void FailsTest(bool sec_cache_is_compressed) {
+    LRUSecondaryCacheOptions secondary_cache_opts;
+    if (sec_cache_is_compressed) {
+      if (!LZ4_Supported()) {
+        ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+        secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+      }
+    } else {
+      secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+    }
+
+    secondary_cache_opts.capacity = 1100;
+    secondary_cache_opts.num_shard_bits = 0;
+    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
+    std::shared_ptr<SecondaryCache> cache =
+        NewLRUSecondaryCache(secondary_cache_opts);
+
+    // Insert and Lookup the first item.
+    Random rnd(301);
+    std::string str1(rnd.RandomString(1000));
+    TestItem item1(str1.data(), str1.length());
+    ASSERT_OK(cache->Insert("k1", &item1, &LRUSecondaryCacheTest::helper_));
+    std::unique_ptr<SecondaryCacheResultHandle> handle1 =
+        cache->Lookup("k1", test_item_creator, true);
+    ASSERT_NE(handle1, nullptr);
+    std::unique_ptr<TestItem> val1 =
+        std::unique_ptr<TestItem>(static_cast<TestItem*>(handle1->Value()));
+    ASSERT_NE(val1, nullptr);
+    ASSERT_EQ(memcmp(val1->Buf(), item1.Buf(), item1.Size()), 0);
+
+    // Insert and Lookup the second item.
+    std::string str2(rnd.RandomString(200));
+    TestItem item2(str2.data(), str2.length());
+    // k1 is evicted.
+    ASSERT_OK(cache->Insert("k2", &item2, &LRUSecondaryCacheTest::helper_));
+    std::unique_ptr<SecondaryCacheResultHandle> handle1_1 =
+        cache->Lookup("k1", test_item_creator, true);
+    ASSERT_EQ(handle1_1, nullptr);
+    std::unique_ptr<SecondaryCacheResultHandle> handle2 =
+        cache->Lookup("k2", test_item_creator, true);
+    ASSERT_NE(handle2, nullptr);
+    std::unique_ptr<TestItem> val2 =
+        std::unique_ptr<TestItem>(static_cast<TestItem*>(handle2->Value()));
+    ASSERT_NE(val2, nullptr);
+    ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0);
+
+    // Create Fails.
+    SetFailCreate(true);
+    std::unique_ptr<SecondaryCacheResultHandle> handle2_1 =
+        cache->Lookup("k2", test_item_creator, true);
+    ASSERT_EQ(handle2_1, nullptr);
+
+    // Save Fails.
+    std::string str3 = rnd.RandomString(10);
+    TestItem item3(str3.data(), str3.length());
+    ASSERT_NOK(
+        cache->Insert("k3", &item3, &LRUSecondaryCacheTest::helper_fail_));
+
+    cache.reset();
+  }
+
+  void BasicIntegrationTest(bool sec_cache_is_compressed) {
+    LRUSecondaryCacheOptions secondary_cache_opts;
+
+    if (sec_cache_is_compressed) {
+      if (!LZ4_Supported()) {
+        ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+        secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+      }
+    } else {
+      secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+    }
+
+    secondary_cache_opts.capacity = 2300;
+    secondary_cache_opts.num_shard_bits = 0;
+    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
+    std::shared_ptr<SecondaryCache> secondary_cache =
+        NewLRUSecondaryCache(secondary_cache_opts);
+    LRUCacheOptions lru_cache_opts(1024, 0, false, 0.5, nullptr,
+                                   kDefaultToAdaptiveMutex,
+                                   kDontChargeCacheMetadata);
+    lru_cache_opts.secondary_cache = secondary_cache;
+    std::shared_ptr<Cache> cache = NewLRUCache(lru_cache_opts);
+    std::shared_ptr<Statistics> stats = CreateDBStatistics();
+
+    Random rnd(301);
+
+    std::string str1 = rnd.RandomString(1010);
+    std::string str1_clone{str1};
+    TestItem* item1 = new TestItem(str1.data(), str1.length());
+    ASSERT_OK(cache->Insert("k1", item1, &LRUSecondaryCacheTest::helper_,
+                            str1.length()));
+
+    std::string str2 = rnd.RandomString(1020);
+    TestItem* item2 = new TestItem(str2.data(), str2.length());
+    // After Insert, lru cache contains k2 and secondary cache contains k1.
+    ASSERT_OK(cache->Insert("k2", item2, &LRUSecondaryCacheTest::helper_,
+                            str2.length()));
+
+    std::string str3 = rnd.RandomString(1020);
+    TestItem* item3 = new TestItem(str3.data(), str3.length());
+    // After Insert, lru cache contains k3 and secondary cache contains k1 and
+    // k2
+    ASSERT_OK(cache->Insert("k3", item3, &LRUSecondaryCacheTest::helper_,
+                            str3.length()));
+
+    Cache::Handle* handle;
+    handle =
+        cache->Lookup("k3", &LRUSecondaryCacheTest::helper_, test_item_creator,
+                      Cache::Priority::LOW, true, stats.get());
+    ASSERT_NE(handle, nullptr);
+    TestItem* val3 = static_cast<TestItem*>(cache->Value(handle));
+    ASSERT_NE(val3, nullptr);
+    ASSERT_EQ(memcmp(val3->Buf(), item3->Buf(), item3->Size()), 0);
+    cache->Release(handle);
+
+    // Lookup an non-existent key.
+    handle =
+        cache->Lookup("k0", &LRUSecondaryCacheTest::helper_, test_item_creator,
+                      Cache::Priority::LOW, true, stats.get());
+    ASSERT_EQ(handle, nullptr);
+
+    // This Lookup should promote k1 and demote k3, so k2 is evicted from the
+    // secondary cache. The lru cache contains k1 and secondary cache contains
+    // k3. item1 was Free(), so it cannot be compared against the item1.
+    handle =
+        cache->Lookup("k1", &LRUSecondaryCacheTest::helper_, test_item_creator,
+                      Cache::Priority::LOW, true, stats.get());
+    ASSERT_NE(handle, nullptr);
+    TestItem* val1_1 = static_cast<TestItem*>(cache->Value(handle));
+    ASSERT_NE(val1_1, nullptr);
+    ASSERT_EQ(memcmp(val1_1->Buf(), str1_clone.data(), str1_clone.size()), 0);
+    cache->Release(handle);
+
+    handle =
+        cache->Lookup("k2", &LRUSecondaryCacheTest::helper_, test_item_creator,
+                      Cache::Priority::LOW, true, stats.get());
+    ASSERT_EQ(handle, nullptr);
+
+    cache.reset();
+    secondary_cache.reset();
+  }
+
+  void BasicIntegrationFailTest(bool sec_cache_is_compressed) {
+    LRUSecondaryCacheOptions secondary_cache_opts;
+
+    if (sec_cache_is_compressed) {
+      if (!LZ4_Supported()) {
+        ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+        secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+      }
+    } else {
+      secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+    }
+
+    secondary_cache_opts.capacity = 2048;
+    secondary_cache_opts.num_shard_bits = 0;
+    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
+    std::shared_ptr<SecondaryCache> secondary_cache =
+        NewLRUSecondaryCache(secondary_cache_opts);
+
+    LRUCacheOptions opts(1024, 0, false, 0.5, nullptr, kDefaultToAdaptiveMutex,
+                         kDontChargeCacheMetadata);
+    opts.secondary_cache = secondary_cache;
+    std::shared_ptr<Cache> cache = NewLRUCache(opts);
+
+    Random rnd(301);
+    std::string str1 = rnd.RandomString(1020);
+    auto item1 =
+        std::unique_ptr<TestItem>(new TestItem(str1.data(), str1.length()));
+    ASSERT_NOK(cache->Insert("k1", item1.get(), nullptr, str1.length()));
+    ASSERT_OK(cache->Insert("k1", item1.get(), &LRUSecondaryCacheTest::helper_,
+                            str1.length()));
+    item1.release();  // Appease clang-analyze "potential memory leak"
+
+    Cache::Handle* handle;
+    handle = cache->Lookup("k2", nullptr, test_item_creator,
+                           Cache::Priority::LOW, true);
+    ASSERT_EQ(handle, nullptr);
+    handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_,
+                           test_item_creator, Cache::Priority::LOW, false);
+    ASSERT_EQ(handle, nullptr);
+
+    cache.reset();
+    secondary_cache.reset();
+  }
+
+  void IntegrationSaveFailTest(bool sec_cache_is_compressed) {
+    LRUSecondaryCacheOptions secondary_cache_opts;
+
+    if (sec_cache_is_compressed) {
+      if (!LZ4_Supported()) {
+        ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+        secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+      }
+    } else {
+      secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+    }
+
+    secondary_cache_opts.capacity = 2048;
+    secondary_cache_opts.num_shard_bits = 0;
+    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
+
+    std::shared_ptr<SecondaryCache> secondary_cache =
+        NewLRUSecondaryCache(secondary_cache_opts);
+
+    LRUCacheOptions opts(1024, 0, false, 0.5, nullptr, kDefaultToAdaptiveMutex,
+                         kDontChargeCacheMetadata);
+    opts.secondary_cache = secondary_cache;
+    std::shared_ptr<Cache> cache = NewLRUCache(opts);
+
+    Random rnd(301);
+    std::string str1 = rnd.RandomString(1020);
+    TestItem* item1 = new TestItem(str1.data(), str1.length());
+    ASSERT_OK(cache->Insert("k1", item1, &LRUSecondaryCacheTest::helper_fail_,
+                            str1.length()));
+    std::string str2 = rnd.RandomString(1020);
+    TestItem* item2 = new TestItem(str2.data(), str2.length());
+    // k1 should be demoted to the secondary cache.
+    ASSERT_OK(cache->Insert("k2", item2, &LRUSecondaryCacheTest::helper_fail_,
+                            str2.length()));
+
+    Cache::Handle* handle;
+    handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_fail_,
+                           test_item_creator, Cache::Priority::LOW, true);
+    ASSERT_NE(handle, nullptr);
+    cache->Release(handle);
+    // This lookup should fail, since k1 demotion would have failed
+    handle = cache->Lookup("k1", &LRUSecondaryCacheTest::helper_fail_,
+                           test_item_creator, Cache::Priority::LOW, true);
+    ASSERT_EQ(handle, nullptr);
+    // Since k1 didn't get promoted, k2 should still be in cache
+    handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_fail_,
+                           test_item_creator, Cache::Priority::LOW, true);
+    ASSERT_NE(handle, nullptr);
+    cache->Release(handle);
+
+    cache.reset();
+    secondary_cache.reset();
+  }
+
+  void IntegrationCreateFailTest(bool sec_cache_is_compressed) {
+    LRUSecondaryCacheOptions secondary_cache_opts;
+
+    if (sec_cache_is_compressed) {
+      if (!LZ4_Supported()) {
+        ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+        secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+      }
+    } else {
+      secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+    }
+
+    secondary_cache_opts.capacity = 2048;
+    secondary_cache_opts.num_shard_bits = 0;
+    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
+
+    std::shared_ptr<SecondaryCache> secondary_cache =
+        NewLRUSecondaryCache(secondary_cache_opts);
+
+    LRUCacheOptions opts(1024, 0, false, 0.5, nullptr, kDefaultToAdaptiveMutex,
+                         kDontChargeCacheMetadata);
+    opts.secondary_cache = secondary_cache;
+    std::shared_ptr<Cache> cache = NewLRUCache(opts);
+
+    Random rnd(301);
+    std::string str1 = rnd.RandomString(1020);
+    TestItem* item1 = new TestItem(str1.data(), str1.length());
+    ASSERT_OK(cache->Insert("k1", item1, &LRUSecondaryCacheTest::helper_,
+                            str1.length()));
+
+    std::string str2 = rnd.RandomString(1020);
+    TestItem* item2 = new TestItem(str2.data(), str2.length());
+    // k1 should be demoted to the secondary cache.
+    ASSERT_OK(cache->Insert("k2", item2, &LRUSecondaryCacheTest::helper_,
+                            str2.length()));
+
+    Cache::Handle* handle;
+    SetFailCreate(true);
+    handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_,
+                           test_item_creator, Cache::Priority::LOW, true);
+    ASSERT_NE(handle, nullptr);
+    cache->Release(handle);
+    // This lookup should fail, since k1 creation would have failed
+    handle = cache->Lookup("k1", &LRUSecondaryCacheTest::helper_,
+                           test_item_creator, Cache::Priority::LOW, true);
+    ASSERT_EQ(handle, nullptr);
+    // Since k1 didn't get promoted, k2 should still be in cache
+    handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_,
+                           test_item_creator, Cache::Priority::LOW, true);
+    ASSERT_NE(handle, nullptr);
+    cache->Release(handle);
+
+    cache.reset();
+    secondary_cache.reset();
+  }
+
+  void IntegrationFullCapacityTest(bool sec_cache_is_compressed) {
+    LRUSecondaryCacheOptions secondary_cache_opts;
+
+    if (sec_cache_is_compressed) {
+      if (!LZ4_Supported()) {
+        ROCKSDB_GTEST_SKIP("This test requires LZ4 support.");
+        secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+      }
+    } else {
+      secondary_cache_opts.compression_type = CompressionType::kNoCompression;
+    }
+
+    secondary_cache_opts.capacity = 2048;
+    secondary_cache_opts.num_shard_bits = 0;
+    secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
+
+    std::shared_ptr<SecondaryCache> secondary_cache =
+        NewLRUSecondaryCache(secondary_cache_opts);
+
+    LRUCacheOptions opts(1024, 0, /*_strict_capacity_limit=*/true, 0.5, nullptr,
+                         kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
+    opts.secondary_cache = secondary_cache;
+    std::shared_ptr<Cache> cache = NewLRUCache(opts);
+
+    Random rnd(301);
+    std::string str1 = rnd.RandomString(1020);
+    TestItem* item1 = new TestItem(str1.data(), str1.length());
+    ASSERT_OK(cache->Insert("k1", item1, &LRUSecondaryCacheTest::helper_,
+                            str1.length()));
+    std::string str2 = rnd.RandomString(1020);
+    TestItem* item2 = new TestItem(str2.data(), str2.length());
+    // k1 should be demoted to the secondary cache.
+    ASSERT_OK(cache->Insert("k2", item2, &LRUSecondaryCacheTest::helper_,
+                            str2.length()));
+
+    Cache::Handle* handle;
+    handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_,
+                           test_item_creator, Cache::Priority::LOW, true);
+    ASSERT_NE(handle, nullptr);
+    // k1 promotion should fail due to the block cache being at capacity,
+    // but the lookup should still succeed
+    Cache::Handle* handle2;
+    handle2 = cache->Lookup("k1", &LRUSecondaryCacheTest::helper_,
+                            test_item_creator, Cache::Priority::LOW, true);
+    ASSERT_NE(handle2, nullptr);
+    // Since k1 didn't get inserted, k2 should still be in cache
+    cache->Release(handle);
+    cache->Release(handle2);
+    handle = cache->Lookup("k2", &LRUSecondaryCacheTest::helper_,
+                           test_item_creator, Cache::Priority::LOW, true);
+    ASSERT_NE(handle, nullptr);
+    cache->Release(handle);
+
+    cache.reset();
+    secondary_cache.reset();
+  }
+
+ private:
+  bool fail_create_;
+};
+
+Cache::CacheItemHelper LRUSecondaryCacheTest::helper_(
+    LRUSecondaryCacheTest::SizeCallback, LRUSecondaryCacheTest::SaveToCallback,
+    LRUSecondaryCacheTest::DeletionCallback);
+
+Cache::CacheItemHelper LRUSecondaryCacheTest::helper_fail_(
+    LRUSecondaryCacheTest::SizeCallback,
+    LRUSecondaryCacheTest::SaveToCallbackFail,
+    LRUSecondaryCacheTest::DeletionCallback);
+
+TEST_F(LRUSecondaryCacheTest, BasicTestWithNoCompression) {
+  BasicTest(false, false);
+}
+
+TEST_F(LRUSecondaryCacheTest, BasicTestWithMemoryAllocatorAndNoCompression) {
+  BasicTest(false, true);
+}
+
+TEST_F(LRUSecondaryCacheTest, BasicTestWithCompression) {
+  BasicTest(true, false);
+}
+
+TEST_F(LRUSecondaryCacheTest, BasicTestWithMemoryAllocatorAndCompression) {
+  BasicTest(true, true);
+}
+
+TEST_F(LRUSecondaryCacheTest, FailsTestWithNoCompression) { FailsTest(false); }
+
+TEST_F(LRUSecondaryCacheTest, FailsTestWithCompression) { FailsTest(true); }
+
+TEST_F(LRUSecondaryCacheTest, BasicIntegrationTestWithNoCompression) {
+  BasicIntegrationTest(false);
+}
+
+TEST_F(LRUSecondaryCacheTest, BasicIntegrationTestWithCompression) {
+  BasicIntegrationTest(true);
+}
+
+TEST_F(LRUSecondaryCacheTest, BasicIntegrationFailTestWithNoCompression) {
+  BasicIntegrationFailTest(false);
+}
+
+TEST_F(LRUSecondaryCacheTest, BasicIntegrationFailTestWithCompression) {
+  BasicIntegrationFailTest(true);
+}
+
+TEST_F(LRUSecondaryCacheTest, IntegrationSaveFailTestWithNoCompression) {
+  IntegrationSaveFailTest(false);
+}
+
+TEST_F(LRUSecondaryCacheTest, IntegrationSaveFailTestWithCompression) {
+  IntegrationSaveFailTest(true);
+}
+
+TEST_F(LRUSecondaryCacheTest, IntegrationCreateFailTestWithNoCompression) {
+  IntegrationCreateFailTest(false);
+}
+
+TEST_F(LRUSecondaryCacheTest, IntegrationCreateFailTestWithCompression) {
+  IntegrationCreateFailTest(true);
+}
+
+TEST_F(LRUSecondaryCacheTest, IntegrationFullCapacityTestWithNoCompression) {
+  IntegrationFullCapacityTest(false);
+}
+
+TEST_F(LRUSecondaryCacheTest, IntegrationFullCapacityTestWithCompression) {
+  IntegrationFullCapacityTest(true);
+}
+
+}  // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

db/c.cc

@@ -2297,11 +2297,6 @@ void rocksdb_block_based_options_set_data_block_hash_ratio(
   options->rep.data_block_hash_table_util_ratio = v;
 }
 
-void rocksdb_block_based_options_set_hash_index_allow_collision(
-    rocksdb_block_based_table_options_t* options, unsigned char v) {
-  options->rep.hash_index_allow_collision = v;
-}
-
 void rocksdb_block_based_options_set_cache_index_and_filter_blocks(
     rocksdb_block_based_table_options_t* options, unsigned char v) {
   options->rep.cache_index_and_filter_blocks = v;

db/compact_files_test.cc

@@ -138,6 +138,9 @@ TEST_F(CompactFilesTest, MultipleLevel) {
     collector->ClearFlushedFiles();
     ASSERT_OK(db->Put(WriteOptions(), ToString(i), ""));
     ASSERT_OK(db->Flush(FlushOptions()));
+    // Ensure background work is fully finished including listener callbacks
+    // before accessing listener state.
+    ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForBackgroundWork());
     auto l0_files = collector->GetFlushedFiles();
     ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files, i));
 
@@ -296,6 +299,9 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
     ASSERT_OK(db->Flush(FlushOptions()));
   }
 
+  // Ensure background work is fully finished including listener callbacks
+  // before accessing listener state.
+  ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForBackgroundWork());
   auto l0_files = collector->GetFlushedFiles();
   EXPECT_EQ(5, l0_files.size());
 
@@ -414,6 +420,9 @@ TEST_F(CompactFilesTest, SentinelCompressionType) {
     ASSERT_OK(db->Put(WriteOptions(), "key", "val"));
     ASSERT_OK(db->Flush(FlushOptions()));
 
+    // Ensure background work is fully finished including listener callbacks
+    // before accessing listener state.
+    ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForBackgroundWork());
     auto l0_files = collector->GetFlushedFiles();
     ASSERT_EQ(1, l0_files.size());
 

db/compaction/compaction.cc

@@ -277,9 +277,9 @@ Compaction::~Compaction() {
 
 bool Compaction::InputCompressionMatchesOutput() const {
   int base_level = input_vstorage_->base_level();
-  bool matches = (GetCompressionType(immutable_options_, input_vstorage_,
-                                     mutable_cf_options_, start_level_,
-                                     base_level) == output_compression_);
+  bool matches =
+      (GetCompressionType(input_vstorage_, mutable_cf_options_, start_level_,
+                          base_level) == output_compression_);
   if (matches) {
     TEST_SYNC_POINT("Compaction::InputCompressionMatchesOutput:Matches");
     return true;

db/compaction/compaction_picker.cc

@@ -100,8 +100,7 @@ bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
 // If enable_compression is false, then compression is always disabled no
 // matter what the values of the other two parameters are.
 // Otherwise, the compression type is determined based on options and level.
-CompressionType GetCompressionType(const ImmutableCFOptions& ioptions,
-                                   const VersionStorageInfo* vstorage,
+CompressionType GetCompressionType(const VersionStorageInfo* vstorage,
                                    const MutableCFOptions& mutable_cf_options,
                                    int level, int base_level,
                                    const bool enable_compression) {
@@ -118,17 +117,19 @@ CompressionType GetCompressionType(const ImmutableCFOptions& ioptions,
   }
   // If the user has specified a different compression level for each level,
   // then pick the compression for that level.
-  if (!ioptions.compression_per_level.empty()) {
+  if (!mutable_cf_options.compression_per_level.empty()) {
     assert(level == 0 || level >= base_level);
     int idx = (level == 0) ? 0 : level - base_level + 1;
 
-    const int n = static_cast<int>(ioptions.compression_per_level.size()) - 1;
+    const int n =
+        static_cast<int>(mutable_cf_options.compression_per_level.size()) - 1;
     // It is possible for level_ to be -1; in that case, we use level
     // 0's compression.  This occurs mostly in backwards compatibility
     // situations when the builder doesn't know what level the file
     // belongs to.  Likewise, if level is beyond the end of the
     // specified compression levels, use the last value.
-    return ioptions.compression_per_level[std::max(0, std::min(idx, n))];
+    return mutable_cf_options
+        .compression_per_level[std::max(0, std::min(idx, n))];
   } else {
     return mutable_cf_options.compression;
   }
@@ -347,9 +348,8 @@ Compaction* CompactionPicker::CompactFiles(
     } else {
       base_level = 1;
     }
-    compression_type =
-        GetCompressionType(ioptions_, vstorage, mutable_cf_options,
-                           output_level, base_level);
+    compression_type = GetCompressionType(vstorage, mutable_cf_options,
+                                          output_level, base_level);
   } else {
     // TODO(ajkr): `CompactionOptions` offers configurable `CompressionType`
     // without configurable `CompressionOptions`, which is inconsistent.
@@ -637,8 +637,7 @@ Compaction* CompactionPicker::CompactRange(
                             ioptions_.compaction_style),
         /* max_compaction_bytes */ LLONG_MAX,
         compact_range_options.target_path_id,
-        GetCompressionType(ioptions_, vstorage, mutable_cf_options,
-                           output_level, 1),
+        GetCompressionType(vstorage, mutable_cf_options, output_level, 1),
         GetCompressionOptions(mutable_cf_options, vstorage, output_level),
         Temperature::kUnknown, compact_range_options.max_subcompactions,
         /* grandparents */ {},
@@ -816,7 +815,7 @@ Compaction* CompactionPicker::CompactRange(
                           ioptions_.level_compaction_dynamic_level_bytes),
       mutable_cf_options.max_compaction_bytes,
       compact_range_options.target_path_id,
-      GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level,
+      GetCompressionType(vstorage, mutable_cf_options, output_level,
                          vstorage->base_level()),
       GetCompressionOptions(mutable_cf_options, vstorage, output_level),
       Temperature::kUnknown, compact_range_options.max_subcompactions,

db/compaction/compaction_picker.h

@@ -304,8 +304,7 @@ bool FindIntraL0Compaction(
     CompactionInputFiles* comp_inputs,
     SequenceNumber earliest_mem_seqno = kMaxSequenceNumber);
 
-CompressionType GetCompressionType(const ImmutableCFOptions& ioptions,
-                                   const VersionStorageInfo* vstorage,
+CompressionType GetCompressionType(const VersionStorageInfo* vstorage,
                                    const MutableCFOptions& mutable_cf_options,
                                    int level, int base_level,
                                    const bool enable_compression = true);

db/compaction/compaction_picker_level.cc

@@ -343,8 +343,8 @@ Compaction* LevelCompactionBuilder::GetCompaction() {
                           ioptions_.level_compaction_dynamic_level_bytes),
       mutable_cf_options_.max_compaction_bytes,
       GetPathId(ioptions_, mutable_cf_options_, output_level_),
-      GetCompressionType(ioptions_, vstorage_, mutable_cf_options_,
-                         output_level_, vstorage_->base_level()),
+      GetCompressionType(vstorage_, mutable_cf_options_, output_level_,
+                         vstorage_->base_level()),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level_),
       Temperature::kUnknown,
       /* max_subcompactions */ 0, std::move(grandparents_), is_manual_,

db/compaction/compaction_picker_universal.cc

@@ -746,19 +746,19 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(
   } else {
     compaction_reason = CompactionReason::kUniversalSortedRunNum;
   }
-  return new Compaction(
-      vstorage_, ioptions_, mutable_cf_options_, mutable_db_options_,
-      std::move(inputs), output_level,
-      MaxFileSizeForLevel(mutable_cf_options_, output_level,
-                          kCompactionStyleUniversal),
-      GetMaxOverlappingBytes(), path_id,
-      GetCompressionType(ioptions_, vstorage_, mutable_cf_options_, start_level,
-                         1, enable_compression),
-      GetCompressionOptions(mutable_cf_options_, vstorage_, start_level,
-                            enable_compression),
-      Temperature::kUnknown,
-      /* max_subcompactions */ 0, grandparents, /* is manual */ false, score_,
-      false /* deletion_compaction */, compaction_reason);
+  return new Compaction(vstorage_, ioptions_, mutable_cf_options_,
+                        mutable_db_options_, std::move(inputs), output_level,
+                        MaxFileSizeForLevel(mutable_cf_options_, output_level,
+                                            kCompactionStyleUniversal),
+                        GetMaxOverlappingBytes(), path_id,
+                        GetCompressionType(vstorage_, mutable_cf_options_,
+                                           start_level, 1, enable_compression),
+                        GetCompressionOptions(mutable_cf_options_, vstorage_,
+                                              start_level, enable_compression),
+                        Temperature::kUnknown,
+                        /* max_subcompactions */ 0, grandparents,
+                        /* is manual */ false, score_,
+                        false /* deletion_compaction */, compaction_reason);
 }
 
 // Look at overall size amplification. If size amplification
@@ -1076,8 +1076,8 @@ Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
       MaxFileSizeForLevel(mutable_cf_options_, output_level,
                           kCompactionStyleUniversal),
       GetMaxOverlappingBytes(), path_id,
-      GetCompressionType(ioptions_, vstorage_, mutable_cf_options_,
-                         output_level, 1, true /* enable_compression */),
+      GetCompressionType(vstorage_, mutable_cf_options_, output_level, 1,
+                         true /* enable_compression */),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level,
                             true /* enable_compression */),
       Temperature::kUnknown,
@@ -1220,8 +1220,7 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
       MaxFileSizeForLevel(mutable_cf_options_, output_level,
                           kCompactionStyleUniversal),
       /* max_grandparent_overlap_bytes */ GetMaxOverlappingBytes(), path_id,
-      GetCompressionType(ioptions_, vstorage_, mutable_cf_options_,
-                         output_level, 1),
+      GetCompressionType(vstorage_, mutable_cf_options_, output_level, 1),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level),
       Temperature::kUnknown,
       /* max_subcompactions */ 0, grandparents, /* is manual */ false, score_,
@@ -1294,8 +1293,8 @@ Compaction* UniversalCompactionBuilder::PickCompactionToOldest(
       MaxFileSizeForLevel(mutable_cf_options_, output_level,
                           kCompactionStyleUniversal),
       GetMaxOverlappingBytes(), path_id,
-      GetCompressionType(ioptions_, vstorage_, mutable_cf_options_,
-                         output_level, 1, true /* enable_compression */),
+      GetCompressionType(vstorage_, mutable_cf_options_, output_level, 1,
+                         true /* enable_compression */),
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level,
                             true /* enable_compression */),
       Temperature::kUnknown,

db/db_bloom_filter_test.cc

@@ -1540,6 +1540,93 @@ TEST_P(DBFilterConstructionCorruptionTestWithParam, DetectCorruption) {
       "TamperFilter");
 }
 
+// RocksDB lite does not support dynamic options
+#ifndef ROCKSDB_LITE
+TEST_P(DBFilterConstructionCorruptionTestWithParam,
+       DynamicallyTurnOnAndOffDetectConstructCorruption) {
+  Options options = CurrentOptions();
+  BlockBasedTableOptions table_options = GetBlockBasedTableOptions();
+  // We intend to turn on
+  // table_options.detect_filter_construct_corruption dynamically
+  // therefore we override this test parmater's value
+  table_options.detect_filter_construct_corruption = false;
+
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  options.create_if_missing = true;
+
+  int num_key = static_cast<int>(GetNumKey());
+  Status s;
+
+  DestroyAndReopen(options);
+
+  // Case 1: !table_options.detect_filter_construct_corruption
+  for (int i = 0; i < num_key; i++) {
+    ASSERT_OK(Put(Key(i), Key(i)));
+  }
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "XXPH3FilterBitsBuilder::Finish::TamperHashEntries", [&](void* arg) {
+        std::deque<uint64_t>* hash_entries_to_corrupt =
+            (std::deque<uint64_t>*)arg;
+        assert(!hash_entries_to_corrupt->empty());
+        *(hash_entries_to_corrupt->begin()) =
+            *(hash_entries_to_corrupt->begin()) ^ uint64_t { 1 };
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  s = Flush();
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearCallBack(
+      "XXPH3FilterBitsBuilder::Finish::"
+      "TamperHashEntries");
+
+  ASSERT_FALSE(table_options.detect_filter_construct_corruption);
+  EXPECT_TRUE(s.ok());
+
+  // Case 2: dynamically turn on
+  // table_options.detect_filter_construct_corruption
+  ASSERT_OK(db_->SetOptions({{"block_based_table_factory",
+                              "{detect_filter_construct_corruption=true;}"}}));
+
+  for (int i = 0; i < num_key; i++) {
+    ASSERT_OK(Put(Key(i), Key(i)));
+  }
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "XXPH3FilterBitsBuilder::Finish::TamperHashEntries", [&](void* arg) {
+        std::deque<uint64_t>* hash_entries_to_corrupt =
+            (std::deque<uint64_t>*)arg;
+        assert(!hash_entries_to_corrupt->empty());
+        *(hash_entries_to_corrupt->begin()) =
+            *(hash_entries_to_corrupt->begin()) ^ uint64_t { 1 };
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  s = Flush();
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearCallBack(
+      "XXPH3FilterBitsBuilder::Finish::"
+      "TamperHashEntries");
+
+  auto updated_table_options =
+      db_->GetOptions().table_factory->GetOptions<BlockBasedTableOptions>();
+  EXPECT_TRUE(updated_table_options->detect_filter_construct_corruption);
+  EXPECT_TRUE(s.IsCorruption());
+  EXPECT_TRUE(s.ToString().find("Filter's hash entries checksum mismatched") !=
+              std::string::npos);
+
+  // Case 3: dynamically turn off
+  // table_options.detect_filter_construct_corruption
+  ASSERT_OK(db_->SetOptions({{"block_based_table_factory",
+                              "{detect_filter_construct_corruption=false;}"}}));
+  updated_table_options =
+      db_->GetOptions().table_factory->GetOptions<BlockBasedTableOptions>();
+  EXPECT_FALSE(updated_table_options->detect_filter_construct_corruption);
+}
+#endif  // ROCKSDB_LITE
+
 namespace {
 // NOTE: This class is referenced by HISTORY.md as a model for a wrapper
 // FilterPolicy selecting among configurations based on context.
@@ -2163,7 +2250,6 @@ class BloomStatsTestWithParam
       options_.table_factory.reset(NewPlainTableFactory(table_options));
     } else {
       BlockBasedTableOptions table_options;
-      table_options.hash_index_allow_collision = false;
       if (partition_filters_) {
         assert(bfp_impl_ != kDeprecatedBlock);
         table_options.partition_filters = partition_filters_;

db/db_compaction_test.cc

@@ -6889,6 +6889,106 @@ TEST_F(DBCompactionTest, FIFOWarm) {
   Destroy(options);
 }
 
+TEST_F(DBCompactionTest, DisableMultiManualCompaction) {
+  const int kNumL0Files = 10;
+
+  Options options = CurrentOptions();
+  options.level0_file_num_compaction_trigger = kNumL0Files;
+  Reopen(options);
+
+  for (int i = 0; i < 10; i++) {
+    ASSERT_OK(Put(Key(i), "value"));
+    if (i % 2) {
+      ASSERT_OK(Flush());
+    }
+  }
+  MoveFilesToLevel(2);
+
+  for (int i = 0; i < 10; i++) {
+    ASSERT_OK(Put(Key(i), "value"));
+    if (i % 2) {
+      ASSERT_OK(Flush());
+    }
+  }
+  MoveFilesToLevel(1);
+
+  // Block compaction queue
+  test::SleepingBackgroundTask sleeping_task_low;
+  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
+                 Env::Priority::LOW);
+
+  port::Thread compact_thread1([&]() {
+    CompactRangeOptions cro;
+    cro.exclusive_manual_compaction = false;
+    std::string begin_str = Key(0);
+    std::string end_str = Key(3);
+    Slice b = begin_str;
+    Slice e = end_str;
+    auto s = db_->CompactRange(cro, &b, &e);
+    ASSERT_TRUE(s.IsIncomplete());
+  });
+
+  port::Thread compact_thread2([&]() {
+    CompactRangeOptions cro;
+    cro.exclusive_manual_compaction = false;
+    std::string begin_str = Key(4);
+    std::string end_str = Key(7);
+    Slice b = begin_str;
+    Slice e = end_str;
+    auto s = db_->CompactRange(cro, &b, &e);
+    ASSERT_TRUE(s.IsIncomplete());
+  });
+
+  db_->DisableManualCompaction();
+
+  compact_thread1.join();
+  compact_thread2.join();
+
+  sleeping_task_low.WakeUp();
+  sleeping_task_low.WaitUntilDone();
+  ASSERT_OK(dbfull()->TEST_WaitForCompact(true));
+}
+
+TEST_F(DBCompactionTest, DisableJustStartedManualCompaction) {
+  const int kNumL0Files = 4;
+
+  Options options = CurrentOptions();
+  options.level0_file_num_compaction_trigger = kNumL0Files;
+  Reopen(options);
+
+  // generate files, but avoid trigger auto compaction
+  for (int i = 0; i < kNumL0Files / 2; i++) {
+    ASSERT_OK(Put(Key(1), "value1"));
+    ASSERT_OK(Put(Key(2), "value2"));
+    ASSERT_OK(Flush());
+  }
+
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::BGWorkCompaction",
+        "DBCompactionTest::DisableJustStartedManualCompaction:"
+        "PreDisableManualCompaction"},
+       {"DBCompactionTest::DisableJustStartedManualCompaction:"
+        "ManualCompactionReturn",
+        "BackgroundCallCompaction:0"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  port::Thread compact_thread([&]() {
+    CompactRangeOptions cro;
+    cro.exclusive_manual_compaction = true;
+    auto s = db_->CompactRange(cro, nullptr, nullptr);
+    ASSERT_TRUE(s.IsIncomplete());
+    TEST_SYNC_POINT(
+        "DBCompactionTest::DisableJustStartedManualCompaction:"
+        "ManualCompactionReturn");
+  });
+  TEST_SYNC_POINT(
+      "DBCompactionTest::DisableJustStartedManualCompaction:"
+      "PreDisableManualCompaction");
+  db_->DisableManualCompaction();
+
+  compact_thread.join();
+}
+
 TEST_F(DBCompactionTest, DisableManualCompactionThreadQueueFull) {
   const int kNumL0Files = 4;
 
@@ -6951,7 +7051,7 @@ TEST_F(DBCompactionTest, DisableManualCompactionThreadQueueFullDBClose) {
 
   SyncPoint::GetInstance()->LoadDependency(
       {{"DBImpl::RunManualCompaction:Scheduled",
-        "DBCompactionTest::DisableManualCompactionThreadQueueFull:"
+        "DBCompactionTest::DisableManualCompactionThreadQueueFullDBClose:"
         "PreDisableManualCompaction"}});
   SyncPoint::GetInstance()->EnableProcessing();
 
@@ -6979,7 +7079,7 @@ TEST_F(DBCompactionTest, DisableManualCompactionThreadQueueFullDBClose) {
   });
 
   TEST_SYNC_POINT(
-      "DBCompactionTest::DisableManualCompactionThreadQueueFull:"
+      "DBCompactionTest::DisableManualCompactionThreadQueueFullDBClose:"
       "PreDisableManualCompaction");
 
   // Generate more files to trigger auto compaction which is scheduled after
@@ -7006,6 +7106,63 @@ TEST_F(DBCompactionTest, DisableManualCompactionThreadQueueFullDBClose) {
   sleeping_task_low.WaitUntilDone();
 }
 
+TEST_F(DBCompactionTest, DBCloseWithManualCompaction) {
+  const int kNumL0Files = 4;
+
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::RunManualCompaction:Scheduled",
+        "DBCompactionTest::DisableManualCompactionThreadQueueFullDBClose:"
+        "PreDisableManualCompaction"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  Options options = CurrentOptions();
+  options.level0_file_num_compaction_trigger = kNumL0Files;
+  Reopen(options);
+
+  // Block compaction queue
+  test::SleepingBackgroundTask sleeping_task_low;
+  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
+                 Env::Priority::LOW);
+
+  // generate files, but avoid trigger auto compaction
+  for (int i = 0; i < kNumL0Files / 2; i++) {
+    ASSERT_OK(Put(Key(1), "value1"));
+    ASSERT_OK(Put(Key(2), "value2"));
+    ASSERT_OK(Flush());
+  }
+
+  port::Thread compact_thread([&]() {
+    CompactRangeOptions cro;
+    cro.exclusive_manual_compaction = true;
+    auto s = db_->CompactRange(cro, nullptr, nullptr);
+    ASSERT_TRUE(s.IsIncomplete());
+  });
+
+  TEST_SYNC_POINT(
+      "DBCompactionTest::DisableManualCompactionThreadQueueFullDBClose:"
+      "PreDisableManualCompaction");
+
+  // Generate more files to trigger auto compaction which is scheduled after
+  // manual compaction. Has to generate 4 more files because existing files are
+  // pending compaction
+  for (int i = 0; i < kNumL0Files; i++) {
+    ASSERT_OK(Put(Key(1), "value1"));
+    ASSERT_OK(Put(Key(2), "value2"));
+    ASSERT_OK(Flush());
+  }
+  ASSERT_EQ(ToString(kNumL0Files + (kNumL0Files / 2)), FilesPerLevel(0));
+
+  // Close DB with manual compaction and auto triggered compaction in the queue.
+  auto s = db_->Close();
+  ASSERT_OK(s);
+
+  // manual compaction thread should return with Incomplete().
+  compact_thread.join();
+
+  sleeping_task_low.WakeUp();
+  sleeping_task_low.WaitUntilDone();
+}
+
 TEST_F(DBCompactionTest,
        DisableManualCompactionDoesNotWaitForDrainingAutomaticCompaction) {
   // When `CompactRangeOptions::exclusive_manual_compaction == true`, we wait

db/db_flush_test.cc

@@ -676,6 +676,7 @@ class TestFlushListener : public EventListener {
   ~TestFlushListener() override {
     prev_fc_info_.status.PermitUncheckedError();  // Ignore the status
   }
+
   void OnTableFileCreated(const TableFileCreationInfo& info) override {
     // remember the info for later checking the FlushJobInfo.
     prev_fc_info_ = info;
@@ -1574,6 +1575,9 @@ TEST_F(DBFlushTest, FireOnFlushCompletedAfterCommittedResult) {
   flush_opts.wait = false;
   ASSERT_OK(db_->Flush(flush_opts));
   t1.join();
+  // Ensure background work is fully finished including listener callbacks
+  // before accessing listener state.
+  ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
   ASSERT_TRUE(listener->completed1);
   ASSERT_TRUE(listener->completed2);
   SyncPoint::GetInstance()->DisableProcessing();
@@ -1999,6 +2003,61 @@ TEST_P(DBFlushTestBlobError, FlushError) {
 }
 
 #ifndef ROCKSDB_LITE
+TEST_F(DBFlushTest, TombstoneVisibleInSnapshot) {
+  class SimpleTestFlushListener : public EventListener {
+   public:
+    explicit SimpleTestFlushListener(DBFlushTest* _test) : test_(_test) {}
+    ~SimpleTestFlushListener() override {}
+
+    void OnFlushBegin(DB* db, const FlushJobInfo& info) override {
+      ASSERT_EQ(static_cast<uint32_t>(0), info.cf_id);
+
+      ASSERT_OK(db->Delete(WriteOptions(), "foo"));
+      snapshot_ = db->GetSnapshot();
+      ASSERT_OK(db->Put(WriteOptions(), "foo", "value"));
+
+      auto* dbimpl = static_cast_with_check<DBImpl>(db);
+      assert(dbimpl);
+
+      ColumnFamilyHandle* cfh = db->DefaultColumnFamily();
+      auto* cfhi = static_cast_with_check<ColumnFamilyHandleImpl>(cfh);
+      assert(cfhi);
+      ASSERT_OK(dbimpl->TEST_SwitchMemtable(cfhi->cfd()));
+    }
+
+    DBFlushTest* test_ = nullptr;
+    const Snapshot* snapshot_ = nullptr;
+  };
+
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  auto* listener = new SimpleTestFlushListener(this);
+  options.listeners.emplace_back(listener);
+  DestroyAndReopen(options);
+
+  ASSERT_OK(db_->Put(WriteOptions(), "foo", "value0"));
+
+  ManagedSnapshot snapshot_guard(db_);
+
+  ColumnFamilyHandle* default_cf = db_->DefaultColumnFamily();
+  ASSERT_OK(db_->Flush(FlushOptions(), default_cf));
+
+  const Snapshot* snapshot = listener->snapshot_;
+  assert(snapshot);
+
+  ReadOptions read_opts;
+  read_opts.snapshot = snapshot;
+
+  // Using snapshot should not see "foo".
+  {
+    std::string value;
+    Status s = db_->Get(read_opts, "foo", &value);
+    ASSERT_TRUE(s.IsNotFound());
+  }
+
+  db_->ReleaseSnapshot(snapshot);
+}
+
 TEST_P(DBAtomicFlushTest, ManualFlushUnder2PC) {
   Options options = CurrentOptions();
   options.create_if_missing = true;

db/db_impl/db_impl.cc

@@ -119,18 +119,16 @@ CompressionType GetCompressionFlush(
   // Compressing memtable flushes might not help unless the sequential load
   // optimization is used for leveled compaction. Otherwise the CPU and
   // latency overhead is not offset by saving much space.
-  if (ioptions.compaction_style == kCompactionStyleUniversal) {
-    if (mutable_cf_options.compaction_options_universal
-            .compression_size_percent < 0) {
-      return mutable_cf_options.compression;
-    } else {
-      return kNoCompression;
-    }
-  } else if (!ioptions.compression_per_level.empty()) {
-    // For leveled compress when min_level_to_compress != 0.
-    return ioptions.compression_per_level[0];
-  } else {
+  if (ioptions.compaction_style == kCompactionStyleUniversal &&
+      mutable_cf_options.compaction_options_universal
+              .compression_size_percent >= 0) {
+    return kNoCompression;
+  }
+  if (mutable_cf_options.compression_per_level.empty()) {
     return mutable_cf_options.compression;
+  } else {
+    // For leveled compress when min_level_to_compress != 0.
+    return mutable_cf_options.compression_per_level[0];
   }
 }
 
@@ -400,14 +398,6 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) {
 
   JobContext job_context(0);
   FindObsoleteFiles(&job_context, true);
-  if (s.ok()) {
-    s = error_handler_.ClearBGError();
-  } else {
-    // NOTE: this is needed to pass ASSERT_STATUS_CHECKED
-    // in the DBSSTTest.DBWithMaxSpaceAllowedRandomized test.
-    // See https://github.com/facebook/rocksdb/pull/7715#issuecomment-754947952
-    error_handler_.GetRecoveryError().PermitUncheckedError();
-  }
   mutex_.Unlock();
 
   job_context.manifest_file_number = 1;
@@ -428,11 +418,31 @@ Status DBImpl::ResumeImpl(DBRecoverContext context) {
             immutable_db_options_.info_log,
             "DB resume requested but could not enable file deletions [%s]",
             s.ToString().c_str());
+        assert(false);
       }
     }
-    ROCKS_LOG_INFO(immutable_db_options_.info_log, "Successfully resumed DB");
   }
+
   mutex_.Lock();
+  if (s.ok()) {
+    // This will notify and unblock threads waiting for error recovery to
+    // finish. Those previouly waiting threads can now proceed, which may
+    // include closing the db.
+    s = error_handler_.ClearBGError();
+  } else {
+    // NOTE: this is needed to pass ASSERT_STATUS_CHECKED
+    // in the DBSSTTest.DBWithMaxSpaceAllowedRandomized test.
+    // See https://github.com/facebook/rocksdb/pull/7715#issuecomment-754947952
+    error_handler_.GetRecoveryError().PermitUncheckedError();
+  }
+
+  if (s.ok()) {
+    ROCKS_LOG_INFO(immutable_db_options_.info_log, "Successfully resumed DB");
+  } else {
+    ROCKS_LOG_INFO(immutable_db_options_.info_log, "Failed to resume DB [%s]",
+                   s.ToString().c_str());
+  }
+
   // Check for shutdown again before scheduling further compactions,
   // since we released and re-acquired the lock above
   if (shutdown_initiated_) {
@@ -527,10 +537,19 @@ Status DBImpl::CloseHelper() {
   // marker. After this we do a variant of the waiting and unschedule work
   // (to consider: moving all the waiting into CancelAllBackgroundWork(true))
   CancelAllBackgroundWork(false);
+
+  // Cancel manual compaction if there's any
+  if (HasPendingManualCompaction()) {
+    DisableManualCompaction();
+  }
   mutex_.Lock();
-  env_->UnSchedule(this, Env::Priority::BOTTOM);
-  env_->UnSchedule(this, Env::Priority::LOW);
-  env_->UnSchedule(this, Env::Priority::HIGH);
+  // Unschedule all tasks for this DB
+  for (uint8_t i = 0; i < static_cast<uint8_t>(TaskType::kCount); i++) {
+    env_->UnSchedule(GetTaskTag(i), Env::Priority::BOTTOM);
+    env_->UnSchedule(GetTaskTag(i), Env::Priority::LOW);
+    env_->UnSchedule(GetTaskTag(i), Env::Priority::HIGH);
+  }
+
   Status ret = Status::OK();
 
   // Wait for background work to finish

db/db_impl/db_impl.h

@@ -1187,7 +1187,10 @@ class DBImpl : public DB {
   // WriteToWAL need different synchronization: log_empty_, alive_log_files_,
   // logs_, logfile_number_. Refer to the definition of each variable below for
   // more description.
-  mutable InstrumentedMutex mutex_;
+  //
+  // `mutex_` can be a hot lock in some workloads, so it deserves dedicated
+  // cachelines.
+  mutable CacheAlignedInstrumentedMutex mutex_;
 
   ColumnFamilyHandleImpl* default_cf_handle_;
   InternalStats* default_cf_internal_stats_;
@@ -1536,7 +1539,6 @@ class DBImpl : public DB {
     ManualCompactionState* manual_compaction_state;  // nullptr if non-manual
     // task limiter token is requested during compaction picking.
     std::unique_ptr<TaskLimiterToken> task_token;
-    bool is_canceled = false;
   };
 
   struct CompactionArg {
@@ -1731,6 +1733,25 @@ class DBImpl : public DB {
     }
   }
 
+  // TaskType is used to identify tasks in thread-pool, currently only
+  // differentiate manual compaction, which could be unscheduled from the
+  // thread-pool.
+  enum class TaskType : uint8_t {
+    kDefault = 0,
+    kManualCompaction = 1,
+    kCount = 2,
+  };
+
+  // Task tag is used to identity tasks in thread-pool, which is
+  // dbImpl obj address + type
+  inline void* GetTaskTag(TaskType type) {
+    return GetTaskTag(static_cast<uint8_t>(type));
+  }
+
+  inline void* GetTaskTag(uint8_t type) {
+    return static_cast<uint8_t*>(static_cast<void*>(this)) + type;
+  }
+
   // REQUIRES: mutex locked and in write thread.
   void AssignAtomicFlushSeq(const autovector<ColumnFamilyData*>& cfds);
 
@@ -1748,8 +1769,12 @@ class DBImpl : public DB {
                          WriteBatch* tmp_batch, size_t* write_with_wal,
                          WriteBatch** to_be_cached_state);
 
+  // rate_limiter_priority is used to charge `DBOptions::rate_limiter`
+  // for automatic WAL flush (`Options::manual_wal_flush` == false)
+  // associated with this WriteToWAL
   IOStatus WriteToWAL(const WriteBatch& merged_batch, log::Writer* log_writer,
-                      uint64_t* log_used, uint64_t* log_size);
+                      uint64_t* log_used, uint64_t* log_size,
+                      Env::IOPriority rate_limiter_priority);
 
   IOStatus WriteToWAL(const WriteThread::WriteGroup& write_group,
                       log::Writer* log_writer, uint64_t* log_used,

db/db_impl/db_impl_compaction_flush.cc

@@ -170,6 +170,7 @@ Status DBImpl::FlushMemTableToOutputFile(
   const bool needs_to_sync_closed_wals =
       logfile_number_ > 0 &&
       versions_->GetColumnFamilySet()->NumberOfColumnFamilies() > 1;
+
   // If needs_to_sync_closed_wals is true, we need to record the current
   // maximum memtable ID of this column family so that a later PickMemtables()
   // call will not pick memtables whose IDs are higher. This is due to the fact
@@ -177,9 +178,33 @@ Status DBImpl::FlushMemTableToOutputFile(
   // happen for this column family in the meantime. The newly created memtables
   // have their data backed by unsynced WALs, thus they cannot be included in
   // this flush job.
+  // Another reason why we must record the current maximum memtable ID of this
+  // column family: SyncClosedLogs() may release db mutex, thus it's possible
+  // for application to continue to insert into memtables increasing db's
+  // sequence number. The application may take a snapshot, but this snapshot is
+  // not included in `snapshot_seqs` which will be passed to flush job because
+  // `snapshot_seqs` has already been computed before this function starts.
+  // Recording the max memtable ID ensures that the flush job does not flush
+  // a memtable without knowing such snapshot(s).
   uint64_t max_memtable_id = needs_to_sync_closed_wals
                                  ? cfd->imm()->GetLatestMemTableID()
                                  : port::kMaxUint64;
+
+  // If needs_to_sync_closed_wals is false, then the flush job will pick ALL
+  // existing memtables of the column family when PickMemTable() is called
+  // later. Although we won't call SyncClosedLogs() in this case, we may still
+  // call the callbacks of the listeners, i.e. NotifyOnFlushBegin() which also
+  // releases and re-acquires the db mutex. In the meantime, the application
+  // can still insert into the memtables and increase the db's sequence number.
+  // The application can take a snapshot, hoping that the latest visible state
+  // to this snapshto is preserved. This is hard to guarantee since db mutex
+  // not held. This newly-created snapshot is not included in `snapshot_seqs`
+  // and the flush job is unaware of its presence. Consequently, the flush job
+  // may drop certain keys when generating the L0, causing incorrect data to be
+  // returned for snapshot read using this snapshot.
+  // To address this, we make sure NotifyOnFlushBegin() executes after memtable
+  // picking so that no new snapshot can be taken between the two functions.
+
   FlushJob flush_job(
       dbname_, cfd, immutable_db_options_, mutable_cf_options, max_memtable_id,
       file_options_for_compaction_, versions_.get(), &mutex_, &shutting_down_,
@@ -192,11 +217,6 @@ Status DBImpl::FlushMemTableToOutputFile(
       &blob_callback_);
   FileMetaData file_meta;
 
-#ifndef ROCKSDB_LITE
-  // may temporarily unlock and lock the mutex.
-  NotifyOnFlushBegin(cfd, &file_meta, mutable_cf_options, job_context->job_id);
-#endif  // ROCKSDB_LITE
-
   Status s;
   bool need_cancel = false;
   IOStatus log_io_s = IOStatus::OK();
@@ -221,6 +241,12 @@ Status DBImpl::FlushMemTableToOutputFile(
   }
   TEST_SYNC_POINT_CALLBACK(
       "DBImpl::FlushMemTableToOutputFile:AfterPickMemtables", &flush_job);
+
+#ifndef ROCKSDB_LITE
+  // may temporarily unlock and lock the mutex.
+  NotifyOnFlushBegin(cfd, &file_meta, mutable_cf_options, job_context->job_id);
+#endif  // ROCKSDB_LITE
+
   bool switched_to_mempurge = false;
   // Within flush_job.Run, rocksdb may call event listener to notify
   // file creation and deletion.
@@ -356,13 +382,14 @@ Status DBImpl::FlushMemTablesToOutputFiles(
                      &earliest_write_conflict_snapshot, &snapshot_checker);
   const auto& bg_flush_arg = bg_flush_args[0];
   ColumnFamilyData* cfd = bg_flush_arg.cfd_;
-  MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
+  // intentional infrequent copy for each flush
+  MutableCFOptions mutable_cf_options_copy = *cfd->GetLatestMutableCFOptions();
   SuperVersionContext* superversion_context =
       bg_flush_arg.superversion_context_;
   Status s = FlushMemTableToOutputFile(
-      cfd, mutable_cf_options, made_progress, job_context, superversion_context,
-      snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker,
-      log_buffer, thread_pri);
+      cfd, mutable_cf_options_copy, made_progress, job_context,
+      superversion_context, snapshot_seqs, earliest_write_conflict_snapshot,
+      snapshot_checker, log_buffer, thread_pri);
   return s;
 }
 
@@ -1763,6 +1790,7 @@ Status DBImpl::RunManualCompaction(
   CompactionArg* ca = nullptr;
 
   bool scheduled = false;
+  Env::Priority thread_pool_priority = Env::Priority::TOTAL;
   bool manual_conflict = false;
   ManualCompactionState manual;
   manual.cfd = cfd;
@@ -1884,9 +1912,9 @@ Status DBImpl::RunManualCompaction(
         manual.done = true;
         manual.status =
             Status::Incomplete(Status::SubCode::kManualCompactionPaused);
-        if (ca && ca->prepicked_compaction) {
-          ca->prepicked_compaction->is_canceled = true;
-        }
+        assert(thread_pool_priority != Env::Priority::TOTAL);
+        env_->UnSchedule(GetTaskTag(TaskType::kManualCompaction),
+                         thread_pool_priority);
         break;
       }
       if (scheduled && manual.incomplete == true) {
@@ -1916,13 +1944,17 @@ Status DBImpl::RunManualCompaction(
         bg_bottom_compaction_scheduled_++;
         ca->compaction_pri_ = Env::Priority::BOTTOM;
         env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca,
-                       Env::Priority::BOTTOM, this,
+                       Env::Priority::BOTTOM,
+                       GetTaskTag(TaskType::kManualCompaction),
                        &DBImpl::UnscheduleCompactionCallback);
+        thread_pool_priority = Env::Priority::BOTTOM;
       } else {
         bg_compaction_scheduled_++;
         ca->compaction_pri_ = Env::Priority::LOW;
-        env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
+        env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW,
+                       GetTaskTag(TaskType::kManualCompaction),
                        &DBImpl::UnscheduleCompactionCallback);
+        thread_pool_priority = Env::Priority::LOW;
       }
       scheduled = true;
       TEST_SYNC_POINT("DBImpl::RunManualCompaction:Scheduled");
@@ -1933,6 +1965,13 @@ Status DBImpl::RunManualCompaction(
   assert(!manual.in_progress);
   assert(HasPendingManualCompaction());
   RemoveManualCompaction(&manual);
+  // if the manual job is unscheduled, try schedule other jobs in case there's
+  // any unscheduled compaction job which was blocked by exclusive manual
+  // compaction.
+  if (manual.status.IsIncomplete() &&
+      manual.status.subcode() == Status::SubCode::kManualCompactionPaused) {
+    MaybeScheduleFlushOrCompaction();
+  }
   bg_cv_.SignalAll();
   return manual.status;
 }
@@ -2661,6 +2700,8 @@ void DBImpl::UnscheduleCompactionCallback(void* arg) {
   delete reinterpret_cast<CompactionArg*>(arg);
   if (ca.prepicked_compaction != nullptr) {
     if (ca.prepicked_compaction->compaction != nullptr) {
+      ca.prepicked_compaction->compaction->ReleaseCompactionFiles(
+          Status::Incomplete(Status::SubCode::kManualCompactionPaused));
       delete ca.prepicked_compaction->compaction;
     }
     delete ca.prepicked_compaction;
@@ -2851,106 +2892,93 @@ void DBImpl::BackgroundCallFlush(Env::Priority thread_pri) {
 void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
                                       Env::Priority bg_thread_pri) {
   bool made_progress = false;
+  JobContext job_context(next_job_id_.fetch_add(1), true);
   TEST_SYNC_POINT("BackgroundCallCompaction:0");
   LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL,
                        immutable_db_options_.info_log.get());
   {
     InstrumentedMutexLock l(&mutex_);
 
-    if (prepicked_compaction && prepicked_compaction->is_canceled) {
-      assert(prepicked_compaction->compaction);
-      ROCKS_LOG_BUFFER(&log_buffer, "[%s] Skip canceled manual compaction job",
-                       prepicked_compaction->compaction->column_family_data()
-                           ->GetName()
-                           .c_str());
-      prepicked_compaction->compaction->ReleaseCompactionFiles(
-          Status::Incomplete(Status::SubCode::kManualCompactionPaused));
-      delete prepicked_compaction->compaction;
-    } else {
-      JobContext job_context(next_job_id_.fetch_add(1), true);
-      // This call will unlock/lock the mutex to wait for current running
-      // IngestExternalFile() calls to finish.
-      WaitForIngestFile();
+    // This call will unlock/lock the mutex to wait for current running
+    // IngestExternalFile() calls to finish.
+    WaitForIngestFile();
 
-      num_running_compactions_++;
+    num_running_compactions_++;
 
-      std::unique_ptr<std::list<uint64_t>::iterator>
-          pending_outputs_inserted_elem(new std::list<uint64_t>::iterator(
-              CaptureCurrentFileNumberInPendingOutputs()));
+    std::unique_ptr<std::list<uint64_t>::iterator>
+        pending_outputs_inserted_elem(new std::list<uint64_t>::iterator(
+            CaptureCurrentFileNumberInPendingOutputs()));
 
-      assert((bg_thread_pri == Env::Priority::BOTTOM &&
-              bg_bottom_compaction_scheduled_) ||
-             (bg_thread_pri == Env::Priority::LOW && bg_compaction_scheduled_));
-      Status s = BackgroundCompaction(&made_progress, &job_context, &log_buffer,
-                                      prepicked_compaction, bg_thread_pri);
-      TEST_SYNC_POINT("BackgroundCallCompaction:1");
-      if (s.IsBusy()) {
-        bg_cv_.SignalAll();  // In case a waiter can proceed despite the error
-        mutex_.Unlock();
-        immutable_db_options_.clock->SleepForMicroseconds(
-            10000);  // prevent hot loop
-        mutex_.Lock();
-      } else if (!s.ok() && !s.IsShutdownInProgress() &&
-                 !s.IsManualCompactionPaused() && !s.IsColumnFamilyDropped()) {
-        // Wait a little bit before retrying background compaction in
-        // case this is an environmental problem and we do not want to
-        // chew up resources for failed compactions for the duration of
-        // the problem.
-        uint64_t error_cnt =
-            default_cf_internal_stats_->BumpAndGetBackgroundErrorCount();
-        bg_cv_.SignalAll();  // In case a waiter can proceed despite the error
-        mutex_.Unlock();
-        log_buffer.FlushBufferToLog();
-        ROCKS_LOG_ERROR(immutable_db_options_.info_log,
-                        "Waiting after background compaction error: %s, "
-                        "Accumulated background error counts: %" PRIu64,
-                        s.ToString().c_str(), error_cnt);
-        LogFlush(immutable_db_options_.info_log);
-        immutable_db_options_.clock->SleepForMicroseconds(1000000);
-        mutex_.Lock();
-      } else if (s.IsManualCompactionPaused()) {
-        assert(prepicked_compaction);
-        ManualCompactionState* m =
-            prepicked_compaction->manual_compaction_state;
-        assert(m);
-        ROCKS_LOG_BUFFER(&log_buffer, "[%s] [JOB %d] Manual compaction paused",
-                         m->cfd->GetName().c_str(), job_context.job_id);
-      }
-
-      ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
-
-      // If compaction failed, we want to delete all temporary files that we
-      // might have created (they might not be all recorded in job_context in
-      // case of a failure). Thus, we force full scan in FindObsoleteFiles()
-      FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress() &&
-                                          !s.IsManualCompactionPaused() &&
-                                          !s.IsColumnFamilyDropped() &&
-                                          !s.IsBusy());
-      TEST_SYNC_POINT("DBImpl::BackgroundCallCompaction:FoundObsoleteFiles");
-
-      // delete unnecessary files if any, this is done outside the mutex
-      if (job_context.HaveSomethingToClean() ||
-          job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) {
-        mutex_.Unlock();
-        // Have to flush the info logs before bg_compaction_scheduled_--
-        // because if bg_flush_scheduled_ becomes 0 and the lock is
-        // released, the deconstructor of DB can kick in and destroy all the
-        // states of DB so info_log might not be available after that point.
-        // It also applies to access other states that DB owns.
-        log_buffer.FlushBufferToLog();
-        if (job_context.HaveSomethingToDelete()) {
-          PurgeObsoleteFiles(job_context);
-          TEST_SYNC_POINT(
-              "DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles");
-        }
-        job_context.Clean();
-        mutex_.Lock();
-      }
-
-      assert(num_running_compactions_ > 0);
-      num_running_compactions_--;
+    assert((bg_thread_pri == Env::Priority::BOTTOM &&
+            bg_bottom_compaction_scheduled_) ||
+           (bg_thread_pri == Env::Priority::LOW && bg_compaction_scheduled_));
+    Status s = BackgroundCompaction(&made_progress, &job_context, &log_buffer,
+                                    prepicked_compaction, bg_thread_pri);
+    TEST_SYNC_POINT("BackgroundCallCompaction:1");
+    if (s.IsBusy()) {
+      bg_cv_.SignalAll();  // In case a waiter can proceed despite the error
+      mutex_.Unlock();
+      immutable_db_options_.clock->SleepForMicroseconds(
+          10000);  // prevent hot loop
+      mutex_.Lock();
+    } else if (!s.ok() && !s.IsShutdownInProgress() &&
+               !s.IsManualCompactionPaused() && !s.IsColumnFamilyDropped()) {
+      // Wait a little bit before retrying background compaction in
+      // case this is an environmental problem and we do not want to
+      // chew up resources for failed compactions for the duration of
+      // the problem.
+      uint64_t error_cnt =
+          default_cf_internal_stats_->BumpAndGetBackgroundErrorCount();
+      bg_cv_.SignalAll();  // In case a waiter can proceed despite the error
+      mutex_.Unlock();
+      log_buffer.FlushBufferToLog();
+      ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+                      "Waiting after background compaction error: %s, "
+                      "Accumulated background error counts: %" PRIu64,
+                      s.ToString().c_str(), error_cnt);
+      LogFlush(immutable_db_options_.info_log);
+      immutable_db_options_.clock->SleepForMicroseconds(1000000);
+      mutex_.Lock();
+    } else if (s.IsManualCompactionPaused()) {
+      assert(prepicked_compaction);
+      ManualCompactionState* m = prepicked_compaction->manual_compaction_state;
+      assert(m);
+      ROCKS_LOG_BUFFER(&log_buffer, "[%s] [JOB %d] Manual compaction paused",
+                       m->cfd->GetName().c_str(), job_context.job_id);
     }
 
+    ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
+
+    // If compaction failed, we want to delete all temporary files that we
+    // might have created (they might not be all recorded in job_context in
+    // case of a failure). Thus, we force full scan in FindObsoleteFiles()
+    FindObsoleteFiles(&job_context, !s.ok() && !s.IsShutdownInProgress() &&
+                                        !s.IsManualCompactionPaused() &&
+                                        !s.IsColumnFamilyDropped() &&
+                                        !s.IsBusy());
+    TEST_SYNC_POINT("DBImpl::BackgroundCallCompaction:FoundObsoleteFiles");
+
+    // delete unnecessary files if any, this is done outside the mutex
+    if (job_context.HaveSomethingToClean() ||
+        job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) {
+      mutex_.Unlock();
+      // Have to flush the info logs before bg_compaction_scheduled_--
+      // because if bg_flush_scheduled_ becomes 0 and the lock is
+      // released, the deconstructor of DB can kick in and destroy all the
+      // states of DB so info_log might not be available after that point.
+      // It also applies to access other states that DB owns.
+      log_buffer.FlushBufferToLog();
+      if (job_context.HaveSomethingToDelete()) {
+        PurgeObsoleteFiles(job_context);
+        TEST_SYNC_POINT("DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles");
+      }
+      job_context.Clean();
+      mutex_.Lock();
+    }
+
+    assert(num_running_compactions_ > 0);
+    num_running_compactions_--;
+
     if (bg_thread_pri == Env::Priority::LOW) {
       bg_compaction_scheduled_--;
     } else {

db/db_impl/db_impl_open.cc

@@ -1729,7 +1729,8 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
         WriteOptions write_options;
         uint64_t log_used, log_size;
         log::Writer* log_writer = impl->logs_.back().writer;
-        s = impl->WriteToWAL(empty_batch, log_writer, &log_used, &log_size);
+        s = impl->WriteToWAL(empty_batch, log_writer, &log_used, &log_size,
+                             Env::IO_TOTAL);
         if (s.ok()) {
           // Need to fsync, otherwise it might get lost after a power reset.
           s = impl->FlushWAL(false);

db/db_impl/db_impl_write.cc

@@ -130,8 +130,31 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   assert(!seq_per_batch_ || batch_cnt != 0);
   if (my_batch == nullptr) {
     return Status::Corruption("Batch is nullptr!");
-  } else if (WriteBatchInternal::TimestampsUpdateNeeded(*my_batch)) {
+  } else if (!disable_memtable &&
+             WriteBatchInternal::TimestampsUpdateNeeded(*my_batch)) {
+    // If writing to memtable, then we require the caller to set/update the
+    // timestamps for the keys in the write batch.
+    // Otherwise, it means we are just writing to the WAL, and we allow
+    // timestamps unset for the keys in the write batch. This can happen if we
+    // use TransactionDB with write-committed policy, and we currently do not
+    // support user-defined timestamp with other policies.
+    // In the prepare phase, a transaction can write the batch to the WAL
+    // without inserting to memtable. The keys in the batch do not have to be
+    // assigned timestamps because they will be used only during recovery if
+    // there is a commit marker which includes their commit timestamp.
     return Status::InvalidArgument("write batch must have timestamp(s) set");
+  } else if (write_options.rate_limiter_priority != Env::IO_TOTAL &&
+             write_options.rate_limiter_priority != Env::IO_USER) {
+    return Status::InvalidArgument(
+        "WriteOptions::rate_limiter_priority only allows "
+        "Env::IO_TOTAL and Env::IO_USER due to implementation constraints");
+  } else if (write_options.rate_limiter_priority != Env::IO_TOTAL &&
+             (write_options.disableWAL || manual_wal_flush_)) {
+    return Status::InvalidArgument(
+        "WriteOptions::rate_limiter_priority currently only supports "
+        "rate-limiting automatic WAL flush, which requires "
+        "`WriteOptions::disableWAL` and "
+        "`DBOptions::manual_wal_flush` both set to false");
   }
   // TODO: this use of operator bool on `tracer_` can avoid unnecessary lock
   // grabs but does not seem thread-safe.
@@ -1147,7 +1170,8 @@ WriteBatch* DBImpl::MergeBatch(const WriteThread::WriteGroup& write_group,
 // write thread. Otherwise this must be called holding log_write_mutex_.
 IOStatus DBImpl::WriteToWAL(const WriteBatch& merged_batch,
                             log::Writer* log_writer, uint64_t* log_used,
-                            uint64_t* log_size) {
+                            uint64_t* log_size,
+                            Env::IOPriority rate_limiter_priority) {
   assert(log_size != nullptr);
   Slice log_entry = WriteBatchInternal::Contents(&merged_batch);
   *log_size = log_entry.size();
@@ -1162,7 +1186,7 @@ IOStatus DBImpl::WriteToWAL(const WriteBatch& merged_batch,
   if (UNLIKELY(needs_locking)) {
     log_write_mutex_.Lock();
   }
-  IOStatus io_s = log_writer->AddRecord(log_entry);
+  IOStatus io_s = log_writer->AddRecord(log_entry, rate_limiter_priority);
 
   if (UNLIKELY(needs_locking)) {
     log_write_mutex_.Unlock();
@@ -1200,7 +1224,8 @@ IOStatus DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
   WriteBatchInternal::SetSequence(merged_batch, sequence);
 
   uint64_t log_size;
-  io_s = WriteToWAL(*merged_batch, log_writer, log_used, &log_size);
+  io_s = WriteToWAL(*merged_batch, log_writer, log_used, &log_size,
+                    write_group.leader->rate_limiter_priority);
   if (to_be_cached_state) {
     cached_recoverable_state_ = *to_be_cached_state;
     cached_recoverable_state_empty_ = false;
@@ -1294,7 +1319,8 @@ IOStatus DBImpl::ConcurrentWriteToWAL(
 
   log::Writer* log_writer = logs_.back().writer;
   uint64_t log_size;
-  io_s = WriteToWAL(*merged_batch, log_writer, log_used, &log_size);
+  io_s = WriteToWAL(*merged_batch, log_writer, log_used, &log_size,
+                    write_group.leader->rate_limiter_priority);
   if (to_be_cached_state) {
     cached_recoverable_state_ = *to_be_cached_state;
     cached_recoverable_state_empty_ = false;

db/db_logical_block_size_cache_test.cc

@@ -167,9 +167,14 @@ TEST_F(DBLogicalBlockSizeCacheTest, CreateColumnFamily) {
 }
 
 TEST_F(DBLogicalBlockSizeCacheTest, CreateColumnFamilies) {
-  // Tests that CreateColumnFamilies will cache the cf_paths,
-  // drop the column family handle won't drop the cache,
-  // drop and then delete the column family handle will drop the cache.
+  // To test:
+  // (1) CreateColumnFamilies will cache the cf_paths in
+  // DBLogicalBlockSizeCache
+  // (2) Dropping column family handles associated with
+  // that cf_paths won't drop the cached cf_paths
+  // (3) Deleting all the column family handles associated
+  //  with that cf_paths will drop the cached cf_paths
+
   Options options;
   options.create_if_missing = true;
   options.env = env_.get();
@@ -190,7 +195,7 @@ TEST_F(DBLogicalBlockSizeCacheTest, CreateColumnFamilies) {
   ASSERT_TRUE(cache_->Contains(cf_path_0_));
   ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_));
 
-  // Drop column family does not drop cache.
+  // Drop column family does not drop cf_path_0_'s entry from cache
   for (ColumnFamilyHandle* cf : cfs) {
     ASSERT_OK(db->DropColumnFamily(cf));
     ASSERT_EQ(2, cache_->Size());
@@ -200,22 +205,24 @@ TEST_F(DBLogicalBlockSizeCacheTest, CreateColumnFamilies) {
     ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_));
   }
 
-  // Delete one handle will not drop cache because another handle is still
-  // referencing cf_path_0_.
+  // Delete one cf handle will not drop cf_path_0_'s entry from cache because
+  // another handle is still referencing cf_path_0_.
   ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[0]));
   ASSERT_EQ(2, cache_->Size());
   ASSERT_TRUE(cache_->Contains(dbname_));
   ASSERT_EQ(1, cache_->GetRefCount(dbname_));
   ASSERT_TRUE(cache_->Contains(cf_path_0_));
-  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
 
-  // Delete the last handle will drop cache.
+  // Delete all cf handles and ensure the ref count of cf_path_0_ in cache_
+  // can be properly decreased by releasing any background reference to the
+  // ColumnFamilyData during db deletion
   ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[1]));
-  ASSERT_EQ(1, cache_->Size());
   ASSERT_TRUE(cache_->Contains(dbname_));
   ASSERT_EQ(1, cache_->GetRefCount(dbname_));
-
   delete db;
+
+  // Now cf_path_0_ in cache_ has been properly decreased and cf_path_0_'s entry
+  // is dropped from cache
   ASSERT_EQ(0, cache_->Size());
   ASSERT_OK(DestroyDB(dbname_, options,
       {{"cf1", cf_options}, {"cf2", cf_options}}));

db/db_properties_test.cc

@@ -1067,11 +1067,17 @@ TEST_F(DBPropertiesTest, EstimateCompressionRatio) {
   const int kNumEntriesPerFile = 1000;
 
   Options options = CurrentOptions();
-  options.compression_per_level = {kNoCompression, kSnappyCompression};
   options.disable_auto_compactions = true;
-  options.num_levels = 2;
+  options.num_levels = 3;
   Reopen(options);
 
+  ASSERT_OK(db_->SetOptions(
+      {{"compression_per_level", "kNoCompression:kSnappyCompression"}}));
+  auto opts = db_->GetOptions();
+  ASSERT_EQ(opts.compression_per_level.size(), 2);
+  ASSERT_EQ(opts.compression_per_level[0], kNoCompression);
+  ASSERT_EQ(opts.compression_per_level[1], kSnappyCompression);
+
   // compression ratio is -1.0 when no open files at level
   ASSERT_EQ(CompressionRatioAtLevel(0), -1.0);
 

db/db_rate_limiter_test.cc

@@ -3,18 +3,26 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <string>
+
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "test_util/testharness.h"
 #include "util/file_checksum_helper.h"
 
 namespace ROCKSDB_NAMESPACE {
 
-class DBRateLimiterTest
+class DBRateLimiterOnReadTest
     : public DBTestBase,
       public ::testing::WithParamInterface<std::tuple<bool, bool, bool>> {
  public:
-  DBRateLimiterTest()
-      : DBTestBase("db_rate_limiter_test", /*env_do_fsync=*/false),
+  explicit DBRateLimiterOnReadTest()
+      : DBTestBase("db_rate_limiter_on_read_test", /*env_do_fsync=*/false),
         use_direct_io_(std::get<0>(GetParam())),
         use_block_cache_(std::get<1>(GetParam())),
         use_readahead_(std::get<2>(GetParam())) {}
@@ -89,20 +97,20 @@ std::string GetTestNameSuffix(
 }
 
 #ifndef ROCKSDB_LITE
-INSTANTIATE_TEST_CASE_P(DBRateLimiterTest, DBRateLimiterTest,
+INSTANTIATE_TEST_CASE_P(DBRateLimiterOnReadTest, DBRateLimiterOnReadTest,
                         ::testing::Combine(::testing::Bool(), ::testing::Bool(),
                                            ::testing::Bool()),
                         GetTestNameSuffix);
 #else   // ROCKSDB_LITE
 // Cannot use direct I/O in lite mode.
-INSTANTIATE_TEST_CASE_P(DBRateLimiterTest, DBRateLimiterTest,
+INSTANTIATE_TEST_CASE_P(DBRateLimiterOnReadTest, DBRateLimiterOnReadTest,
                         ::testing::Combine(::testing::Values(false),
                                            ::testing::Bool(),
                                            ::testing::Bool()),
                         GetTestNameSuffix);
 #endif  // ROCKSDB_LITE
 
-TEST_P(DBRateLimiterTest, Get) {
+TEST_P(DBRateLimiterOnReadTest, Get) {
   if (use_direct_io_ && !IsDirectIOSupported()) {
     return;
   }
@@ -130,7 +138,7 @@ TEST_P(DBRateLimiterTest, Get) {
   }
 }
 
-TEST_P(DBRateLimiterTest, NewMultiGet) {
+TEST_P(DBRateLimiterOnReadTest, NewMultiGet) {
   // The new void-returning `MultiGet()` APIs use `MultiRead()`, which does not
   // yet support rate limiting.
   if (use_direct_io_ && !IsDirectIOSupported()) {
@@ -161,7 +169,7 @@ TEST_P(DBRateLimiterTest, NewMultiGet) {
   ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_USER));
 }
 
-TEST_P(DBRateLimiterTest, OldMultiGet) {
+TEST_P(DBRateLimiterOnReadTest, OldMultiGet) {
   // The old `vector<Status>`-returning `MultiGet()` APIs use `Read()`, which
   // supports rate limiting.
   if (use_direct_io_ && !IsDirectIOSupported()) {
@@ -193,7 +201,7 @@ TEST_P(DBRateLimiterTest, OldMultiGet) {
   ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER));
 }
 
-TEST_P(DBRateLimiterTest, Iterator) {
+TEST_P(DBRateLimiterOnReadTest, Iterator) {
   if (use_direct_io_ && !IsDirectIOSupported()) {
     return;
   }
@@ -223,7 +231,7 @@ TEST_P(DBRateLimiterTest, Iterator) {
 
 #if !defined(ROCKSDB_LITE)
 
-TEST_P(DBRateLimiterTest, VerifyChecksum) {
+TEST_P(DBRateLimiterOnReadTest, VerifyChecksum) {
   if (use_direct_io_ && !IsDirectIOSupported()) {
     return;
   }
@@ -237,7 +245,7 @@ TEST_P(DBRateLimiterTest, VerifyChecksum) {
   ASSERT_EQ(expected, options_.rate_limiter->GetTotalRequests(Env::IO_USER));
 }
 
-TEST_P(DBRateLimiterTest, VerifyFileChecksums) {
+TEST_P(DBRateLimiterOnReadTest, VerifyFileChecksums) {
   if (use_direct_io_ && !IsDirectIOSupported()) {
     return;
   }
@@ -253,6 +261,182 @@ TEST_P(DBRateLimiterTest, VerifyFileChecksums) {
 
 #endif  // !defined(ROCKSDB_LITE)
 
+class DBRateLimiterOnWriteTest : public DBTestBase {
+ public:
+  explicit DBRateLimiterOnWriteTest()
+      : DBTestBase("db_rate_limiter_on_write_test", /*env_do_fsync=*/false) {}
+
+  void Init() {
+    options_ = GetOptions();
+    ASSERT_OK(TryReopenWithColumnFamilies({"default"}, options_));
+    Random rnd(301);
+    for (int i = 0; i < kNumFiles; i++) {
+      ASSERT_OK(Put(0, kStartKey, rnd.RandomString(2)));
+      ASSERT_OK(Put(0, kEndKey, rnd.RandomString(2)));
+      ASSERT_OK(Flush(0));
+    }
+  }
+
+  Options GetOptions() {
+    Options options = CurrentOptions();
+    options.disable_auto_compactions = true;
+    options.rate_limiter.reset(NewGenericRateLimiter(
+        1 << 20 /* rate_bytes_per_sec */, 100 * 1000 /* refill_period_us */,
+        10 /* fairness */, RateLimiter::Mode::kWritesOnly));
+    options.table_factory.reset(
+        NewBlockBasedTableFactory(BlockBasedTableOptions()));
+    return options;
+  }
+
+ protected:
+  inline const static int64_t kNumFiles = 3;
+  inline const static std::string kStartKey = "a";
+  inline const static std::string kEndKey = "b";
+  Options options_;
+};
+
+TEST_F(DBRateLimiterOnWriteTest, Flush) {
+  std::int64_t prev_total_request = 0;
+
+  Init();
+
+  std::int64_t actual_flush_request =
+      options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL) -
+      prev_total_request;
+  std::int64_t exepcted_flush_request = kNumFiles;
+  EXPECT_EQ(actual_flush_request, exepcted_flush_request);
+  EXPECT_EQ(actual_flush_request,
+            options_.rate_limiter->GetTotalRequests(Env::IO_HIGH));
+}
+
+TEST_F(DBRateLimiterOnWriteTest, Compact) {
+  Init();
+
+  // Pre-comaction:
+  // level-0 : `kNumFiles` SST files overlapping on [kStartKey, kEndKey]
+#ifndef ROCKSDB_LITE
+  std::string files_per_level_pre_compaction = std::to_string(kNumFiles);
+  ASSERT_EQ(files_per_level_pre_compaction, FilesPerLevel(0 /* cf */));
+#endif  // !ROCKSDB_LITE
+
+  std::int64_t prev_total_request =
+      options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL);
+  ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_LOW));
+
+  Compact(kStartKey, kEndKey);
+
+  std::int64_t actual_compaction_request =
+      options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL) -
+      prev_total_request;
+
+  // Post-comaction:
+  // level-0 : 0 SST file
+  // level-1 : 1 SST file
+#ifndef ROCKSDB_LITE
+  std::string files_per_level_post_compaction = "0,1";
+  ASSERT_EQ(files_per_level_post_compaction, FilesPerLevel(0 /* cf */));
+#endif  // !ROCKSDB_LITE
+
+  std::int64_t exepcted_compaction_request = 1;
+  EXPECT_EQ(actual_compaction_request, exepcted_compaction_request);
+  EXPECT_EQ(actual_compaction_request,
+            options_.rate_limiter->GetTotalRequests(Env::IO_LOW));
+}
+
+class DBRateLimiterOnWriteWALTest
+    : public DBRateLimiterOnWriteTest,
+      public ::testing::WithParamInterface<std::tuple<
+          bool /* WriteOptions::disableWal */,
+          bool /* Options::manual_wal_flush */,
+          Env::IOPriority /* WriteOptions::rate_limiter_priority */>> {
+ public:
+  static std::string GetTestNameSuffix(
+      ::testing::TestParamInfo<std::tuple<bool, bool, Env::IOPriority>> info) {
+    std::ostringstream oss;
+    if (std::get<0>(info.param)) {
+      oss << "DisableWAL";
+    } else {
+      oss << "EnableWAL";
+    }
+    if (std::get<1>(info.param)) {
+      oss << "_ManualWALFlush";
+    } else {
+      oss << "_AutoWALFlush";
+    }
+    if (std::get<2>(info.param) == Env::IO_USER) {
+      oss << "_RateLimitAutoWALFlush";
+    } else if (std::get<2>(info.param) == Env::IO_TOTAL) {
+      oss << "_NoRateLimitAutoWALFlush";
+    } else {
+      oss << "_RateLimitAutoWALFlushWithIncorrectPriority";
+    }
+    return oss.str();
+  }
+
+  explicit DBRateLimiterOnWriteWALTest()
+      : disable_wal_(std::get<0>(GetParam())),
+        manual_wal_flush_(std::get<1>(GetParam())),
+        rate_limiter_priority_(std::get<2>(GetParam())) {}
+
+  void Init() {
+    options_ = GetOptions();
+    options_.manual_wal_flush = manual_wal_flush_;
+    Reopen(options_);
+  }
+
+  WriteOptions GetWriteOptions() {
+    WriteOptions write_options;
+    write_options.disableWAL = disable_wal_;
+    write_options.rate_limiter_priority = rate_limiter_priority_;
+    return write_options;
+  }
+
+ protected:
+  bool disable_wal_;
+  bool manual_wal_flush_;
+  Env::IOPriority rate_limiter_priority_;
+};
+
+INSTANTIATE_TEST_CASE_P(
+    DBRateLimiterOnWriteWALTest, DBRateLimiterOnWriteWALTest,
+    ::testing::Values(std::make_tuple(false, false, Env::IO_TOTAL),
+                      std::make_tuple(false, false, Env::IO_USER),
+                      std::make_tuple(false, false, Env::IO_HIGH),
+                      std::make_tuple(false, true, Env::IO_USER),
+                      std::make_tuple(true, false, Env::IO_USER)),
+    DBRateLimiterOnWriteWALTest::GetTestNameSuffix);
+
+TEST_P(DBRateLimiterOnWriteWALTest, AutoWalFlush) {
+  Init();
+
+  const bool no_rate_limit_auto_wal_flush =
+      (rate_limiter_priority_ == Env::IO_TOTAL);
+  const bool valid_arg = (rate_limiter_priority_ == Env::IO_USER &&
+                          !disable_wal_ && !manual_wal_flush_);
+
+  std::int64_t prev_total_request =
+      options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL);
+  ASSERT_EQ(0, options_.rate_limiter->GetTotalRequests(Env::IO_USER));
+
+  Status s = Put("foo", "v1", GetWriteOptions());
+
+  if (no_rate_limit_auto_wal_flush || valid_arg) {
+    EXPECT_TRUE(s.ok());
+  } else {
+    EXPECT_TRUE(s.IsInvalidArgument());
+    EXPECT_TRUE(s.ToString().find("WriteOptions::rate_limiter_priority") !=
+                std::string::npos);
+  }
+
+  std::int64_t actual_auto_wal_flush_request =
+      options_.rate_limiter->GetTotalRequests(Env::IO_TOTAL) -
+      prev_total_request;
+  std::int64_t expected_auto_wal_flush_request = valid_arg ? 1 : 0;
+
+  EXPECT_EQ(actual_auto_wal_flush_request, expected_auto_wal_flush_request);
+  EXPECT_EQ(actual_auto_wal_flush_request,
+            options_.rate_limiter->GetTotalRequests(Env::IO_USER));
+}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_test2.cc

@@ -21,7 +21,6 @@
 #include "rocksdb/persistent_cache.h"
 #include "rocksdb/trace_record.h"
 #include "rocksdb/trace_record_result.h"
-#include "rocksdb/utilities/backup_engine.h"
 #include "rocksdb/utilities/replayer.h"
 #include "rocksdb/wal_filter.h"
 #include "test_util/testutil.h"
@@ -410,6 +409,9 @@ TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) {
     ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
     ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[2]));
     ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[3]));
+    // Ensure background work is fully finished including listener callbacks
+    // before accessing listener state.
+    ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
   };
 
   // Create some data and flush "default" and "nikitich" so that they
@@ -588,6 +590,11 @@ TEST_F(DBTest2, SharedWriteBufferLimitAcrossDB) {
     ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
     ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[2]));
     ASSERT_OK(static_cast<DBImpl*>(db2)->TEST_WaitForFlushMemTable());
+    // Ensure background work is fully finished including listener callbacks
+    // before accessing listener state.
+    ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+    ASSERT_OK(
+        static_cast_with_check<DBImpl>(db2)->TEST_WaitForBackgroundWork());
   };
 
   // Trigger a flush on cf2
@@ -6901,122 +6908,18 @@ TEST_F(DBTest2, LastLevelStatistics) {
             options.statistics->getTickerCount(WARM_FILE_READ_COUNT));
 }
 
-class FileTemperatureTestFS : public FileSystemWrapper {
- public:
-  explicit FileTemperatureTestFS(SpecialEnv* env)
-      : FileSystemWrapper(env->GetFileSystem()) {}
+TEST_F(DBTest2, CheckpointFileTemperature) {
+  class NoLinkTestFS : public FileTemperatureTestFS {
+    using FileTemperatureTestFS::FileTemperatureTestFS;
 
-  static const char* kClassName() { return "TestFileSystem"; }
-  const char* Name() const override { return kClassName(); }
-
-  IOStatus NewSequentialFile(const std::string& fname, const FileOptions& opts,
-                             std::unique_ptr<FSSequentialFile>* result,
-                             IODebugContext* dbg) override {
-    auto filename = GetFileName(fname);
-    uint64_t number;
-    FileType type;
-    auto r = ParseFileName(filename, &number, &type);
-    assert(r);
-    if (type == kTableFile) {
-      auto emplaced =
-          requested_sst_file_temperatures_.emplace(number, opts.temperature);
-      assert(emplaced.second);  // assume no duplication
-    }
-    return target()->NewSequentialFile(fname, opts, result, dbg);
-  }
-
-  IOStatus LinkFile(const std::string& s, const std::string& t,
-                    const IOOptions& options, IODebugContext* dbg) override {
-    auto filename = GetFileName(s);
-    uint64_t number;
-    FileType type;
-    auto r = ParseFileName(filename, &number, &type);
-    assert(r);
-    // return not supported to force checkpoint copy the file instead of just
-    // link
-    if (type == kTableFile) {
+    IOStatus LinkFile(const std::string&, const std::string&, const IOOptions&,
+                      IODebugContext*) override {
+      // return not supported to force checkpoint copy the file instead of just
+      // link
       return IOStatus::NotSupported();
     }
-    return target()->LinkFile(s, t, options, dbg);
-  }
-
-  const std::map<uint64_t, Temperature>& RequestedSstFileTemperatures() {
-    return requested_sst_file_temperatures_;
-  }
-
-  void ClearRequestedFileTemperatures() {
-    requested_sst_file_temperatures_.clear();
-  }
-
- private:
-  std::map<uint64_t, Temperature> requested_sst_file_temperatures_;
-
-  std::string GetFileName(const std::string& fname) {
-    auto filename = fname.substr(fname.find_last_of(kFilePathSeparator) + 1);
-    // workaround only for Windows that the file path could contain both Windows
-    // FilePathSeparator and '/'
-    filename = filename.substr(filename.find_last_of('/') + 1);
-    return filename;
-  }
-};
-
-TEST_F(DBTest2, BackupFileTemperature) {
-  std::shared_ptr<FileTemperatureTestFS> test_fs =
-      std::make_shared<FileTemperatureTestFS>(env_);
-  std::unique_ptr<Env> backup_env(new CompositeEnvWrapper(env_, test_fs));
-  Options options = CurrentOptions();
-  options.bottommost_temperature = Temperature::kWarm;
-  options.level0_file_num_compaction_trigger = 2;
-  Reopen(options);
-
-  // generate a bottommost file and a non-bottommost file
-  ASSERT_OK(Put("foo", "bar"));
-  ASSERT_OK(Put("bar", "bar"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Put("foo", "bar"));
-  ASSERT_OK(Put("bar", "bar"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_OK(Put("foo", "bar"));
-  ASSERT_OK(Put("bar", "bar"));
-  ASSERT_OK(Flush());
-  auto size = GetSstSizeHelper(Temperature::kWarm);
-  ASSERT_GT(size, 0);
-
-  std::map<uint64_t, Temperature> temperatures;
-  std::vector<LiveFileStorageInfo> infos;
-  ASSERT_OK(
-      dbfull()->GetLiveFilesStorageInfo(LiveFilesStorageInfoOptions(), &infos));
-  for (auto info : infos) {
-    temperatures.emplace(info.file_number, info.temperature);
-  }
-  BackupEngine* backup_engine;
-  auto backup_options = BackupEngineOptions(
-      dbname_ + kFilePathSeparator + "tempbk", backup_env.get());
-  auto s = BackupEngine::Open(backup_env.get(), backup_options, &backup_engine);
-  ASSERT_OK(s);
-  s = backup_engine->CreateNewBackup(db_);
-  ASSERT_OK(s);
-
-  // checking src file src_temperature hints: 2 sst files: 1 sst is kWarm,
-  // another is kUnknown
-  auto file_temperatures = test_fs->RequestedSstFileTemperatures();
-  ASSERT_EQ(file_temperatures.size(), 2);
-  bool has_only_one_warm_sst = false;
-  for (const auto& file_temperature : file_temperatures) {
-    ASSERT_EQ(temperatures.at(file_temperature.first), file_temperature.second);
-    if (file_temperature.second == Temperature::kWarm) {
-      ASSERT_FALSE(has_only_one_warm_sst);
-      has_only_one_warm_sst = true;
-    }
-  }
-  ASSERT_TRUE(has_only_one_warm_sst);
-  Close();
-}
-
-TEST_F(DBTest2, CheckpointFileTemperature) {
-  std::shared_ptr<FileTemperatureTestFS> test_fs =
-      std::make_shared<FileTemperatureTestFS>(env_);
+  };
+  auto test_fs = std::make_shared<NoLinkTestFS>(env_->GetFileSystem());
   std::unique_ptr<Env> env(new CompositeEnvWrapper(env_, test_fs));
   Options options = CurrentOptions();
   options.bottommost_temperature = Temperature::kWarm;

db/db_test_util.h

@@ -687,6 +687,50 @@ class SpecialEnv : public EnvWrapper {
 };
 
 #ifndef ROCKSDB_LITE
+class FileTemperatureTestFS : public FileSystemWrapper {
+ public:
+  explicit FileTemperatureTestFS(const std::shared_ptr<FileSystem>& fs)
+      : FileSystemWrapper(fs) {}
+
+  static const char* kClassName() { return "FileTemperatureTestFS"; }
+  const char* Name() const override { return kClassName(); }
+
+  IOStatus NewSequentialFile(const std::string& fname, const FileOptions& opts,
+                             std::unique_ptr<FSSequentialFile>* result,
+                             IODebugContext* dbg) override {
+    auto filename = GetFileName(fname);
+    uint64_t number;
+    FileType type;
+    auto r = ParseFileName(filename, &number, &type);
+    assert(r);
+    if (type == kTableFile) {
+      auto emplaced =
+          requested_sst_file_temperatures_.emplace(number, opts.temperature);
+      assert(emplaced.second);  // assume no duplication
+    }
+    return target()->NewSequentialFile(fname, opts, result, dbg);
+  }
+
+  const std::map<uint64_t, Temperature>& RequestedSstFileTemperatures() {
+    return requested_sst_file_temperatures_;
+  }
+
+  void ClearRequestedFileTemperatures() {
+    requested_sst_file_temperatures_.clear();
+  }
+
+ protected:
+  std::map<uint64_t, Temperature> requested_sst_file_temperatures_;
+
+  std::string GetFileName(const std::string& fname) {
+    auto filename = fname.substr(fname.find_last_of(kFilePathSeparator) + 1);
+    // workaround only for Windows that the file path could contain both Windows
+    // FilePathSeparator and '/'
+    filename = filename.substr(filename.find_last_of('/') + 1);
+    return filename;
+  }
+};
+
 class OnFileDeletionListener : public EventListener {
  public:
   OnFileDeletionListener() : matched_count_(0), expected_file_name_("") {}

db/external_sst_file_ingestion_job.cc

@@ -777,10 +777,11 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
         const std::vector<FileMetaData*>& level_files =
             vstorage->LevelFiles(lvl);
         const SequenceNumber level_largest_seqno =
-            (*max_element(level_files.begin(), level_files.end(),
-                          [](FileMetaData* f1, FileMetaData* f2) {
-                            return f1->fd.largest_seqno < f2->fd.largest_seqno;
-                          }))
+            (*std::max_element(level_files.begin(), level_files.end(),
+                               [](FileMetaData* f1, FileMetaData* f2) {
+                                 return f1->fd.largest_seqno <
+                                        f2->fd.largest_seqno;
+                               }))
                 ->fd.largest_seqno;
         // should only assign seqno to current level's largest seqno when
         // the file fits

db/listener_test.cc

@@ -340,6 +340,9 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) {
   for (int i = 1; i < 8; ++i) {
     ASSERT_OK(Flush(i));
     ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
+    // Ensure background work is fully finished including listener callbacks
+    // before accessing listener state.
+    ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
     ASSERT_EQ(listener->flushed_dbs_.size(), i);
     ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
   }
@@ -462,6 +465,13 @@ TEST_F(EventListenerTest, MultiDBMultiListeners) {
     }
   }
 
+  for (int d = 0; d < kNumDBs; ++d) {
+    // Ensure background work is fully finished including listener callbacks
+    // before accessing listener state.
+    ASSERT_OK(
+        static_cast_with_check<DBImpl>(dbs[d])->TEST_WaitForBackgroundWork());
+  }
+
   for (auto* listener : listeners) {
     int pos = 0;
     for (size_t c = 0; c < cf_names.size(); ++c) {
@@ -523,10 +533,10 @@ TEST_F(EventListenerTest, DisableBGCompaction) {
     ASSERT_OK(db_->Flush(fo, handles_[1]));
     db_->GetColumnFamilyMetaData(handles_[1], &cf_meta);
   }
-  ASSERT_GE(listener->slowdown_count, kSlowdownTrigger * 9);
-  // We don't want the listener executing during DBTestBase::Close() due to
-  // race on handles_.
+  // Ensure background work is fully finished including listener callbacks
+  // before accessing listener state.
   ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+  ASSERT_GE(listener->slowdown_count, kSlowdownTrigger * 9);
 }
 
 class TestCompactionReasonListener : public EventListener {

db/log_test.cc

@@ -17,6 +17,7 @@
 #include "util/coding.h"
 #include "util/crc32c.h"
 #include "util/random.h"
+#include "utilities/memory_allocators.h"
 
 namespace ROCKSDB_NAMESPACE {
 namespace log {
@@ -918,6 +919,73 @@ INSTANTIATE_TEST_CASE_P(
                        ::testing::Values(CompressionType::kNoCompression,
                                          CompressionType::kZSTD)));
 
+class StreamingCompressionTest
+    : public ::testing::TestWithParam<std::tuple<int, CompressionType>> {};
+
+TEST_P(StreamingCompressionTest, Basic) {
+  size_t input_size = std::get<0>(GetParam());
+  CompressionType compression_type = std::get<1>(GetParam());
+  if (!StreamingCompressionTypeSupported(compression_type)) {
+    ROCKSDB_GTEST_SKIP("Test requires support for compression type");
+    return;
+  }
+  CompressionOptions opts;
+  constexpr uint32_t compression_format_version = 2;
+  StreamingCompress* compress = StreamingCompress::Create(
+      compression_type, opts, compression_format_version, kBlockSize);
+  StreamingUncompress* uncompress = StreamingUncompress::Create(
+      compression_type, compression_format_version, kBlockSize);
+  MemoryAllocator* allocator = new DefaultMemoryAllocator();
+  std::string input_buffer = BigString("abc", input_size);
+  std::vector<std::string> compressed_buffers;
+  size_t remaining;
+  // Call compress till the entire input is consumed
+  do {
+    char* output_buffer = (char*)allocator->Allocate(kBlockSize);
+    size_t output_size;
+    remaining = compress->Compress(input_buffer.c_str(), input_size,
+                                   output_buffer, &output_size);
+    if (output_size > 0) {
+      std::string compressed_buffer;
+      compressed_buffer.assign(output_buffer, output_size);
+      compressed_buffers.emplace_back(std::move(compressed_buffer));
+    }
+    allocator->Deallocate((void*)output_buffer);
+  } while (remaining > 0);
+  std::string uncompressed_buffer = "";
+  int ret_val = 0;
+  size_t output_size;
+  char* uncompressed_output_buffer = (char*)allocator->Allocate(kBlockSize);
+  // Uncompress the fragments and concatenate them.
+  for (int i = 0; i < (int)compressed_buffers.size(); i++) {
+    // Call uncompress till either the entire input is consumed or the output
+    // buffer size is equal to the allocated output buffer size.
+    do {
+      ret_val = uncompress->Uncompress(
+          compressed_buffers[i].c_str(), compressed_buffers[i].size(),
+          uncompressed_output_buffer, &output_size);
+      if (output_size > 0) {
+        std::string uncompressed_fragment;
+        uncompressed_fragment.assign(uncompressed_output_buffer, output_size);
+        uncompressed_buffer += uncompressed_fragment;
+      }
+    } while (ret_val > 0 || output_size == kBlockSize);
+  }
+  allocator->Deallocate((void*)uncompressed_output_buffer);
+  delete allocator;
+  delete compress;
+  delete uncompress;
+  // The final return value from uncompress() should be 0.
+  ASSERT_EQ(ret_val, 0);
+  ASSERT_EQ(input_buffer, uncompressed_buffer);
+}
+
+INSTANTIATE_TEST_CASE_P(
+    StreamingCompression, StreamingCompressionTest,
+    ::testing::Combine(::testing::Values(10, 100, 1000, kBlockSize,
+                                         kBlockSize * 2),
+                       ::testing::Values(CompressionType::kZSTD)));
+
 }  // namespace log
 }  // namespace ROCKSDB_NAMESPACE
 

db/log_writer.cc

@@ -50,7 +50,8 @@ IOStatus Writer::Close() {
   return s;
 }
 
-IOStatus Writer::AddRecord(const Slice& slice) {
+IOStatus Writer::AddRecord(const Slice& slice,
+                           Env::IOPriority rate_limiter_priority) {
   const char* ptr = slice.data();
   size_t left = slice.size();
 
@@ -73,7 +74,8 @@ IOStatus Writer::AddRecord(const Slice& slice) {
         // kRecyclableHeaderSize being <= 11)
         assert(header_size <= 11);
         s = dest_->Append(Slice("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",
-                                static_cast<size_t>(leftover)));
+                                static_cast<size_t>(leftover)),
+                          0 /* crc32c_checksum */, rate_limiter_priority);
         if (!s.ok()) {
           break;
         }
@@ -99,7 +101,7 @@ IOStatus Writer::AddRecord(const Slice& slice) {
       type = recycle_log_files_ ? kRecyclableMiddleType : kMiddleType;
     }
 
-    s = EmitPhysicalRecord(type, ptr, fragment_length);
+    s = EmitPhysicalRecord(type, ptr, fragment_length, rate_limiter_priority);
     ptr += fragment_length;
     left -= fragment_length;
     begin = false;
@@ -107,7 +109,7 @@ IOStatus Writer::AddRecord(const Slice& slice) {
 
   if (s.ok()) {
     if (!manual_flush_) {
-      s = dest_->Flush();
+      s = dest_->Flush(rate_limiter_priority);
     }
   }
 
@@ -141,7 +143,8 @@ IOStatus Writer::AddCompressionTypeRecord() {
 
 bool Writer::TEST_BufferIsEmpty() { return dest_->TEST_BufferIsEmpty(); }
 
-IOStatus Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
+IOStatus Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n,
+                                    Env::IOPriority rate_limiter_priority) {
   assert(n <= 0xffff);  // Must fit in two bytes
 
   size_t header_size;
@@ -180,9 +183,10 @@ IOStatus Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
   EncodeFixed32(buf, crc);
 
   // Write the header and the payload
-  IOStatus s = dest_->Append(Slice(buf, header_size));
+  IOStatus s = dest_->Append(Slice(buf, header_size), 0 /* crc32c_checksum */,
+                             rate_limiter_priority);
   if (s.ok()) {
-    s = dest_->Append(Slice(ptr, n), payload_crc);
+    s = dest_->Append(Slice(ptr, n), payload_crc, rate_limiter_priority);
   }
   block_offset_ += header_size + n;
   return s;

db/log_writer.h

@@ -13,6 +13,7 @@
 
 #include "db/log_format.h"
 #include "rocksdb/compression_type.h"
+#include "rocksdb/env.h"
 #include "rocksdb/io_status.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"
@@ -81,7 +82,8 @@ class Writer {
 
   ~Writer();
 
-  IOStatus AddRecord(const Slice& slice);
+  IOStatus AddRecord(const Slice& slice,
+                     Env::IOPriority rate_limiter_priority = Env::IO_TOTAL);
   IOStatus AddCompressionTypeRecord();
 
   WritableFileWriter* file() { return dest_.get(); }
@@ -106,7 +108,9 @@ class Writer {
   // record type stored in the header.
   uint32_t type_crc_[kMaxRecordType + 1];
 
-  IOStatus EmitPhysicalRecord(RecordType type, const char* ptr, size_t length);
+  IOStatus EmitPhysicalRecord(
+      RecordType type, const char* ptr, size_t length,
+      Env::IOPriority rate_limiter_priority = Env::IO_TOTAL);
 
   // If true, it does not flush after each write. Instead it relies on the upper
   // layer to manually does the flush by calling ::WriteBuffer()

db/write_batch.cc

@@ -800,6 +800,7 @@ Status WriteBatch::Put(ColumnFamilyHandle* column_family, const Slice& key,
   }
 
   needs_in_place_update_ts_ = true;
+  has_key_with_ts_ = true;
   std::string dummy_ts(ts_sz, '\0');
   std::array<Slice, 2> key_with_ts{{key, dummy_ts}};
   return WriteBatchInternal::Put(this, cf_id, SliceParts(key_with_ts.data(), 2),
@@ -812,6 +813,7 @@ Status WriteBatch::Put(ColumnFamilyHandle* column_family, const Slice& key,
   if (!s.ok()) {
     return s;
   }
+  has_key_with_ts_ = true;
   assert(column_family);
   uint32_t cf_id = column_family->GetID();
   std::array<Slice, 2> key_with_ts{{key, ts}};
@@ -1002,6 +1004,7 @@ Status WriteBatch::Delete(ColumnFamilyHandle* column_family, const Slice& key) {
   }
 
   needs_in_place_update_ts_ = true;
+  has_key_with_ts_ = true;
   std::string dummy_ts(ts_sz, '\0');
   std::array<Slice, 2> key_with_ts{{key, dummy_ts}};
   return WriteBatchInternal::Delete(this, cf_id,
@@ -1015,6 +1018,7 @@ Status WriteBatch::Delete(ColumnFamilyHandle* column_family, const Slice& key,
     return s;
   }
   assert(column_family);
+  has_key_with_ts_ = true;
   uint32_t cf_id = column_family->GetID();
   std::array<Slice, 2> key_with_ts{{key, ts}};
   return WriteBatchInternal::Delete(this, cf_id,
@@ -1115,6 +1119,7 @@ Status WriteBatch::SingleDelete(ColumnFamilyHandle* column_family,
   }
 
   needs_in_place_update_ts_ = true;
+  has_key_with_ts_ = true;
   std::string dummy_ts(ts_sz, '\0');
   std::array<Slice, 2> key_with_ts{{key, dummy_ts}};
   return WriteBatchInternal::SingleDelete(this, cf_id,
@@ -1127,6 +1132,7 @@ Status WriteBatch::SingleDelete(ColumnFamilyHandle* column_family,
   if (!s.ok()) {
     return s;
   }
+  has_key_with_ts_ = true;
   assert(column_family);
   uint32_t cf_id = column_family->GetID();
   std::array<Slice, 2> key_with_ts{{key, ts}};

db/write_batch_internal.h

@@ -228,6 +228,10 @@ class WriteBatchInternal {
   static bool TimestampsUpdateNeeded(const WriteBatch& wb) {
     return wb.needs_in_place_update_ts_;
   }
+
+  static bool HasKeyWithTimestamp(const WriteBatch& wb) {
+    return wb.has_key_with_ts_;
+  }
 };
 
 // LocalSavePoint is similar to a scope guard

db/write_batch_test.cc

@@ -1009,6 +1009,29 @@ TEST_F(WriteBatchTest, UpdateTimestamps) {
       {4, cf4.GetComparator()},
       {5, cf5.GetComparator()}};
 
+  static constexpr size_t timestamp_size = sizeof(uint64_t);
+
+  {
+    WriteBatch wb1, wb2, wb3, wb4, wb5, wb6, wb7;
+    ASSERT_OK(wb1.Put(&cf0, "key", "value"));
+    ASSERT_FALSE(WriteBatchInternal::HasKeyWithTimestamp(wb1));
+    ASSERT_OK(wb2.Put(&cf4, "key", "value"));
+    ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb2));
+    ASSERT_OK(wb3.Put(&cf4, "key", /*ts=*/std::string(timestamp_size, '\xfe'),
+                      "value"));
+    ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb3));
+    ASSERT_OK(wb4.Delete(&cf4, "key",
+                         /*ts=*/std::string(timestamp_size, '\xfe')));
+    ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb4));
+    ASSERT_OK(wb5.Delete(&cf4, "key"));
+    ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb5));
+    ASSERT_OK(wb6.SingleDelete(&cf4, "key"));
+    ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb6));
+    ASSERT_OK(wb7.SingleDelete(&cf4, "key",
+                               /*ts=*/std::string(timestamp_size, '\xfe')));
+    ASSERT_TRUE(WriteBatchInternal::HasKeyWithTimestamp(wb7));
+  }
+
   WriteBatch batch;
   // Write to the batch. We will assign timestamps later.
   for (const auto& key_str : key_strs) {
@@ -1017,7 +1040,6 @@ TEST_F(WriteBatchTest, UpdateTimestamps) {
     ASSERT_OK(batch.Put(&cf5, key_str, "value"));
   }
 
-  static constexpr size_t timestamp_size = sizeof(uint64_t);
   const auto checker1 = [](uint32_t cf) {
     if (cf == 4 || cf == 5) {
       return timestamp_size;

db/write_thread.cc

@@ -471,6 +471,11 @@ size_t WriteThread::EnterAsBatchGroupLeader(Writer* leader,
       break;
     }
 
+    if (w->rate_limiter_priority != leader->rate_limiter_priority) {
+      // Do not mix writes with different rate limiter priorities.
+      break;
+    }
+
     if (w->batch == nullptr) {
       // Do not include those writes with nullptr batch. Those are not writes,
       // those are something else. They want to be alone

db/write_thread.h

@@ -117,6 +117,7 @@ class WriteThread {
     bool sync;
     bool no_slowdown;
     bool disable_wal;
+    Env::IOPriority rate_limiter_priority;
     bool disable_memtable;
     size_t batch_cnt;  // if non-zero, number of sub-batches in the write batch
     size_t protection_bytes_per_key;
@@ -141,6 +142,7 @@ class WriteThread {
           sync(false),
           no_slowdown(false),
           disable_wal(false),
+          rate_limiter_priority(Env::IOPriority::IO_TOTAL),
           disable_memtable(false),
           batch_cnt(0),
           protection_bytes_per_key(0),
@@ -163,6 +165,7 @@ class WriteThread {
           sync(write_options.sync),
           no_slowdown(write_options.no_slowdown),
           disable_wal(write_options.disableWAL),
+          rate_limiter_priority(write_options.rate_limiter_priority),
           disable_memtable(_disable_memtable),
           batch_cnt(_batch_cnt),
           protection_bytes_per_key(_batch->GetProtectionBytesPerKey()),

db_stress_tool/db_stress_common.h

@@ -176,6 +176,7 @@ DECLARE_int32(range_deletion_width);
 DECLARE_uint64(rate_limiter_bytes_per_sec);
 DECLARE_bool(rate_limit_bg_reads);
 DECLARE_bool(rate_limit_user_ops);
+DECLARE_bool(rate_limit_auto_wal_flush);
 DECLARE_uint64(sst_file_manager_bytes_per_sec);
 DECLARE_uint64(sst_file_manager_bytes_per_truncate);
 DECLARE_bool(use_txn);

db_stress_tool/db_stress_gflags.cc

@@ -550,6 +550,12 @@ DEFINE_bool(rate_limit_user_ops, false,
             "When true use Env::IO_USER priority level to charge internal rate "
             "limiter for reads associated with user operations.");
 
+DEFINE_bool(rate_limit_auto_wal_flush, false,
+            "When true use Env::IO_USER priority level to charge internal rate "
+            "limiter for automatic WAL flush (`Options::manual_wal_flush` == "
+            "false) after the user "
+            "write operation.");
+
 DEFINE_uint64(sst_file_manager_bytes_per_sec, 0,
               "Set `Options::sst_file_manager` to delete at this rate. By "
               "default the deletion rate is unbounded.");

db_stress_tool/db_stress_listener.cc

@@ -7,6 +7,7 @@
 
 #include <cstdint>
 
+#include "file/file_util.h"
 #include "rocksdb/file_system.h"
 #include "util/coding_lean.h"
 
@@ -37,10 +38,31 @@ UniqueIdVerifier::UniqueIdVerifier(const std::string& db_name, Env* env)
     exit(1);
   }
 
+  // Avoid relying on ReopenWritableFile which is not supported by all
+  // file systems. Create a new file and copy the old file contents to it.
+  std::string tmp_path = path_ + ".tmp";
+  st = fs->FileExists(tmp_path, opts, /*dbg*/ nullptr);
+  if (st.IsNotFound()) {
+    st = fs->RenameFile(path_, tmp_path, opts, /*dbg*/ nullptr);
+    // Either it should succeed or fail because src path doesn't exist
+    assert(st.ok() || st.IsPathNotFound());
+  } else {
+    // If path_ and tmp_path both exist, retain tmp_path as its
+    // guaranteed to be more complete. The order of operations are -
+    // 1. Rename path_ to tmp_path
+    // 2. Parse tmp_path contents
+    // 3. Create path_
+    // 4. Copy tmp_path contents to path_
+    // 5. Delete tmp_path
+    st = fs->DeleteFile(path_, opts, /*dbg*/ nullptr);
+    assert(st.ok() || st.IsPathNotFound());
+  }
+
+  uint64_t size = 0;
   {
     std::unique_ptr<FSSequentialFile> reader;
-    Status s =
-        fs->NewSequentialFile(path_, FileOptions(), &reader, /*dbg*/ nullptr);
+    Status s = fs->NewSequentialFile(tmp_path, FileOptions(), &reader,
+                                     /*dbg*/ nullptr);
     if (s.ok()) {
       // Load from file
       std::string id(24U, '\0');
@@ -60,37 +82,57 @@ UniqueIdVerifier::UniqueIdVerifier(const std::string& db_name, Env* env)
             fprintf(stdout, "Warning: clearing corrupt unique id file\n");
             id_set_.clear();
             reader.reset();
-            s = fs->DeleteFile(path_, opts, /*dbg*/ nullptr);
+            s = fs->DeleteFile(tmp_path, opts, /*dbg*/ nullptr);
             assert(s.ok());
+            size = 0;
           }
           break;
         }
+        size += 24U;
         VerifyNoWrite(id);
       }
     } else {
       // Newly created is ok.
       // But FileSystem doesn't tell us whether non-existence was the cause of
       // the failure. (Issue #9021)
-      Status s2 = fs->FileExists(path_, opts, /*dbg*/ nullptr);
+      Status s2 = fs->FileExists(tmp_path, opts, /*dbg*/ nullptr);
       if (!s2.IsNotFound()) {
         fprintf(stderr, "Error opening unique id file: %s\n",
                 s.ToString().c_str());
         assert(false);
       }
+      size = 0;
     }
   }
   fprintf(stdout, "(Re-)verified %zu unique IDs\n", id_set_.size());
-  Status s = fs->ReopenWritableFile(path_, FileOptions(), &data_file_writer_,
-                                    /*dbg*/ nullptr);
-  if (!s.ok()) {
-    fprintf(stderr, "Error opening unique id file for append: %s\n",
-            s.ToString().c_str());
+
+  std::unique_ptr<FSWritableFile> file_writer;
+  st = fs->NewWritableFile(path_, FileOptions(), &file_writer, /*dbg*/ nullptr);
+  if (!st.ok()) {
+    fprintf(stderr, "Error creating the unique ids file: %s\n",
+            st.ToString().c_str());
     assert(false);
   }
+  data_file_writer_.reset(
+      new WritableFileWriter(std::move(file_writer), path_, FileOptions()));
+
+  if (size > 0) {
+    st = CopyFile(fs.get(), tmp_path, data_file_writer_, size,
+                  /*use_fsync*/ true, /*io_tracer*/ nullptr,
+                  /*temparature*/ Temperature::kHot);
+    if (!st.ok()) {
+      fprintf(stderr, "Error copying contents of old unique id file: %s\n",
+              st.ToString().c_str());
+      assert(false);
+    }
+  }
+  st = fs->DeleteFile(tmp_path, opts, /*dbg*/ nullptr);
+  assert(st.ok() || st.IsPathNotFound());
 }
 
 UniqueIdVerifier::~UniqueIdVerifier() {
-  data_file_writer_->Close(IOOptions(), /*dbg*/ nullptr);
+  IOStatus s = data_file_writer_->Close();
+  assert(s.ok());
 }
 
 void UniqueIdVerifier::VerifyNoWrite(const std::string& id) {
@@ -112,14 +154,13 @@ void UniqueIdVerifier::Verify(const std::string& id) {
   if (id_set_.size() >= 4294967) {
     return;
   }
-  IOStatus s =
-      data_file_writer_->Append(Slice(id), IOOptions(), /*dbg*/ nullptr);
+  IOStatus s = data_file_writer_->Append(Slice(id));
   if (!s.ok()) {
     fprintf(stderr, "Error writing to unique id file: %s\n",
             s.ToString().c_str());
     assert(false);
   }
-  s = data_file_writer_->Flush(IOOptions(), /*dbg*/ nullptr);
+  s = data_file_writer_->Flush();
   if (!s.ok()) {
     fprintf(stderr, "Error flushing unique id file: %s\n",
             s.ToString().c_str());

db_stress_tool/db_stress_listener.h

@@ -10,6 +10,7 @@
 #include <unordered_set>
 
 #include "file/filename.h"
+#include "file/writable_file_writer.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/file_system.h"
@@ -39,7 +40,7 @@ class UniqueIdVerifier {
   std::mutex mutex_;
   // IDs persisted to a hidden file inside DB dir
   std::string path_;
-  std::unique_ptr<FSWritableFile> data_file_writer_;
+  std::unique_ptr<WritableFileWriter> data_file_writer_;
   // Starting byte for which 8 bytes to check in memory within 24 byte ID
   size_t offset_;
   // Working copy of the set of 8 byte pieces

db_stress_tool/db_stress_test_base.cc

@@ -485,6 +485,9 @@ void StressTest::PreloadDbAndReopenAsReadOnly(int64_t number_of_keys,
   if (FLAGS_sync) {
     write_opts.sync = true;
   }
+  if (FLAGS_rate_limit_auto_wal_flush) {
+    write_opts.rate_limiter_priority = Env::IO_USER;
+  }
   char value[100];
   int cf_idx = 0;
   Status s;
@@ -640,6 +643,9 @@ void StressTest::OperateDb(ThreadState* thread) {
   read_opts.rate_limiter_priority =
       FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL;
   WriteOptions write_opts;
+  if (FLAGS_rate_limit_auto_wal_flush) {
+    write_opts.rate_limiter_priority = Env::IO_USER;
+  }
   auto shared = thread->shared;
   char value[100];
   std::string from_db;
@@ -1744,6 +1750,8 @@ Status StressTest::TestCheckpoint(ThreadState* thread,
   if (s.ok()) {
     Options options(options_);
     options.listeners.clear();
+    // Avoid race condition in trash handling after delete checkpoint_db
+    options.sst_file_manager.reset();
     std::vector<ColumnFamilyDescriptor> cf_descs;
     // TODO(ajkr): `column_family_names_` is not safe to access here when
     // `clear_column_family_one_in != 0`. But we can't easily switch to

db_stress_tool/no_batched_ops_stress.cc

@@ -271,6 +271,9 @@ class NonBatchedOpsStressTest : public StressTest {
     Transaction* txn = nullptr;
     if (use_txn) {
       WriteOptions wo;
+      if (FLAGS_rate_limit_auto_wal_flush) {
+        wo.rate_limiter_priority = Env::IO_USER;
+      }
       Status s = NewTxn(wo, &txn);
       if (!s.ok()) {
         fprintf(stderr, "NewTxn: %s\n", s.ToString().c_str());

docs/Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
-gem 'github-pages', '~> 209'
+gem 'github-pages', '~> 225'
 
 gem "webrick", "~> 1.7"

docs/Gemfile.lock

@@ -1,7 +1,7 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.4)
+    activesupport (6.0.4.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -14,60 +14,78 @@ GEM
       execjs
     coffee-script-source (1.11.1)
     colorator (1.1.0)
-    commonmarker (0.17.13)
-      ruby-enum (~> 0.5)
-    concurrent-ruby (1.1.7)
-    dnsruby (1.61.5)
+    commonmarker (0.23.4)
+    concurrent-ruby (1.1.9)
+    dnsruby (1.61.9)
       simpleidn (~> 0.1)
-    em-websocket (0.5.2)
+    em-websocket (0.5.3)
       eventmachine (>= 0.12.9)
-      http_parser.rb (~> 0.6.0)
-    ethon (0.12.0)
-      ffi (>= 1.3.0)
+      http_parser.rb (~> 0)
+    ethon (0.15.0)
+      ffi (>= 1.15.0)
     eventmachine (1.2.7)
-    execjs (2.7.0)
-    faraday (1.3.0)
+    execjs (2.8.1)
+    faraday (1.10.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0)
+      faraday-multipart (~> 1.0)
       faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.0)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      faraday-retry (~> 1.0)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-multipart (1.0.3)
       multipart-post (>= 1.2, < 3)
-      ruby2_keywords
-    faraday-net_http (1.0.0)
-    ffi (1.14.2)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    faraday-retry (1.0.3)
+    ffi (1.15.5)
     forwardable-extended (2.6.0)
     gemoji (3.0.1)
-    github-pages (209)
-      github-pages-health-check (= 1.16.1)
+    github-pages (225)
+      github-pages-health-check (= 1.17.9)
       jekyll (= 3.9.0)
       jekyll-avatar (= 0.7.0)
       jekyll-coffeescript (= 1.1.1)
-      jekyll-commonmark-ghpages (= 0.1.6)
+      jekyll-commonmark-ghpages (= 0.2.0)
       jekyll-default-layout (= 0.1.4)
       jekyll-feed (= 0.15.1)
       jekyll-gist (= 1.5.0)
       jekyll-github-metadata (= 2.13.0)
+      jekyll-include-cache (= 0.2.1)
       jekyll-mentions (= 1.6.0)
       jekyll-optional-front-matter (= 0.3.2)
       jekyll-paginate (= 1.1.0)
       jekyll-readme-index (= 0.3.0)
       jekyll-redirect-from (= 0.16.0)
       jekyll-relative-links (= 0.6.1)
-      jekyll-remote-theme (= 0.4.2)
+      jekyll-remote-theme (= 0.4.3)
       jekyll-sass-converter (= 1.5.2)
-      jekyll-seo-tag (= 2.6.1)
+      jekyll-seo-tag (= 2.8.0)
       jekyll-sitemap (= 1.4.0)
       jekyll-swiss (= 1.0.0)
-      jekyll-theme-architect (= 0.1.1)
-      jekyll-theme-cayman (= 0.1.1)
-      jekyll-theme-dinky (= 0.1.1)
-      jekyll-theme-hacker (= 0.1.2)
-      jekyll-theme-leap-day (= 0.1.1)
-      jekyll-theme-merlot (= 0.1.1)
-      jekyll-theme-midnight (= 0.1.1)
-      jekyll-theme-minimal (= 0.1.1)
-      jekyll-theme-modernist (= 0.1.1)
-      jekyll-theme-primer (= 0.5.4)
-      jekyll-theme-slate (= 0.1.1)
-      jekyll-theme-tactile (= 0.1.1)
-      jekyll-theme-time-machine (= 0.1.1)
+      jekyll-theme-architect (= 0.2.0)
+      jekyll-theme-cayman (= 0.2.0)
+      jekyll-theme-dinky (= 0.2.0)
+      jekyll-theme-hacker (= 0.2.0)
+      jekyll-theme-leap-day (= 0.2.0)
+      jekyll-theme-merlot (= 0.2.0)
+      jekyll-theme-midnight (= 0.2.0)
+      jekyll-theme-minimal (= 0.2.0)
+      jekyll-theme-modernist (= 0.2.0)
+      jekyll-theme-primer (= 0.6.0)
+      jekyll-theme-slate (= 0.2.0)
+      jekyll-theme-tactile (= 0.2.0)
+      jekyll-theme-time-machine (= 0.2.0)
       jekyll-titles-from-headings (= 0.5.3)
       jemoji (= 0.12.0)
       kramdown (= 2.3.1)
@@ -75,19 +93,19 @@ GEM
       liquid (= 4.0.3)
       mercenary (~> 0.3)
       minima (= 2.5.1)
-      nokogiri (>= 1.10.4, < 2.0)
-      rouge (= 3.23.0)
+      nokogiri (>= 1.12.5, < 2.0)
+      rouge (= 3.26.0)
       terminal-table (~> 1.4)
-    github-pages-health-check (1.16.1)
+    github-pages-health-check (1.17.9)
       addressable (~> 2.3)
       dnsruby (~> 1.60)
       octokit (~> 4.0)
-      public_suffix (~> 3.0)
+      public_suffix (>= 3.0, < 5.0)
       typhoeus (~> 1.3)
     html-pipeline (2.14.0)
       activesupport (>= 2)
       nokogiri (>= 1.4)
-    http_parser.rb (0.6.0)
+    http_parser.rb (0.8.0)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
     jekyll (3.9.0)
@@ -108,12 +126,12 @@ GEM
     jekyll-coffeescript (1.1.1)
       coffee-script (~> 2.2)
       coffee-script-source (~> 1.11.1)
-    jekyll-commonmark (1.3.1)
-      commonmarker (~> 0.14)
-      jekyll (>= 3.7, < 5.0)
-    jekyll-commonmark-ghpages (0.1.6)
-      commonmarker (~> 0.17.6)
-      jekyll-commonmark (~> 1.2)
+    jekyll-commonmark (1.4.0)
+      commonmarker (~> 0.22)
+    jekyll-commonmark-ghpages (0.2.0)
+      commonmarker (~> 0.23.4)
+      jekyll (~> 3.9.0)
+      jekyll-commonmark (~> 1.4.0)
       rouge (>= 2.0, < 4.0)
     jekyll-default-layout (0.1.4)
       jekyll (~> 3.0)
@@ -124,6 +142,8 @@ GEM
     jekyll-github-metadata (2.13.0)
       jekyll (>= 3.4, < 5.0)
       octokit (~> 4.0, != 4.4.0)
+    jekyll-include-cache (0.2.1)
+      jekyll (>= 3.7, < 5.0)
     jekyll-mentions (1.6.0)
       html-pipeline (~> 2.3)
       jekyll (>= 3.7, < 5.0)
@@ -136,57 +156,57 @@ GEM
       jekyll (>= 3.3, < 5.0)
     jekyll-relative-links (0.6.1)
       jekyll (>= 3.3, < 5.0)
-    jekyll-remote-theme (0.4.2)
+    jekyll-remote-theme (0.4.3)
       addressable (~> 2.0)
       jekyll (>= 3.5, < 5.0)
       jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0)
       rubyzip (>= 1.3.0, < 3.0)
     jekyll-sass-converter (1.5.2)
       sass (~> 3.4)
-    jekyll-seo-tag (2.6.1)
-      jekyll (>= 3.3, < 5.0)
+    jekyll-seo-tag (2.8.0)
+      jekyll (>= 3.8, < 5.0)
     jekyll-sitemap (1.4.0)
       jekyll (>= 3.7, < 5.0)
     jekyll-swiss (1.0.0)
-    jekyll-theme-architect (0.1.1)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-cayman (0.1.1)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-dinky (0.1.1)
-      jekyll (~> 3.5)
-      jekyll-seo-tag (~> 2.0)
-    jekyll-theme-hacker (0.1.2)
+    jekyll-theme-architect (0.2.0)
       jekyll (> 3.5, < 5.0)
       jekyll-seo-tag (~> 2.0)
-    jekyll-theme-leap-day (0.1.1)
-      jekyll (~> 3.5)
+    jekyll-theme-cayman (0.2.0)
+      jekyll (> 3.5, < 5.0)
       jekyll-seo-tag (~> 2.0)
-    jekyll-theme-merlot (0.1.1)
-      jekyll (~> 3.5)
+    jekyll-theme-dinky (0.2.0)
+      jekyll (> 3.5, < 5.0)
       jekyll-seo-tag (~> 2.0)
-    jekyll-theme-midnight (0.1.1)
-      jekyll (~> 3.5)
+    jekyll-theme-hacker (0.2.0)
+      jekyll (> 3.5, < 5.0)
       jekyll-seo-tag (~> 2.0)
-    jekyll-theme-minimal (0.1.1)
-      jekyll (~> 3.5)
+    jekyll-theme-leap-day (0.2.0)
+      jekyll (> 3.5, < 5.0)
       jekyll-seo-tag (~> 2.0)
-    jekyll-theme-modernist (0.1.1)
-      jekyll (~> 3.5)
+    jekyll-theme-merlot (0.2.0)
+      jekyll (> 3.5, < 5.0)
       jekyll-seo-tag (~> 2.0)
-    jekyll-theme-primer (0.5.4)
+    jekyll-theme-midnight (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-minimal (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-modernist (0.2.0)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-primer (0.6.0)
       jekyll (> 3.5, < 5.0)
       jekyll-github-metadata (~> 2.9)
       jekyll-seo-tag (~> 2.0)
-    jekyll-theme-slate (0.1.1)
-      jekyll (~> 3.5)
+    jekyll-theme-slate (0.2.0)
+      jekyll (> 3.5, < 5.0)
       jekyll-seo-tag (~> 2.0)
-    jekyll-theme-tactile (0.1.1)
-      jekyll (~> 3.5)
+    jekyll-theme-tactile (0.2.0)
+      jekyll (> 3.5, < 5.0)
       jekyll-seo-tag (~> 2.0)
-    jekyll-theme-time-machine (0.1.1)
-      jekyll (~> 3.5)
+    jekyll-theme-time-machine (0.2.0)
+      jekyll (> 3.5, < 5.0)
       jekyll-seo-tag (~> 2.0)
     jekyll-titles-from-headings (0.5.3)
       jekyll (>= 3.3, < 5.0)
@@ -201,36 +221,34 @@ GEM
     kramdown-parser-gfm (1.1.0)
       kramdown (~> 2.0)
     liquid (4.0.3)
-    listen (3.4.0)
+    listen (3.7.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     mercenary (0.3.6)
-    mini_portile2 (2.6.1)
+    mini_portile2 (2.8.0)
     minima (2.5.1)
       jekyll (>= 3.5, < 5.0)
       jekyll-feed (~> 0.9)
       jekyll-seo-tag (~> 2.1)
-    minitest (5.14.3)
+    minitest (5.15.0)
     multipart-post (2.1.1)
-    nokogiri (1.12.5)
-      mini_portile2 (~> 2.6.1)
+    nokogiri (1.13.3)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    octokit (4.20.0)
+    octokit (4.22.0)
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
-    public_suffix (3.1.1)
-    racc (1.5.2)
-    rb-fsevent (0.10.4)
+    public_suffix (4.0.6)
+    racc (1.6.0)
+    rb-fsevent (0.11.1)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     rexml (3.2.5)
-    rouge (3.23.0)
-    ruby-enum (0.8.0)
-      i18n
-    ruby2_keywords (0.0.2)
-    rubyzip (2.3.0)
+    rouge (3.26.0)
+    ruby2_keywords (0.0.5)
+    rubyzip (2.3.2)
     safe_yaml (1.0.5)
     sass (3.7.4)
       sass-listen (~> 4.0.0)
@@ -240,7 +258,7 @@ GEM
     sawyer (0.8.2)
       addressable (>= 2.3.5)
       faraday (> 0.8, < 2.0)
-    simpleidn (0.1.1)
+    simpleidn (0.2.1)
       unf (~> 0.1.4)
     terminal-table (1.8.0)
       unicode-display_width (~> 1.1, >= 1.1.1)
@@ -251,16 +269,16 @@ GEM
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.7)
-    unicode-display_width (1.7.0)
+    unf_ext (0.0.8)
+    unicode-display_width (1.8.0)
     webrick (1.7.0)
-    zeitwerk (2.4.2)
+    zeitwerk (2.5.4)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  github-pages (~> 209)
+  github-pages (~> 225)
   webrick (~> 1.7)
 
 BUNDLED WITH

docs/_layouts/home.html

@@ -1,6 +1,15 @@
 <!DOCTYPE html>
 <html>
   {% include head.html %}
+  <div class="socialBanner">
+    <div>
+      Support Ukraine 🇺🇦 
+      <a href="https://opensource.facebook.com/support-ukraine">
+        Help Provide Humanitarian Aid to Ukraine
+      </a>
+      .
+    </div>
+  </div>
   <body>
     {% include nav.html alwayson=true %}
     <div class="navPusher">

docs/css/main.scss

@@ -147,3 +147,13 @@ h5:hover .header-link,
 h6:hover .header-link {
   opacity: 1;
 }
+
+/* Social Banner */
+.socialBanner {
+	font-weight: bold;
+	font-size: 20px;
+	padding: 20px;
+	max-width: 768px;
+	margin: 0 auto;
+	text-align: center;
+  }

env/env_test.cc

@@ -3110,6 +3110,204 @@ TEST_F(EnvTest, CreateCompositeEnv) {
 }
 #endif  // ROCKSDB_LITE
 
+// Forward declaration
+class ReadAsyncFS;
+
+struct MockIOHandle {
+  std::function<void(const FSReadRequest&, void*)> cb;
+  void* cb_arg;
+  bool create_io_error;
+};
+
+// ReadAsyncFS and ReadAsyncRandomAccessFile mocks the FS doing asynchronous
+// reads by creating threads that submit read requests and then calling Poll API
+// to obtain those results.
+class ReadAsyncRandomAccessFile : public FSRandomAccessFileOwnerWrapper {
+ public:
+  ReadAsyncRandomAccessFile(ReadAsyncFS& fs,
+                            std::unique_ptr<FSRandomAccessFile>& file)
+      : FSRandomAccessFileOwnerWrapper(std::move(file)), fs_(fs) {}
+
+  IOStatus ReadAsync(FSReadRequest& req, const IOOptions& opts,
+                     std::function<void(const FSReadRequest&, void*)> cb,
+                     void* cb_arg, void** io_handle, IOHandleDeleter* del_fn,
+                     IODebugContext* dbg) override;
+
+ private:
+  ReadAsyncFS& fs_;
+  std::unique_ptr<FSRandomAccessFile> file_;
+  int counter = 0;
+};
+
+class ReadAsyncFS : public FileSystemWrapper {
+ public:
+  explicit ReadAsyncFS(const std::shared_ptr<FileSystem>& wrapped)
+      : FileSystemWrapper(wrapped) {}
+
+  static const char* kClassName() { return "ReadAsyncFS"; }
+  const char* Name() const override { return kClassName(); }
+
+  IOStatus NewRandomAccessFile(const std::string& fname,
+                               const FileOptions& opts,
+                               std::unique_ptr<FSRandomAccessFile>* result,
+                               IODebugContext* dbg) override {
+    std::unique_ptr<FSRandomAccessFile> file;
+    IOStatus s = target()->NewRandomAccessFile(fname, opts, &file, dbg);
+    EXPECT_OK(s);
+    result->reset(new ReadAsyncRandomAccessFile(*this, file));
+    return s;
+  }
+
+  IOStatus Poll(std::vector<void*>& io_handles,
+                size_t /*min_completions*/) override {
+    // Wait for the threads completion.
+    for (auto& t : workers) {
+      t.join();
+    }
+
+    for (size_t i = 0; i < io_handles.size(); i++) {
+      MockIOHandle* handle = static_cast<MockIOHandle*>(io_handles[i]);
+      if (handle->create_io_error) {
+        FSReadRequest req;
+        req.status = IOStatus::IOError();
+        handle->cb(req, handle->cb_arg);
+      }
+    }
+    return IOStatus::OK();
+  }
+
+  std::vector<std::thread> workers;
+};
+
+IOStatus ReadAsyncRandomAccessFile::ReadAsync(
+    FSReadRequest& req, const IOOptions& opts,
+    std::function<void(const FSReadRequest&, void*)> cb, void* cb_arg,
+    void** io_handle, IOHandleDeleter* del_fn, IODebugContext* dbg) {
+  IOHandleDeleter deletefn = [](void* args) -> void {
+    delete (static_cast<MockIOHandle*>(args));
+    args = nullptr;
+  };
+  *del_fn = deletefn;
+
+  // Allocate and populate io_handle.
+  MockIOHandle* mock_handle = new MockIOHandle();
+  bool create_io_error = false;
+  if (counter % 2) {
+    create_io_error = true;
+  }
+  mock_handle->create_io_error = create_io_error;
+  mock_handle->cb = cb;
+  mock_handle->cb_arg = cb_arg;
+  *io_handle = static_cast<void*>(mock_handle);
+  counter++;
+
+  // Submit read request asynchronously.
+  std::function<void(FSReadRequest)> submit_request =
+      [&opts, cb, cb_arg, io_handle, del_fn, dbg, create_io_error,
+       this](FSReadRequest _req) {
+        if (!create_io_error) {
+          target()->ReadAsync(_req, opts, cb, cb_arg, io_handle, del_fn, dbg);
+        }
+      };
+
+  fs_.workers.emplace_back(submit_request, req);
+  return IOStatus::OK();
+}
+
+class TestAsyncRead : public testing::Test {
+ public:
+  TestAsyncRead() { env_ = Env::Default(); }
+  Env* env_;
+};
+
+// Tests the default implementation of ReadAsync API.
+TEST_F(TestAsyncRead, ReadAsync) {
+  EnvOptions soptions;
+  std::shared_ptr<ReadAsyncFS> fs =
+      std::make_shared<ReadAsyncFS>(env_->GetFileSystem());
+
+  std::string fname = test::PerThreadDBPath(env_, "testfile");
+
+  const size_t kSectorSize = 4096;
+  const size_t kNumSectors = 8;
+
+  // 1. create & write to a file.
+  {
+    std::unique_ptr<FSWritableFile> wfile;
+    ASSERT_OK(
+        fs->NewWritableFile(fname, FileOptions(), &wfile, nullptr /*dbg*/));
+
+    for (size_t i = 0; i < kNumSectors; ++i) {
+      auto data = NewAligned(kSectorSize * 8, static_cast<char>(i + 1));
+      Slice slice(data.get(), kSectorSize);
+      ASSERT_OK(wfile->Append(slice, IOOptions(), nullptr));
+    }
+    ASSERT_OK(wfile->Close(IOOptions(), nullptr));
+  }
+  // 2. Read file
+  {
+    std::unique_ptr<FSRandomAccessFile> file;
+    ASSERT_OK(fs->NewRandomAccessFile(fname, FileOptions(), &file, nullptr));
+
+    IOOptions opts;
+    std::vector<void*> io_handles(kNumSectors);
+    std::vector<FSReadRequest> reqs(kNumSectors);
+    std::vector<std::unique_ptr<char, Deleter>> data;
+    std::vector<size_t> vals;
+    IOHandleDeleter del_fn;
+    uint64_t offset = 0;
+
+    // Initialize read requests
+    for (size_t i = 0; i < kNumSectors; i++) {
+      reqs[i].offset = offset;
+      reqs[i].len = kSectorSize;
+      data.emplace_back(NewAligned(kSectorSize, 0));
+      reqs[i].scratch = data.back().get();
+      vals.push_back(i);
+      offset += kSectorSize;
+    }
+
+    // callback function passed to async read.
+    std::function<void(const FSReadRequest&, void*)> callback =
+        [&](const FSReadRequest& req, void* cb_arg) {
+          assert(cb_arg != nullptr);
+          size_t i = *(reinterpret_cast<size_t*>(cb_arg));
+          reqs[i].offset = req.offset;
+          reqs[i].result = req.result;
+          reqs[i].status = req.status;
+        };
+
+    // Submit asynchronous read requests.
+    for (size_t i = 0; i < kNumSectors; i++) {
+      void* cb_arg = static_cast<void*>(&(vals[i]));
+      ASSERT_OK(file->ReadAsync(reqs[i], opts, callback, cb_arg,
+                                &(io_handles[i]), &del_fn, nullptr));
+    }
+
+    // Poll for the submitted requests.
+    fs->Poll(io_handles, kNumSectors);
+
+    // Check the status of read requests.
+    for (size_t i = 0; i < kNumSectors; i++) {
+      if (i % 2) {
+        ASSERT_EQ(reqs[i].status, IOStatus::IOError());
+      } else {
+        auto buf = NewAligned(kSectorSize * 8, static_cast<char>(i + 1));
+        Slice expected_data(buf.get(), kSectorSize);
+
+        ASSERT_EQ(reqs[i].offset, i * kSectorSize);
+        ASSERT_OK(reqs[i].status);
+        ASSERT_EQ(expected_data.ToString(), reqs[i].result.ToString());
+      }
+    }
+
+    // Delete io_handles.
+    for (size_t i = 0; i < io_handles.size(); i++) {
+      del_fn(io_handles[i]);
+    }
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

env/fs_posix.cc

@@ -753,8 +753,10 @@ class PosixFileSystem : public FileSystem {
                     const IOOptions& /*opts*/,
                     IODebugContext* /*dbg*/) override {
     if (link(src.c_str(), target.c_str()) != 0) {
-      if (errno == EXDEV) {
-        return IOStatus::NotSupported("No cross FS links allowed");
+      if (errno == EXDEV || errno == ENOTSUP) {
+        return IOStatus::NotSupported(errno == EXDEV
+                                          ? "No cross FS links allowed"
+                                          : "Links not supported by FS");
       }
       return IOError("while link file to " + target, src, errno);
     }

env/fs_remap.cc

@@ -110,12 +110,45 @@ IOStatus RemapFileSystem::NewDirectory(const std::string& dir,
                                        const IOOptions& options,
                                        std::unique_ptr<FSDirectory>* result,
                                        IODebugContext* dbg) {
+  // A hassle to remap DirFsyncOptions::renamed_new_name
+  class RemapFSDirectory : public FSDirectoryWrapper {
+   public:
+    RemapFSDirectory(RemapFileSystem* fs, std::unique_ptr<FSDirectory>&& t)
+        : FSDirectoryWrapper(std::move(t)), fs_(fs) {}
+    IOStatus FsyncWithDirOptions(
+        const IOOptions& options, IODebugContext* dbg,
+        const DirFsyncOptions& dir_fsync_options) override {
+      if (dir_fsync_options.renamed_new_name.empty()) {
+        return FSDirectoryWrapper::FsyncWithDirOptions(options, dbg,
+                                                       dir_fsync_options);
+      } else {
+        auto status_and_enc_path =
+            fs_->EncodePath(dir_fsync_options.renamed_new_name);
+        if (status_and_enc_path.first.ok()) {
+          DirFsyncOptions mapped_options = dir_fsync_options;
+          mapped_options.renamed_new_name = status_and_enc_path.second;
+          return FSDirectoryWrapper::FsyncWithDirOptions(options, dbg,
+                                                         mapped_options);
+        } else {
+          return status_and_enc_path.first;
+        }
+      }
+    }
+
+   private:
+    RemapFileSystem* const fs_;
+  };
+
   auto status_and_enc_path = EncodePathWithNewBasename(dir);
   if (!status_and_enc_path.first.ok()) {
     return status_and_enc_path.first;
   }
-  return FileSystemWrapper::NewDirectory(status_and_enc_path.second, options,
-                                         result, dbg);
+  IOStatus ios = FileSystemWrapper::NewDirectory(status_and_enc_path.second,
+                                                 options, result, dbg);
+  if (ios.ok()) {
+    *result = std::make_unique<RemapFSDirectory>(this, std::move(*result));
+  }
+  return ios;
 }
 
 IOStatus RemapFileSystem::FileExists(const std::string& fname,
@@ -293,7 +326,7 @@ IOStatus RemapFileSystem::GetAbsolutePath(const std::string& db_path,
                                           const IOOptions& options,
                                           std::string* output_path,
                                           IODebugContext* dbg) {
-  auto status_and_enc_path = EncodePath(db_path);
+  auto status_and_enc_path = EncodePathWithNewBasename(db_path);
   if (!status_and_enc_path.first.ok()) {
     return status_and_enc_path.first;
   }

examples/rocksdb_option_file_example.ini

@@ -139,5 +139,4 @@
   pin_l0_filter_and_index_blocks_in_cache=false
   pin_top_level_index_and_filter=false
   index_type=kBinarySearch
-  hash_index_allow_collision=true
   flush_block_policy_factory=FlushBlockBySizePolicyFactory

file/file_util.cc

@@ -18,13 +18,13 @@ namespace ROCKSDB_NAMESPACE {
 
 // Utility function to copy a file up to a specified length
 IOStatus CopyFile(FileSystem* fs, const std::string& source,
-                  const std::string& destination, uint64_t size, bool use_fsync,
+                  std::unique_ptr<WritableFileWriter>& dest_writer,
+                  uint64_t size, bool use_fsync,
                   const std::shared_ptr<IOTracer>& io_tracer,
                   const Temperature temperature) {
   FileOptions soptions;
   IOStatus io_s;
   std::unique_ptr<SequentialFileReader> src_reader;
-  std::unique_ptr<WritableFileWriter> dest_writer;
 
   {
     soptions.temperature = temperature;
@@ -33,11 +33,6 @@ IOStatus CopyFile(FileSystem* fs, const std::string& source,
     if (!io_s.ok()) {
       return io_s;
     }
-    std::unique_ptr<FSWritableFile> destfile;
-    io_s = fs->NewWritableFile(destination, soptions, &destfile, nullptr);
-    if (!io_s.ok()) {
-      return io_s;
-    }
 
     if (size == 0) {
       // default argument means copy everything
@@ -48,8 +43,6 @@ IOStatus CopyFile(FileSystem* fs, const std::string& source,
     }
     src_reader.reset(
         new SequentialFileReader(std::move(srcfile), source, io_tracer));
-    dest_writer.reset(
-        new WritableFileWriter(std::move(destfile), destination, soptions));
   }
 
   char buffer[4096];
@@ -72,6 +65,30 @@ IOStatus CopyFile(FileSystem* fs, const std::string& source,
   return dest_writer->Sync(use_fsync);
 }
 
+IOStatus CopyFile(FileSystem* fs, const std::string& source,
+                  const std::string& destination, uint64_t size, bool use_fsync,
+                  const std::shared_ptr<IOTracer>& io_tracer,
+                  const Temperature temperature) {
+  FileOptions options;
+  IOStatus io_s;
+  std::unique_ptr<WritableFileWriter> dest_writer;
+
+  {
+    options.temperature = temperature;
+    std::unique_ptr<FSWritableFile> destfile;
+    io_s = fs->NewWritableFile(destination, options, &destfile, nullptr);
+    if (!io_s.ok()) {
+      return io_s;
+    }
+
+    dest_writer.reset(
+        new WritableFileWriter(std::move(destfile), destination, options));
+  }
+
+  return CopyFile(fs, source, dest_writer, size, use_fsync, io_tracer,
+                  temperature);
+}
+
 // Utility function to create a file with the provided contents
 IOStatus CreateFile(FileSystem* fs, const std::string& destination,
                     const std::string& contents, bool use_fsync) {

file/file_util.h

@@ -19,6 +19,11 @@
 namespace ROCKSDB_NAMESPACE {
 // use_fsync maps to options.use_fsync, which determines the way that
 // the file is synced after copying.
+extern IOStatus CopyFile(FileSystem* fs, const std::string& source,
+                         std::unique_ptr<WritableFileWriter>& dest_writer,
+                         uint64_t size, bool use_fsync,
+                         const std::shared_ptr<IOTracer>& io_tracer,
+                         const Temperature temperature);
 extern IOStatus CopyFile(FileSystem* fs, const std::string& source,
                          const std::string& destination, uint64_t size,
                          bool use_fsync,

file/writable_file_writer.cc

@@ -41,8 +41,8 @@ IOStatus WritableFileWriter::Create(const std::shared_ptr<FileSystem>& fs,
   return io_s;
 }
 
-IOStatus WritableFileWriter::Append(const Slice& data,
-                                    uint32_t crc32c_checksum) {
+IOStatus WritableFileWriter::Append(const Slice& data, uint32_t crc32c_checksum,
+                                    Env::IOPriority op_rate_limiter_priority) {
   const char* src = data.data();
   size_t left = data.size();
   IOStatus s;
@@ -79,7 +79,7 @@ IOStatus WritableFileWriter::Append(const Slice& data,
   // Flush only when buffered I/O
   if (!use_direct_io() && (buf_.Capacity() - buf_.CurrentSize()) < left) {
     if (buf_.CurrentSize() > 0) {
-      s = Flush();
+      s = Flush(op_rate_limiter_priority);
       if (!s.ok()) {
         return s;
       }
@@ -109,7 +109,7 @@ IOStatus WritableFileWriter::Append(const Slice& data,
           src += appended;
 
           if (left > 0) {
-            s = Flush();
+            s = Flush(op_rate_limiter_priority);
             if (!s.ok()) {
               break;
             }
@@ -119,7 +119,7 @@ IOStatus WritableFileWriter::Append(const Slice& data,
     } else {
       assert(buf_.CurrentSize() == 0);
       buffered_data_crc32c_checksum_ = crc32c_checksum;
-      s = WriteBufferedWithChecksum(src, left);
+      s = WriteBufferedWithChecksum(src, left, op_rate_limiter_priority);
     }
   } else {
     // In this case, either we do not need to do the data verification or
@@ -139,7 +139,7 @@ IOStatus WritableFileWriter::Append(const Slice& data,
         src += appended;
 
         if (left > 0) {
-          s = Flush();
+          s = Flush(op_rate_limiter_priority);
           if (!s.ok()) {
             break;
           }
@@ -150,9 +150,9 @@ IOStatus WritableFileWriter::Append(const Slice& data,
       assert(buf_.CurrentSize() == 0);
       if (perform_data_verification_ && buffered_data_with_checksum_) {
         buffered_data_crc32c_checksum_ = crc32c::Value(src, left);
-        s = WriteBufferedWithChecksum(src, left);
+        s = WriteBufferedWithChecksum(src, left, op_rate_limiter_priority);
       } else {
-        s = WriteBuffered(src, left);
+        s = WriteBuffered(src, left, op_rate_limiter_priority);
       }
     }
   }
@@ -164,7 +164,8 @@ IOStatus WritableFileWriter::Append(const Slice& data,
   return s;
 }
 
-IOStatus WritableFileWriter::Pad(const size_t pad_bytes) {
+IOStatus WritableFileWriter::Pad(const size_t pad_bytes,
+                                 Env::IOPriority op_rate_limiter_priority) {
   assert(pad_bytes < kDefaultPageSize);
   size_t left = pad_bytes;
   size_t cap = buf_.Capacity() - buf_.CurrentSize();
@@ -178,7 +179,7 @@ IOStatus WritableFileWriter::Pad(const size_t pad_bytes) {
     buf_.PadWith(append_bytes, 0);
     left -= append_bytes;
     if (left > 0) {
-      IOStatus s = Flush();
+      IOStatus s = Flush(op_rate_limiter_priority);
       if (!s.ok()) {
         return s;
       }
@@ -294,7 +295,7 @@ IOStatus WritableFileWriter::Close() {
 
 // write out the cached data to the OS cache or storage if direct I/O
 // enabled
-IOStatus WritableFileWriter::Flush() {
+IOStatus WritableFileWriter::Flush(Env::IOPriority op_rate_limiter_priority) {
   IOStatus s;
   TEST_KILL_RANDOM_WITH_WEIGHT("WritableFileWriter::Flush:0", REDUCE_ODDS2);
 
@@ -303,17 +304,19 @@ IOStatus WritableFileWriter::Flush() {
 #ifndef ROCKSDB_LITE
       if (pending_sync_) {
         if (perform_data_verification_ && buffered_data_with_checksum_) {
-          s = WriteDirectWithChecksum();
+          s = WriteDirectWithChecksum(op_rate_limiter_priority);
         } else {
-          s = WriteDirect();
+          s = WriteDirect(op_rate_limiter_priority);
         }
       }
 #endif  // !ROCKSDB_LITE
     } else {
       if (perform_data_verification_ && buffered_data_with_checksum_) {
-        s = WriteBufferedWithChecksum(buf_.BufferStart(), buf_.CurrentSize());
+        s = WriteBufferedWithChecksum(buf_.BufferStart(), buf_.CurrentSize(),
+                                      op_rate_limiter_priority);
       } else {
-        s = WriteBuffered(buf_.BufferStart(), buf_.CurrentSize());
+        s = WriteBuffered(buf_.BufferStart(), buf_.CurrentSize(),
+                          op_rate_limiter_priority);
       }
     }
     if (!s.ok()) {
@@ -479,7 +482,8 @@ IOStatus WritableFileWriter::RangeSync(uint64_t offset, uint64_t nbytes) {
 
 // This method writes to disk the specified data and makes use of the rate
 // limiter if available
-IOStatus WritableFileWriter::WriteBuffered(const char* data, size_t size) {
+IOStatus WritableFileWriter::WriteBuffered(
+    const char* data, size_t size, Env::IOPriority op_rate_limiter_priority) {
   IOStatus s;
   assert(!use_direct_io());
   const char* src = data;
@@ -489,10 +493,14 @@ IOStatus WritableFileWriter::WriteBuffered(const char* data, size_t size) {
 
   while (left > 0) {
     size_t allowed;
-    if (rate_limiter_ != nullptr) {
-      allowed = rate_limiter_->RequestToken(
-          left, 0 /* alignment */, writable_file_->GetIOPriority(), stats_,
-          RateLimiter::OpType::kWrite);
+    Env::IOPriority rate_limiter_priority_used =
+        WritableFileWriter::DecideRateLimiterPriority(
+            writable_file_->GetIOPriority(), op_rate_limiter_priority);
+    if (rate_limiter_ != nullptr &&
+        rate_limiter_priority_used != Env::IO_TOTAL) {
+      allowed = rate_limiter_->RequestToken(left, 0 /* alignment */,
+                                            rate_limiter_priority_used, stats_,
+                                            RateLimiter::OpType::kWrite);
     } else {
       allowed = left;
     }
@@ -562,8 +570,8 @@ IOStatus WritableFileWriter::WriteBuffered(const char* data, size_t size) {
   return s;
 }
 
-IOStatus WritableFileWriter::WriteBufferedWithChecksum(const char* data,
-                                                       size_t size) {
+IOStatus WritableFileWriter::WriteBufferedWithChecksum(
+    const char* data, size_t size, Env::IOPriority op_rate_limiter_priority) {
   IOStatus s;
   assert(!use_direct_io());
   assert(perform_data_verification_ && buffered_data_with_checksum_);
@@ -577,12 +585,15 @@ IOStatus WritableFileWriter::WriteBufferedWithChecksum(const char* data,
   // TODO: need to be improved since it sort of defeats the purpose of the rate
   // limiter
   size_t data_size = left;
-  if (rate_limiter_ != nullptr) {
+  Env::IOPriority rate_limiter_priority_used =
+      WritableFileWriter::DecideRateLimiterPriority(
+          writable_file_->GetIOPriority(), op_rate_limiter_priority);
+  if (rate_limiter_ != nullptr && rate_limiter_priority_used != Env::IO_TOTAL) {
     while (data_size > 0) {
       size_t tmp_size;
-      tmp_size = rate_limiter_->RequestToken(
-          data_size, buf_.Alignment(), writable_file_->GetIOPriority(), stats_,
-          RateLimiter::OpType::kWrite);
+      tmp_size = rate_limiter_->RequestToken(data_size, buf_.Alignment(),
+                                             rate_limiter_priority_used, stats_,
+                                             RateLimiter::OpType::kWrite);
       data_size -= tmp_size;
     }
   }
@@ -674,7 +685,8 @@ void WritableFileWriter::Crc32cHandoffChecksumCalculation(const char* data,
 // only write on aligned
 // offsets.
 #ifndef ROCKSDB_LITE
-IOStatus WritableFileWriter::WriteDirect() {
+IOStatus WritableFileWriter::WriteDirect(
+    Env::IOPriority op_rate_limiter_priority) {
   assert(use_direct_io());
   IOStatus s;
   const size_t alignment = buf_.Alignment();
@@ -701,7 +713,11 @@ IOStatus WritableFileWriter::WriteDirect() {
   while (left > 0) {
     // Check how much is allowed
     size_t size;
-    if (rate_limiter_ != nullptr) {
+    Env::IOPriority rate_limiter_priority_used =
+        WritableFileWriter::DecideRateLimiterPriority(
+            writable_file_->GetIOPriority(), op_rate_limiter_priority);
+    if (rate_limiter_ != nullptr &&
+        rate_limiter_priority_used != Env::IO_TOTAL) {
       size = rate_limiter_->RequestToken(left, buf_.Alignment(),
                                          writable_file_->GetIOPriority(),
                                          stats_, RateLimiter::OpType::kWrite);
@@ -762,7 +778,8 @@ IOStatus WritableFileWriter::WriteDirect() {
   return s;
 }
 
-IOStatus WritableFileWriter::WriteDirectWithChecksum() {
+IOStatus WritableFileWriter::WriteDirectWithChecksum(
+    Env::IOPriority op_rate_limiter_priority) {
   assert(use_direct_io());
   assert(perform_data_verification_ && buffered_data_with_checksum_);
   IOStatus s;
@@ -798,7 +815,10 @@ IOStatus WritableFileWriter::WriteDirectWithChecksum() {
   // TODO: need to be improved since it sort of defeats the purpose of the rate
   // limiter
   size_t data_size = left;
-  if (rate_limiter_ != nullptr) {
+  Env::IOPriority rate_limiter_priority_used =
+      WritableFileWriter::DecideRateLimiterPriority(
+          writable_file_->GetIOPriority(), op_rate_limiter_priority);
+  if (rate_limiter_ != nullptr && rate_limiter_priority_used != Env::IO_TOTAL) {
     while (data_size > 0) {
       size_t size;
       size = rate_limiter_->RequestToken(data_size, buf_.Alignment(),
@@ -860,4 +880,18 @@ IOStatus WritableFileWriter::WriteDirectWithChecksum() {
   return s;
 }
 #endif  // !ROCKSDB_LITE
+Env::IOPriority WritableFileWriter::DecideRateLimiterPriority(
+    Env::IOPriority writable_file_io_priority,
+    Env::IOPriority op_rate_limiter_priority) {
+  if (writable_file_io_priority == Env::IO_TOTAL &&
+      op_rate_limiter_priority == Env::IO_TOTAL) {
+    return Env::IO_TOTAL;
+  } else if (writable_file_io_priority == Env::IO_TOTAL) {
+    return op_rate_limiter_priority;
+  } else if (op_rate_limiter_priority == Env::IO_TOTAL) {
+    return writable_file_io_priority;
+  } else {
+    return op_rate_limiter_priority;
+  }
+}
 }  // namespace ROCKSDB_NAMESPACE

file/writable_file_writer.h

@@ -234,11 +234,13 @@ class WritableFileWriter {
 
   // When this Append API is called, if the crc32c_checksum is not provided, we
   // will calculate the checksum internally.
-  IOStatus Append(const Slice& data, uint32_t crc32c_checksum = 0);
+  IOStatus Append(const Slice& data, uint32_t crc32c_checksum = 0,
+                  Env::IOPriority op_rate_limiter_priority = Env::IO_TOTAL);
 
-  IOStatus Pad(const size_t pad_bytes);
+  IOStatus Pad(const size_t pad_bytes,
+               Env::IOPriority op_rate_limiter_priority = Env::IO_TOTAL);
 
-  IOStatus Flush();
+  IOStatus Flush(Env::IOPriority op_rate_limiter_priority = Env::IO_TOTAL);
 
   IOStatus Close();
 
@@ -271,15 +273,21 @@ class WritableFileWriter {
   const char* GetFileChecksumFuncName() const;
 
  private:
+  static Env::IOPriority DecideRateLimiterPriority(
+      Env::IOPriority writable_file_io_priority,
+      Env::IOPriority op_rate_limiter_priority);
+
   // Used when os buffering is OFF and we are writing
   // DMA such as in Direct I/O mode
 #ifndef ROCKSDB_LITE
-  IOStatus WriteDirect();
-  IOStatus WriteDirectWithChecksum();
+  IOStatus WriteDirect(Env::IOPriority op_rate_limiter_priority);
+  IOStatus WriteDirectWithChecksum(Env::IOPriority op_rate_limiter_priority);
 #endif  // !ROCKSDB_LITE
-  // Normal write
-  IOStatus WriteBuffered(const char* data, size_t size);
-  IOStatus WriteBufferedWithChecksum(const char* data, size_t size);
+  // Normal write.
+  IOStatus WriteBuffered(const char* data, size_t size,
+                         Env::IOPriority op_rate_limiter_priority);
+  IOStatus WriteBufferedWithChecksum(const char* data, size_t size,
+                                     Env::IOPriority op_rate_limiter_priority);
   IOStatus RangeSync(uint64_t offset, uint64_t nbytes);
   IOStatus SyncInternal(bool use_fsync);
 };

include/rocksdb/advanced_options.h

@@ -471,6 +471,14 @@ struct AdvancedColumnFamilyOptions {
   // according to compression_per_level[1], L3 using compression_per_level[2]
   // and L4 using compression_per_level[3]. Compaction for each level can
   // change when data grows.
+  //
+  // NOTE: if the vector size is smaller than the level number, the undefined
+  // lower level uses the last option in the vector, for example, for 3 level
+  // LSM tree the following settings are the same:
+  // {kNoCompression, kSnappyCompression}
+  // {kNoCompression, kSnappyCompression, kSnappyCompression}
+  //
+  // Dynamically changeable through SetOptions() API
   std::vector<CompressionType> compression_per_level;
 
   // Number of levels for this database

include/rocksdb/c.h

@@ -872,9 +872,9 @@ extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_data_block_index
     rocksdb_block_based_table_options_t*, int);  // uses one of the above enums
 extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_data_block_hash_ratio(
     rocksdb_block_based_table_options_t* options, double v);
-extern ROCKSDB_LIBRARY_API void
-rocksdb_block_based_options_set_hash_index_allow_collision(
-    rocksdb_block_based_table_options_t*, unsigned char);
+// rocksdb_block_based_options_set_hash_index_allow_collision()
+// is removed since BlockBasedTableOptions.hash_index_allow_collision()
+// is removed
 extern ROCKSDB_LIBRARY_API void
 rocksdb_block_based_options_set_cache_index_and_filter_blocks(
     rocksdb_block_based_table_options_t*, unsigned char);

include/rocksdb/cache.h

@@ -27,6 +27,7 @@
 #include <memory>
 #include <string>
 
+#include "rocksdb/compression_type.h"
 #include "rocksdb/memory_allocator.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/statistics.h"
@@ -127,6 +128,53 @@ extern std::shared_ptr<Cache> NewLRUCache(
 
 extern std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts);
 
+// EXPERIMENTAL
+// Options structure for configuring a SecondaryCache instance based on
+// LRUCache. The LRUCacheOptions.secondary_cache is not used and
+// should not be set.
+struct LRUSecondaryCacheOptions : LRUCacheOptions {
+  // The compression method (if any) that is used to compress data.
+  CompressionType compression_type = CompressionType::kLZ4Compression;
+
+  // compress_format_version can have two values:
+  // compress_format_version == 1 -- decompressed size is not included in the
+  // block header.
+  // compress_format_version == 2 -- decompressed size is included in the block
+  // header in varint32 format.
+  uint32_t compress_format_version = 2;
+
+  LRUSecondaryCacheOptions() {}
+  LRUSecondaryCacheOptions(
+      size_t _capacity, int _num_shard_bits, bool _strict_capacity_limit,
+      double _high_pri_pool_ratio,
+      std::shared_ptr<MemoryAllocator> _memory_allocator = nullptr,
+      bool _use_adaptive_mutex = kDefaultToAdaptiveMutex,
+      CacheMetadataChargePolicy _metadata_charge_policy =
+          kDefaultCacheMetadataChargePolicy,
+      CompressionType _compression_type = CompressionType::kLZ4Compression,
+      uint32_t _compress_format_version = 2)
+      : LRUCacheOptions(_capacity, _num_shard_bits, _strict_capacity_limit,
+                        _high_pri_pool_ratio, std::move(_memory_allocator),
+                        _use_adaptive_mutex, _metadata_charge_policy),
+        compression_type(_compression_type),
+        compress_format_version(_compress_format_version) {}
+};
+
+// EXPERIMENTAL
+// Create a new Secondary Cache that is implemented on top of LRUCache.
+extern std::shared_ptr<SecondaryCache> NewLRUSecondaryCache(
+    size_t capacity, int num_shard_bits = -1,
+    bool strict_capacity_limit = false, double high_pri_pool_ratio = 0.5,
+    std::shared_ptr<MemoryAllocator> memory_allocator = nullptr,
+    bool use_adaptive_mutex = kDefaultToAdaptiveMutex,
+    CacheMetadataChargePolicy metadata_charge_policy =
+        kDefaultCacheMetadataChargePolicy,
+    CompressionType compression_type = CompressionType::kLZ4Compression,
+    uint32_t compress_format_version = 2);
+
+extern std::shared_ptr<SecondaryCache> NewLRUSecondaryCache(
+    const LRUSecondaryCacheOptions& opts);
+
 // Similar to NewLRUCache, but create a cache based on CLOCK algorithm with
 // better concurrent performance in some cases. See util/clock_cache.cc for
 // more detail.

include/rocksdb/env.h

@@ -959,8 +959,16 @@ class WritableFile {
   // Use the returned alignment value to allocate
   // aligned buffer for Direct I/O
   virtual size_t GetRequiredBufferAlignment() const { return kDefaultPageSize; }
+
   /*
-   * Change the priority in rate limiter if rate limiting is enabled.
+   * If rate limiting is enabled, change the file-granularity priority used in
+   * rate-limiting writes.
+   *
+   * In the presence of finer-granularity priority such as
+   * `WriteOptions::rate_limiter_priority`, this file-granularity priority may
+   * be overridden by a non-Env::IO_TOTAL finer-granularity priority and used as
+   * a fallback for Env::IO_TOTAL finer-granularity priority.
+   *
    * If rate limiting is not enabled, this call has no effect.
    */
   virtual void SetIOPriority(Env::IOPriority pri) { io_priority_ = pri; }

include/rocksdb/file_system.h

@@ -221,10 +221,10 @@ struct IODebugContext {
   }
 };
 
-// IOHandle is used by underlying file system to store any information it needs
-// during Async Read requests.
+// A function pointer type for custom destruction of void pointer passed to
+// ReadAsync API. RocksDB/caller is responsible for deleting the void pointer
+// allocated by FS in ReadAsync API.
 using IOHandleDeleter = std::function<void(void*)>;
-using IOHandle = std::unique_ptr<void, IOHandleDeleter>;
 
 // The FileSystem, FSSequentialFile, FSRandomAccessFile, FSWritableFile,
 // FSRandomRWFileclass, and FSDIrectory classes define the interface between
@@ -647,13 +647,15 @@ class FileSystem : public Customizable {
 
   // EXPERIMENTAL
   // Poll for completion of read IO requests. The Poll() method should call the
-  // callback functions to indicate completion of read requests. If Poll is not
-  // supported it means callee should be informed of IO completions via the
-  // callback on another thread.
+  // callback functions to indicate completion of read requests.
+  // Underlying FS is required to support Poll API. Poll implementation should
+  // ensure that the callback gets called at IO completion, and return only
+  // after the callback has been called.
+  //
   //
   // Default implementation is to return IOStatus::OK.
 
-  virtual IOStatus Poll(std::vector<IOHandle*>& /*io_handles*/,
+  virtual IOStatus Poll(std::vector<void*>& /*io_handles*/,
                         size_t /*min_completions*/) {
     return IOStatus::OK();
   }
@@ -865,9 +867,13 @@ class FSRandomAccessFile {
   // cb_arg should be used by the callback to track the original request
   // submitted.
   //
-  // This API should also populate IOHandle which should be used by
+  // This API should also populate io_handle which should be used by
   // underlying FileSystem to store the context in order to distinguish the read
-  // requests at their side.
+  // requests at their side and provide the custom deletion function in del_fn.
+  // RocksDB guarantees that the del_fn for io_handle will be called after
+  // receiving the callback. Furthermore, RocksDB guarantees that if it calls
+  // the Poll API for this io_handle, del_fn will be called after the Poll
+  // returns. RocksDB is responsible for managing the lifetime of io_handle.
   //
   // req contains the request offset and size passed as input parameter of read
   // request and result and status fields are output parameter set by underlying
@@ -877,7 +883,7 @@ class FSRandomAccessFile {
   virtual IOStatus ReadAsync(
       FSReadRequest& req, const IOOptions& opts,
       std::function<void(const FSReadRequest&, void*)> cb, void* cb_arg,
-      IOHandle* /*io_handle*/, IODebugContext* dbg) {
+      void** /*io_handle*/, IOHandleDeleter* /*del_fn*/, IODebugContext* dbg) {
     req.status =
         Read(req.offset, req.len, opts, &(req.result), req.scratch, dbg);
     cb(req, cb_arg);
@@ -1025,6 +1031,17 @@ class FSWritableFile {
     write_hint_ = hint;
   }
 
+  /*
+   * If rate limiting is enabled, change the file-granularity priority used in
+   * rate-limiting writes.
+   *
+   * In the presence of finer-granularity priority such as
+   * `WriteOptions::rate_limiter_priority`, this file-granularity priority may
+   * be overridden by a non-Env::IO_TOTAL finer-granularity priority and used as
+   * a fallback for Env::IO_TOTAL finer-granularity priority.
+   *
+   * If rate limiting is not enabled, this call has no effect.
+   */
   virtual void SetIOPriority(Env::IOPriority pri) { io_priority_ = pri; }
 
   virtual Env::IOPriority GetIOPriority() { return io_priority_; }
@@ -1470,7 +1487,7 @@ class FileSystemWrapper : public FileSystem {
                                const std::string& header) const override;
 #endif  // ROCKSDB_LITE
 
-  virtual IOStatus Poll(std::vector<IOHandle*>& io_handles,
+  virtual IOStatus Poll(std::vector<void*>& io_handles,
                         size_t min_completions) override {
     return target_->Poll(io_handles, min_completions);
   }
@@ -1557,9 +1574,9 @@ class FSRandomAccessFileWrapper : public FSRandomAccessFile {
   }
   IOStatus ReadAsync(FSReadRequest& req, const IOOptions& opts,
                      std::function<void(const FSReadRequest&, void*)> cb,
-                     void* cb_arg, IOHandle* io_handle,
+                     void* cb_arg, void** io_handle, IOHandleDeleter* del_fn,
                      IODebugContext* dbg) override {
-    return target()->ReadAsync(req, opts, cb, cb_arg, io_handle, dbg);
+    return target()->ReadAsync(req, opts, cb, cb_arg, io_handle, del_fn, dbg);
   }
   Temperature GetTemperature() const override {
     return target_->GetTemperature();

include/rocksdb/io_status.h

@@ -28,7 +28,7 @@ class IOStatus : public Status {
   using Code = Status::Code;
   using SubCode = Status::SubCode;
 
-  enum IOErrorScope {
+  enum IOErrorScope : unsigned char {
     kIOErrorScopeFileSystem,
     kIOErrorScopeFile,
     kIOErrorScopeRange,
@@ -57,11 +57,13 @@ class IOStatus : public Status {
 
   void SetRetryable(bool retryable) { retryable_ = retryable; }
   void SetDataLoss(bool data_loss) { data_loss_ = data_loss; }
-  void SetScope(IOErrorScope scope) { scope_ = scope; }
+  void SetScope(IOErrorScope scope) {
+    scope_ = static_cast<unsigned char>(scope);
+  }
 
   bool GetRetryable() const { return retryable_; }
   bool GetDataLoss() const { return data_loss_; }
-  IOErrorScope GetScope() const { return scope_; }
+  IOErrorScope GetScope() const { return static_cast<IOErrorScope>(scope_); }
 
   // Return a success status.
   static IOStatus OK() { return IOStatus(); }
@@ -137,15 +139,9 @@ class IOStatus : public Status {
 
  private:
   friend IOStatus status_to_io_status(Status&&);
-  bool retryable_;
-  bool data_loss_;
-  IOErrorScope scope_;
 
   explicit IOStatus(Code _code, SubCode _subcode = kNone)
-      : Status(_code, _subcode),
-        retryable_(false),
-        data_loss_(false),
-        scope_(kIOErrorScopeFileSystem) {}
+      : Status(_code, _subcode, false, false, kIOErrorScopeFileSystem) {}
 
   IOStatus(Code _code, SubCode _subcode, const Slice& msg, const Slice& msg2);
   IOStatus(Code _code, const Slice& msg, const Slice& msg2)
@@ -154,10 +150,7 @@ class IOStatus : public Status {
 
 inline IOStatus::IOStatus(Code _code, SubCode _subcode, const Slice& msg,
                           const Slice& msg2)
-    : Status(_code, _subcode),
-      retryable_(false),
-      data_loss_(false),
-      scope_(kIOErrorScopeFileSystem) {
+    : Status(_code, _subcode, false, false, kIOErrorScopeFileSystem) {
   assert(code_ != kOk);
   assert(subcode_ != kMaxSubCode);
   const size_t len1 = msg.size();
@@ -249,18 +242,10 @@ inline bool IOStatus::operator!=(const IOStatus& rhs) const {
 }
 
 inline IOStatus status_to_io_status(Status&& status) {
-  if (status.ok()) {
-    // Fast path
-    return IOStatus::OK();
-  } else {
-    const char* state = status.getState();
-    if (state) {
-      return IOStatus(status.code(), status.subcode(),
-                      Slice(state, strlen(status.getState()) + 1), Slice());
-    } else {
-      return IOStatus(status.code(), status.subcode());
-    }
-  }
+  IOStatus io_s;
+  Status& s = io_s;
+  s = std::move(status);
+  return io_s;
 }
 
 }  // namespace ROCKSDB_NAMESPACE

include/rocksdb/options.h

@@ -1659,13 +1659,29 @@ struct WriteOptions {
   // Default: false
   bool memtable_insert_hint_per_batch;
 
+  // For writes associated with this option, charge the internal rate
+  // limiter (see `DBOptions::rate_limiter`) at the specified priority. The
+  // special value `Env::IO_TOTAL` disables charging the rate limiter.
+  //
+  // Currently the support covers automatic WAL flushes, which happen during
+  // live updates (`Put()`, `Write()`, `Delete()`, etc.)
+  // when `WriteOptions::disableWAL == false`
+  // and `DBOptions::manual_wal_flush == false`.
+  //
+  // Only `Env::IO_USER` and `Env::IO_TOTAL` are allowed
+  // due to implementation constraints.
+  //
+  // Default: `Env::IO_TOTAL`
+  Env::IOPriority rate_limiter_priority;
+
   WriteOptions()
       : sync(false),
         disableWAL(false),
         ignore_missing_column_families(false),
         no_slowdown(false),
         low_pri(false),
-        memtable_insert_hint_per_batch(false) {}
+        memtable_insert_hint_per_batch(false),
+        rate_limiter_priority(Env::IO_TOTAL) {}
 };
 
 // Options that control flush operations

include/rocksdb/status.h

@@ -35,7 +35,14 @@ namespace ROCKSDB_NAMESPACE {
 class Status {
  public:
   // Create a success status.
-  Status() : code_(kOk), subcode_(kNone), sev_(kNoError), state_(nullptr) {}
+  Status()
+      : code_(kOk),
+        subcode_(kNone),
+        sev_(kNoError),
+        retryable_(false),
+        data_loss_(false),
+        scope_(0),
+        state_(nullptr) {}
   ~Status() {
 #ifdef ROCKSDB_ASSERT_STATUS_CHECKED
     if (!checked_) {
@@ -454,6 +461,9 @@ class Status {
   Code code_;
   SubCode subcode_;
   Severity sev_;
+  bool retryable_;
+  bool data_loss_;
+  unsigned char scope_;
   // A nullptr state_ (which is at least the case for OK) means the extra
   // message is empty.
   std::unique_ptr<const char[]> state_;
@@ -462,7 +472,21 @@ class Status {
 #endif  // ROCKSDB_ASSERT_STATUS_CHECKED
 
   explicit Status(Code _code, SubCode _subcode = kNone)
-      : code_(_code), subcode_(_subcode), sev_(kNoError) {}
+      : code_(_code),
+        subcode_(_subcode),
+        sev_(kNoError),
+        retryable_(false),
+        data_loss_(false),
+        scope_(0) {}
+
+  explicit Status(Code _code, SubCode _subcode, bool retryable, bool data_loss,
+                  unsigned char scope)
+      : code_(_code),
+        subcode_(_subcode),
+        sev_(kNoError),
+        retryable_(retryable),
+        data_loss_(data_loss),
+        scope_(scope) {}
 
   Status(Code _code, SubCode _subcode, const Slice& msg, const Slice& msg2,
          Severity sev = kNoError);
@@ -479,12 +503,22 @@ class Status {
 };
 
 inline Status::Status(const Status& s)
-    : code_(s.code_), subcode_(s.subcode_), sev_(s.sev_) {
+    : code_(s.code_),
+      subcode_(s.subcode_),
+      sev_(s.sev_),
+      retryable_(s.retryable_),
+      data_loss_(s.data_loss_),
+      scope_(s.scope_) {
   s.MarkChecked();
   state_ = (s.state_ == nullptr) ? nullptr : CopyState(s.state_.get());
 }
 inline Status::Status(const Status& s, Severity sev)
-    : code_(s.code_), subcode_(s.subcode_), sev_(sev) {
+    : code_(s.code_),
+      subcode_(s.subcode_),
+      sev_(sev),
+      retryable_(s.retryable_),
+      data_loss_(s.data_loss_),
+      scope_(s.scope_) {
   s.MarkChecked();
   state_ = (s.state_ == nullptr) ? nullptr : CopyState(s.state_.get());
 }
@@ -495,6 +529,9 @@ inline Status& Status::operator=(const Status& s) {
     code_ = s.code_;
     subcode_ = s.subcode_;
     sev_ = s.sev_;
+    retryable_ = s.retryable_;
+    data_loss_ = s.data_loss_;
+    scope_ = s.scope_;
     state_ = (s.state_ == nullptr) ? nullptr : CopyState(s.state_.get());
   }
   return *this;
@@ -523,6 +560,12 @@ inline Status& Status::operator=(Status&& s)
     s.subcode_ = kNone;
     sev_ = std::move(s.sev_);
     s.sev_ = kNoError;
+    retryable_ = std::move(s.retryable_);
+    s.retryable_ = false;
+    data_loss_ = std::move(s.data_loss_);
+    s.data_loss_ = false;
+    scope_ = std::move(s.scope_);
+    s.scope_ = 0;
     state_ = std::move(s.state_);
   }
   return *this;

include/rocksdb/table.h

@@ -227,9 +227,8 @@ struct BlockBasedTableOptions {
   // kDataBlockBinaryAndHash.
   double data_block_hash_table_util_ratio = 0.75;
 
-  // This option is now deprecated. No matter what value it is set to,
-  // it will behave as if hash_index_allow_collision=true.
-  bool hash_index_allow_collision = true;
+  // Option hash_index_allow_collision is now deleted.
+  // It will behave as if hash_index_allow_collision=true.
 
   // Use the specified checksum type. Newly created table files will be
   // protected with this checksum type. Old table files will still be readable,
@@ -372,6 +371,10 @@ struct BlockBasedTableOptions {
   // useful in detecting software bugs or CPU+memory malfunction.
   // Turning on this feature increases filter construction time by 30%.
   //
+  // This parameter can be changed dynamically by
+  // DB::SetOptions({{"block_based_table_factory",
+  //                  "{detect_filter_construct_corruption=true;}"}});
+  //
   // TODO: optimize this performance
   bool detect_filter_construct_corruption = false;
 

include/rocksdb/utilities/ldb_cmd.h

@@ -61,6 +61,14 @@ class LDBCommand {
   static const std::string ARG_CREATE_IF_MISSING;
   static const std::string ARG_NO_VALUE;
   static const std::string ARG_DISABLE_CONSISTENCY_CHECKS;
+  static const std::string ARG_ENABLE_BLOB_FILES;
+  static const std::string ARG_MIN_BLOB_SIZE;
+  static const std::string ARG_BLOB_FILE_SIZE;
+  static const std::string ARG_BLOB_COMPRESSION_TYPE;
+  static const std::string ARG_ENABLE_BLOB_GARBAGE_COLLECTION;
+  static const std::string ARG_BLOB_GARBAGE_COLLECTION_AGE_CUTOFF;
+  static const std::string ARG_BLOB_GARBAGE_COLLECTION_FORCE_THRESHOLD;
+  static const std::string ARG_BLOB_COMPACTION_READAHEAD_SIZE;
 
   struct ParsedParams {
     std::string cmd;
@@ -173,6 +181,10 @@ class LDBCommand {
   // The value passed to options.force_consistency_checks.
   bool force_consistency_checks_;
 
+  bool enable_blob_files_;
+
+  bool enable_blob_garbage_collection_;
+
   bool create_if_missing_;
 
   /**
@@ -233,9 +245,18 @@ class LDBCommand {
                       const std::string& option, int& value,
                       LDBCommandExecuteResult& exec_state);
 
+  bool ParseDoubleOption(const std::map<std::string, std::string>& options,
+                         const std::string& option, double& value,
+                         LDBCommandExecuteResult& exec_state);
+
   bool ParseStringOption(const std::map<std::string, std::string>& options,
                          const std::string& option, std::string* value);
 
+  bool ParseCompressionTypeOption(
+      const std::map<std::string, std::string>& options,
+      const std::string& option, CompressionType& value,
+      LDBCommandExecuteResult& exec_state);
+
   /**
    * Returns the value of the specified option as a boolean.
    * default_val is used if the option is not found in options.

include/rocksdb/utilities/write_batch_with_index.h

@@ -275,6 +275,7 @@ class WriteBatchWithIndex : public WriteBatchBase {
   friend class WritePreparedTxn;
   friend class WriteUnpreparedTxn;
   friend class WriteBatchWithIndex_SubBatchCnt_Test;
+  friend class WriteBatchWithIndexInternal;
   // Returns the number of sub-batches inside the write batch. A sub-batch
   // starts right before inserting a key that is a duplicate of a key in the
   // last sub-batch.

include/rocksdb/version.h

@@ -9,8 +9,8 @@
 
 #include "rocksdb/rocksdb_namespace.h"
 
-#define ROCKSDB_MAJOR 6
-#define ROCKSDB_MINOR 29
+#define ROCKSDB_MAJOR 7
+#define ROCKSDB_MINOR 0
 #define ROCKSDB_PATCH 0
 
 // Do not use these. We made the mistake of declaring macros starting with

include/rocksdb/write_batch.h

@@ -444,6 +444,10 @@ class WriteBatch : public WriteBatchBase {
   // timestamps to desired values.
   bool needs_in_place_update_ts_ = false;
 
+  // True if the write batch contains at least one key from a column family
+  // that enables user-defined timestamp.
+  bool has_key_with_ts_ = false;
+
  protected:
   std::string rep_;  // See comment in write_batch.cc for the format of rep_
 };

java/rocksjni/backupablejni.cc

@@ -16,6 +16,7 @@
 
 #include "include/org_rocksdb_BackupEngineOptions.h"
 #include "rocksdb/utilities/backup_engine.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 
 ///////////////////////////////////////////////////////////////////////////
@@ -35,7 +36,7 @@ jlong Java_org_rocksdb_BackupEngineOptions_newBackupEngineOptions(
   }
   auto* bopt = new ROCKSDB_NAMESPACE::BackupEngineOptions(cpath);
   env->ReleaseStringUTFChars(jpath, cpath);
-  return reinterpret_cast<jlong>(bopt);
+  return GET_CPLUSPLUS_POINTER(bopt);
 }
 
 /*

java/rocksjni/backupenginejni.cc

@@ -12,6 +12,7 @@
 
 #include "include/org_rocksdb_BackupEngine.h"
 #include "rocksdb/utilities/backup_engine.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 
 /*
@@ -31,7 +32,7 @@ jlong Java_org_rocksdb_BackupEngine_open(JNIEnv* env, jclass /*jcls*/,
       rocks_env, *backup_engine_options, &backup_engine);
 
   if (status.ok()) {
-    return reinterpret_cast<jlong>(backup_engine);
+    return GET_CPLUSPLUS_POINTER(backup_engine);
   } else {
     ROCKSDB_NAMESPACE::RocksDBExceptionJni::ThrowNew(env, status);
     return 0;

java/rocksjni/cassandra_compactionfilterjni.cc

@@ -6,6 +6,7 @@
 #include <jni.h>
 
 #include "include/org_rocksdb_CassandraCompactionFilter.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "utilities/cassandra/cassandra_compaction_filter.h"
 
 /*
@@ -20,5 +21,5 @@ jlong Java_org_rocksdb_CassandraCompactionFilter_createNewCassandraCompactionFil
       new ROCKSDB_NAMESPACE::cassandra::CassandraCompactionFilter(
           purge_ttl_on_expiration, gc_grace_period_in_seconds);
   // set the native handle to our native compaction filter
-  return reinterpret_cast<jlong>(compaction_filter);
+  return GET_CPLUSPLUS_POINTER(compaction_filter);
 }

java/rocksjni/cassandra_value_operator.cc

@@ -6,6 +6,7 @@
 #include <jni.h>
 #include <stdio.h>
 #include <stdlib.h>
+
 #include <memory>
 #include <string>
 
@@ -17,6 +18,7 @@
 #include "rocksdb/slice_transform.h"
 #include "rocksdb/statistics.h"
 #include "rocksdb/table.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 #include "utilities/cassandra/merge_operator.h"
 
@@ -31,7 +33,7 @@ jlong Java_org_rocksdb_CassandraValueMergeOperator_newSharedCassandraValueMergeO
   auto* op = new std::shared_ptr<ROCKSDB_NAMESPACE::MergeOperator>(
       new ROCKSDB_NAMESPACE::cassandra::CassandraValueMergeOperator(
           gcGracePeriodInSeconds, operands_limit));
-  return reinterpret_cast<jlong>(op);
+  return GET_CPLUSPLUS_POINTER(op);
 }
 
 /*

java/rocksjni/checkpoint.cc

@@ -6,14 +6,17 @@
 // This file implements the "bridge" between Java and C++ and enables
 // calling c++ ROCKSDB_NAMESPACE::Checkpoint methods from Java side.
 
+#include "rocksdb/utilities/checkpoint.h"
+
 #include <jni.h>
 #include <stdio.h>
 #include <stdlib.h>
+
 #include <string>
 
 #include "include/org_rocksdb_Checkpoint.h"
 #include "rocksdb/db.h"
-#include "rocksdb/utilities/checkpoint.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 /*
  * Class:     org_rocksdb_Checkpoint
@@ -26,7 +29,7 @@ jlong Java_org_rocksdb_Checkpoint_newCheckpoint(JNIEnv* /*env*/,
   auto* db = reinterpret_cast<ROCKSDB_NAMESPACE::DB*>(jdb_handle);
   ROCKSDB_NAMESPACE::Checkpoint* checkpoint;
   ROCKSDB_NAMESPACE::Checkpoint::Create(db, &checkpoint);
-  return reinterpret_cast<jlong>(checkpoint);
+  return GET_CPLUSPLUS_POINTER(checkpoint);
 }
 
 /*

java/rocksjni/clock_cache.cc

@@ -6,10 +6,12 @@
 // This file implements the "bridge" between Java and C++ for
 // ROCKSDB_NAMESPACE::ClockCache.
 
+#include "cache/clock_cache.h"
+
 #include <jni.h>
 
-#include "cache/clock_cache.h"
 #include "include/org_rocksdb_ClockCache.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 
 /*
  * Class:     org_rocksdb_ClockCache
@@ -23,7 +25,7 @@ jlong Java_org_rocksdb_ClockCache_newClockCache(
       ROCKSDB_NAMESPACE::NewClockCache(
           static_cast<size_t>(jcapacity), static_cast<int>(jnum_shard_bits),
           static_cast<bool>(jstrict_capacity_limit)));
-  return reinterpret_cast<jlong>(sptr_clock_cache);
+  return GET_CPLUSPLUS_POINTER(sptr_clock_cache);
 }
 
 /*

java/rocksjni/compact_range_options.cc

@@ -10,6 +10,7 @@
 
 #include "include/org_rocksdb_CompactRangeOptions.h"
 #include "rocksdb/options.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 
 /*
@@ -20,7 +21,7 @@
 jlong Java_org_rocksdb_CompactRangeOptions_newCompactRangeOptions(
     JNIEnv* /*env*/, jclass /*jclazz*/) {
   auto* options = new ROCKSDB_NAMESPACE::CompactRangeOptions();
-  return reinterpret_cast<jlong>(options);
+  return GET_CPLUSPLUS_POINTER(options);
 }
 
 

java/rocksjni/compaction_filter_factory.cc

@@ -7,10 +7,12 @@
 // ROCKSDB_NAMESPACE::CompactionFilterFactory.
 
 #include <jni.h>
+
 #include <memory>
 
 #include "include/org_rocksdb_AbstractCompactionFilterFactory.h"
 #include "rocksjni/compaction_filter_factory_jnicallback.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 
 /*
  * Class:     org_rocksdb_AbstractCompactionFilterFactory
@@ -23,7 +25,7 @@ jlong Java_org_rocksdb_AbstractCompactionFilterFactory_createNewCompactionFilter
       new ROCKSDB_NAMESPACE::CompactionFilterFactoryJniCallback(env, jobj);
   auto* ptr_sptr_cff = new std::shared_ptr<
       ROCKSDB_NAMESPACE::CompactionFilterFactoryJniCallback>(cff);
-  return reinterpret_cast<jlong>(ptr_sptr_cff);
+  return GET_CPLUSPLUS_POINTER(ptr_sptr_cff);
 }
 
 /*

java/rocksjni/compaction_job_info.cc

@@ -10,6 +10,7 @@
 
 #include "include/org_rocksdb_CompactionJobInfo.h"
 #include "rocksdb/listener.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 
 /*
@@ -20,7 +21,7 @@
 jlong Java_org_rocksdb_CompactionJobInfo_newCompactionJobInfo(
     JNIEnv*, jclass) {
   auto* compact_job_info = new ROCKSDB_NAMESPACE::CompactionJobInfo();
-  return reinterpret_cast<jlong>(compact_job_info);
+  return GET_CPLUSPLUS_POINTER(compact_job_info);
 }
 
 /*
@@ -227,5 +228,5 @@ jlong Java_org_rocksdb_CompactionJobInfo_stats(
       reinterpret_cast<ROCKSDB_NAMESPACE::CompactionJobInfo*>(jhandle);
   auto* stats = new ROCKSDB_NAMESPACE::CompactionJobStats();
   stats->Add(compact_job_info->stats);
-  return reinterpret_cast<jlong>(stats);
+  return GET_CPLUSPLUS_POINTER(stats);
 }

java/rocksjni/compaction_job_stats.cc

@@ -6,10 +6,12 @@
 // This file implements the "bridge" between Java and C++ for
 // ROCKSDB_NAMESPACE::CompactionJobStats.
 
+#include "rocksdb/compaction_job_stats.h"
+
 #include <jni.h>
 
 #include "include/org_rocksdb_CompactionJobStats.h"
-#include "rocksdb/compaction_job_stats.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 
 /*
@@ -20,7 +22,7 @@
 jlong Java_org_rocksdb_CompactionJobStats_newCompactionJobStats(
     JNIEnv*, jclass) {
   auto* compact_job_stats = new ROCKSDB_NAMESPACE::CompactionJobStats();
-  return reinterpret_cast<jlong>(compact_job_stats);
+  return GET_CPLUSPLUS_POINTER(compact_job_stats);
 }
 
 /*

java/rocksjni/compaction_options.cc

@@ -10,9 +10,9 @@
 
 #include "include/org_rocksdb_CompactionOptions.h"
 #include "rocksdb/options.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 
-
 /*
  * Class:     org_rocksdb_CompactionOptions
  * Method:    newCompactionOptions
@@ -21,7 +21,7 @@
 jlong Java_org_rocksdb_CompactionOptions_newCompactionOptions(
     JNIEnv*, jclass) {
   auto* compact_opts = new ROCKSDB_NAMESPACE::CompactionOptions();
-  return reinterpret_cast<jlong>(compact_opts);
+  return GET_CPLUSPLUS_POINTER(compact_opts);
 }
 
 /*

java/rocksjni/compaction_options_fifo.cc

@@ -10,6 +10,7 @@
 
 #include "include/org_rocksdb_CompactionOptionsFIFO.h"
 #include "rocksdb/advanced_options.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 
 /*
  * Class:     org_rocksdb_CompactionOptionsFIFO
@@ -19,7 +20,7 @@
 jlong Java_org_rocksdb_CompactionOptionsFIFO_newCompactionOptionsFIFO(
     JNIEnv*, jclass) {
   const auto* opt = new ROCKSDB_NAMESPACE::CompactionOptionsFIFO();
-  return reinterpret_cast<jlong>(opt);
+  return GET_CPLUSPLUS_POINTER(opt);
 }
 
 /*

java/rocksjni/compaction_options_universal.cc

@@ -10,6 +10,7 @@
 
 #include "include/org_rocksdb_CompactionOptionsUniversal.h"
 #include "rocksdb/advanced_options.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 
 /*
@@ -20,7 +21,7 @@
 jlong Java_org_rocksdb_CompactionOptionsUniversal_newCompactionOptionsUniversal(
     JNIEnv*, jclass) {
   const auto* opt = new ROCKSDB_NAMESPACE::CompactionOptionsUniversal();
-  return reinterpret_cast<jlong>(opt);
+  return GET_CPLUSPLUS_POINTER(opt);
 }
 
 /*

java/rocksjni/comparator.cc

@@ -9,12 +9,14 @@
 #include <jni.h>
 #include <stdio.h>
 #include <stdlib.h>
+
 #include <functional>
 #include <string>
 
 #include "include/org_rocksdb_AbstractComparator.h"
 #include "include/org_rocksdb_NativeComparatorWrapper.h"
 #include "rocksjni/comparatorjnicallback.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 
 /*
@@ -29,7 +31,7 @@ jlong Java_org_rocksdb_AbstractComparator_createNewComparator(
           copt_handle);
   auto* c =
       new ROCKSDB_NAMESPACE::ComparatorJniCallback(env, jcomparator, copt);
-  return reinterpret_cast<jlong>(c);
+  return GET_CPLUSPLUS_POINTER(c);
 }
 
 /*

java/rocksjni/compression_options.cc

@@ -10,6 +10,7 @@
 
 #include "include/org_rocksdb_CompressionOptions.h"
 #include "rocksdb/advanced_options.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 
 /*
  * Class:     org_rocksdb_CompressionOptions
@@ -19,7 +20,7 @@
 jlong Java_org_rocksdb_CompressionOptions_newCompressionOptions(
     JNIEnv*, jclass) {
   const auto* opt = new ROCKSDB_NAMESPACE::CompressionOptions();
-  return reinterpret_cast<jlong>(opt);
+  return GET_CPLUSPLUS_POINTER(opt);
 }
 
 /*

java/rocksjni/concurrent_task_limiter.cc

@@ -6,6 +6,7 @@
 #include <string>
 
 #include "include/org_rocksdb_ConcurrentTaskLimiterImpl.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 
 /*
@@ -25,7 +26,7 @@ jlong Java_org_rocksdb_ConcurrentTaskLimiterImpl_newConcurrentTaskLimiterImpl0(
   auto* ptr = new std::shared_ptr<ROCKSDB_NAMESPACE::ConcurrentTaskLimiter>(
       ROCKSDB_NAMESPACE::NewConcurrentTaskLimiter(name, limit));
 
-  return reinterpret_cast<jlong>(ptr);
+  return GET_CPLUSPLUS_POINTER(ptr);
 }
 
 /*

java/rocksjni/config_options.cc

@@ -11,6 +11,7 @@
 
 #include "include/org_rocksdb_ConfigOptions.h"
 #include "rocksdb/convenience.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/portal.h"
 
 /*
@@ -32,7 +33,7 @@ void Java_org_rocksdb_ConfigOptions_disposeInternal(JNIEnv *, jobject,
  */
 jlong Java_org_rocksdb_ConfigOptions_newConfigOptions(JNIEnv *, jclass) {
   auto *cfg_opt = new ROCKSDB_NAMESPACE::ConfigOptions();
-  return reinterpret_cast<jlong>(cfg_opt);
+  return GET_CPLUSPLUS_POINTER(cfg_opt);
 }
 
 /*

java/rocksjni/cplusplus_to_java_convert.h

@@ -0,0 +1,37 @@
+// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+/*
+ * This macro is used for 32 bit OS. In 32 bit OS, the result number is a
+ negative number if we use reinterpret_cast<jlong>(pointer).
+ * For example, jlong ptr = reinterpret_cast<jlong>(pointer), ptr is a negative
+ number in 32 bit OS.
+ * If we check ptr using ptr > 0, it fails. For example, the following code is
+ not correct.
+ * if (jblock_cache_handle > 0) {
+      std::shared_ptr<ROCKSDB_NAMESPACE::Cache> *pCache =
+          reinterpret_cast<std::shared_ptr<ROCKSDB_NAMESPACE::Cache> *>(
+              jblock_cache_handle);
+      options.block_cache = *pCache;
+    }
+ * But the result number is positive number if we do
+ reinterpret_cast<size_t>(pointer) first and then cast it to jlong. size_t is 4
+ bytes long in 32 bit OS and 8 bytes long in 64 bit OS.
+ static_cast<jlong>(reinterpret_cast<size_t>(_pointer)) is also working in 64
+ bit OS.
+ *
+ * We don't need an opposite cast because it works from jlong to c++ pointer in
+ both 32 bit and 64 bit OS.
+ * For example, the following code is working in both 32 bit and 64 bit OS.
+ jblock_cache_handle is jlong.
+ *   std::shared_ptr<ROCKSDB_NAMESPACE::Cache> *pCache =
+          reinterpret_cast<std::shared_ptr<ROCKSDB_NAMESPACE::Cache> *>(
+              jblock_cache_handle);
+*/
+
+#define GET_CPLUSPLUS_POINTER(_pointer) \
+  static_cast<jlong>(reinterpret_cast<size_t>(_pointer))

java/rocksjni/env.cc

@@ -17,6 +17,7 @@
 #include "include/org_rocksdb_RocksMemEnv.h"
 #include "include/org_rocksdb_TimedEnv.h"
 #include "portal.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 
 /*
  * Class:     org_rocksdb_Env
@@ -25,7 +26,7 @@
  */
 jlong Java_org_rocksdb_Env_getDefaultEnvInternal(
     JNIEnv*, jclass) {
-  return reinterpret_cast<jlong>(ROCKSDB_NAMESPACE::Env::Default());
+  return GET_CPLUSPLUS_POINTER(ROCKSDB_NAMESPACE::Env::Default());
 }
 
 /*
@@ -162,7 +163,7 @@ jobjectArray Java_org_rocksdb_Env_getThreadList(
 jlong Java_org_rocksdb_RocksMemEnv_createMemEnv(
     JNIEnv*, jclass, jlong jbase_env_handle) {
   auto* base_env = reinterpret_cast<ROCKSDB_NAMESPACE::Env*>(jbase_env_handle);
-  return reinterpret_cast<jlong>(ROCKSDB_NAMESPACE::NewMemEnv(base_env));
+  return GET_CPLUSPLUS_POINTER(ROCKSDB_NAMESPACE::NewMemEnv(base_env));
 }
 
 /*
@@ -185,7 +186,7 @@ void Java_org_rocksdb_RocksMemEnv_disposeInternal(
 jlong Java_org_rocksdb_TimedEnv_createTimedEnv(
     JNIEnv*, jclass, jlong jbase_env_handle) {
   auto* base_env = reinterpret_cast<ROCKSDB_NAMESPACE::Env*>(jbase_env_handle);
-  return reinterpret_cast<jlong>(ROCKSDB_NAMESPACE::NewTimedEnv(base_env));
+  return GET_CPLUSPLUS_POINTER(ROCKSDB_NAMESPACE::NewTimedEnv(base_env));
 }
 
 /*

java/rocksjni/env_options.cc

@@ -11,6 +11,7 @@
 
 #include "include/org_rocksdb_EnvOptions.h"
 #include "rocksdb/env.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 
 #define ENV_OPTIONS_SET_BOOL(_jhandle, _opt)                          \
   reinterpret_cast<ROCKSDB_NAMESPACE::EnvOptions *>(_jhandle)->_opt = \
@@ -35,7 +36,7 @@
 jlong Java_org_rocksdb_EnvOptions_newEnvOptions__(
     JNIEnv*, jclass) {
   auto *env_opt = new ROCKSDB_NAMESPACE::EnvOptions();
-  return reinterpret_cast<jlong>(env_opt);
+  return GET_CPLUSPLUS_POINTER(env_opt);
 }
 
 /*
@@ -48,7 +49,7 @@ jlong Java_org_rocksdb_EnvOptions_newEnvOptions__J(
   auto *db_options =
       reinterpret_cast<ROCKSDB_NAMESPACE::DBOptions *>(jdboptions_handle);
   auto *env_opt = new ROCKSDB_NAMESPACE::EnvOptions(*db_options);
-  return reinterpret_cast<jlong>(env_opt);
+  return GET_CPLUSPLUS_POINTER(env_opt);
 }
 
 /*

java/rocksjni/event_listener.cc

@@ -11,6 +11,7 @@
 #include <memory>
 
 #include "include/org_rocksdb_AbstractEventListener.h"
+#include "rocksjni/cplusplus_to_java_convert.h"
 #include "rocksjni/event_listener_jnicallback.h"
 #include "rocksjni/portal.h"
 
@@ -28,7 +29,7 @@ jlong Java_org_rocksdb_AbstractEventListener_createNewEventListener(
       new std::shared_ptr<ROCKSDB_NAMESPACE::EventListener>(
           new ROCKSDB_NAMESPACE::EventListenerJniCallback(
               env, jobj, enabled_event_callbacks));
-  return reinterpret_cast<jlong>(sptr_event_listener);
+  return GET_CPLUSPLUS_POINTER(sptr_event_listener);
 }
 
 /*