Fixing crash test and other refactoring

.github/actions/install-maven/action.yml

@@ -4,8 +4,8 @@ runs:
   steps:
   - name: Install Maven
     run: |
-      wget --no-check-certificate https://archive.apache.org/dist/maven/maven-3/3.9.11/binaries/apache-maven-3.9.11-bin.tar.gz
-      tar zxf apache-maven-3.9.11-bin.tar.gz
-      echo "export M2_HOME=$(pwd)/apache-maven-3.9.11" >> $GITHUB_ENV
-      echo "$(pwd)/apache-maven-3.9.11/bin" >> $GITHUB_PATH
+      wget --no-check-certificate https://dlcdn.apache.org/maven/maven-3/3.9.10/binaries/apache-maven-3.9.10-bin.tar.gz
+      tar zxf apache-maven-3.9.10-bin.tar.gz
+      echo "export M2_HOME=$(pwd)/apache-maven-3.9.10" >> $GITHUB_ENV
+      echo "$(pwd)/apache-maven-3.9.10/bin" >> $GITHUB_PATH
     shell: bash

.github/workflows/nightly.yml

@@ -27,6 +27,18 @@ jobs:
         git config --global --add safe.directory /__w/rocksdb/rocksdb
         tools/check_format_compatible.sh
     - uses: "./.github/actions/post-steps"
+  build-linux-run-microbench:
+    if: ${{ github.repository_owner == 'facebook' }}
+    runs-on:
+      labels: 16-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/pre-steps"
+    - run: DEBUG_LEVEL=0 make -j32 run_microbench
+    - uses: "./.github/actions/post-steps"
   build-linux-non-shm:
     if: ${{ github.repository_owner == 'facebook' }}
     runs-on:
@@ -79,6 +91,15 @@ jobs:
     steps:
     - uses: actions/checkout@v4.1.0
     - uses: "./.github/actions/windows-build-steps"
+  build-windows-vs2022:
+    if: ${{ github.repository_owner == 'facebook' }}
+    runs-on: windows-2022
+    env:
+      CMAKE_GENERATOR: Visual Studio 17 2022
+      CMAKE_PORTABLE: 1
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/windows-build-steps"
   build-linux-arm-test-full:
     if: ${{ github.repository_owner == 'facebook' }}
     runs-on:
@@ -89,59 +110,3 @@ jobs:
       - run: sudo apt-get update && sudo apt-get install -y build-essential libgflags-dev
       - run: make V=1 J=4 -j4 check
       - uses: "./.github/actions/post-steps"
-  build-examples:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - name: Build examples
-      run: make V=1 -j4 static_lib && cd examples && make V=1 -j4
-    - uses: "./.github/actions/post-steps"
-  build-fuzzers:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - name: Build rocksdb lib
-      run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
-    - name: Build fuzzers
-      run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
-    - uses: "./.github/actions/post-steps"
-  build-linux-gcc-11-no_test_run:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench
-    - uses: "./.github/actions/post-steps"
-  build-linux-cmake-with-folly-lite-no-test:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    env:
-      CC: gcc-10
-      CXX: g++-10
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-folly"
-    - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)"
-    - uses: "./.github/actions/post-steps"

.github/workflows/pr-jobs.yml

@@ -109,6 +109,22 @@ jobs:
     - uses: "./.github/actions/build-folly"
     - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)"
     - uses: "./.github/actions/post-steps"
+  build-linux-cmake-with-folly-lite-no-test:
+    if: ${{ github.repository_owner == 'facebook' }}
+    runs-on:
+      labels: 16-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    env:
+      CC: gcc-10
+      CXX: g++-10
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/pre-steps"
+    - uses: "./.github/actions/setup-folly"
+    - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)"
+    - uses: "./.github/actions/post-steps"
   build-linux-make-with-folly:
     if: ${{ github.repository_owner == 'facebook' }}
     runs-on:
@@ -182,7 +198,7 @@ jobs:
     - uses: actions/checkout@v4.1.0
     - uses: "./.github/actions/pre-steps"
     - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check
-    - run: "./sst_dump --help | grep -E -q 'Supported built-in compression types: kNoCompression$' # Verify no compiled in compression\n"
+    - run: "./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression\n"
     - uses: "./.github/actions/post-steps"
   # ======================== Linux No Test Runs ======================= #
   build-linux-release:
@@ -226,6 +242,34 @@ jobs:
     - run: apt-get remove -y libgflags-dev
     - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
     - run: if ./db_stress --version; then false; else true; fi
+  build-examples:
+    if: ${{ github.repository_owner == 'facebook' }}
+    runs-on:
+      labels: 4-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/pre-steps"
+    - name: Build examples
+      run: make V=1 -j4 static_lib && cd examples && make V=1 -j4
+    - uses: "./.github/actions/post-steps"
+  build-fuzzers:
+    if: ${{ github.repository_owner == 'facebook' }}
+    runs-on:
+      labels: 4-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/pre-steps"
+    - name: Build rocksdb lib
+      run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
+    - name: Build fuzzers
+      run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
+    - uses: "./.github/actions/post-steps"
   build-linux-clang-no_test_run:
     if: ${{ github.repository_owner == 'facebook' }}
     runs-on:
@@ -273,7 +317,18 @@ jobs:
     - uses: "./.github/actions/pre-steps"
     - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all
     - uses: "./.github/actions/post-steps"
-
+  build-linux-gcc-11-no_test_run:
+    if: ${{ github.repository_owner == 'facebook' }}
+    runs-on:
+      labels: 16-core-ubuntu
+    container:
+      image: zjay437/rocksdb:0.6
+      options: --shm-size=16gb
+    steps:
+    - uses: actions/checkout@v4.1.0
+    - uses: "./.github/actions/pre-steps"
+    - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench
+    - uses: "./.github/actions/post-steps"
   # ======================== Linux Other Checks ======================= #
   build-linux-clang10-clang-analyze:
     if: ${{ github.repository_owner == 'facebook' }}
@@ -414,11 +469,11 @@ jobs:
     - uses: "./.github/actions/post-steps"
   # ======================== Windows with Tests ======================= #
   # NOTE: some windows jobs are in "nightly" to save resources
-  build-windows-vs2022:
+  build-windows-vs2019:
     if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: windows-2022
+    runs-on: windows-2019
     env:
-      CMAKE_GENERATOR: Visual Studio 17 2022
+      CMAKE_GENERATOR: Visual Studio 16 2019
       CMAKE_PORTABLE: 1
     steps:
     - uses: actions/checkout@v4.1.0
@@ -429,7 +484,7 @@ jobs:
     runs-on:
       labels: 4-core-ubuntu
     container:
-      image: evolvedbinary/rocksjava:centos7_x64-be
+      image: evolvedbinary/rocksjava:centos6_x64-be
       options: --shm-size=16gb
     steps:
     # The docker image is intentionally based on an OS that has an older GLIBC version.
@@ -457,7 +512,7 @@ jobs:
     runs-on:
       labels: 4-core-ubuntu
     container:
-      image: evolvedbinary/rocksjava:centos7_x64-be
+      image: evolvedbinary/rocksjava:centos6_x64-be
       options: --shm-size=16gb
     steps:
     # The docker image is intentionally based on an OS that has an older GLIBC version.
@@ -552,7 +607,7 @@ jobs:
     runs-on:
       labels: 4-core-ubuntu
     container:
-      image: evolvedbinary/rocksjava:alpine3_x64-be
+      image: evolvedbinary/rocksjava:rockylinux8_x64-be
       options: --shm-size=16gb
     steps:
     - uses: actions/checkout@v4.1.0

BUCK

@@ -88,7 +88,6 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/memtable_list.cc",
         "db/merge_helper.cc",
         "db/merge_operator.cc",
-        "db/multi_scan.cc",
         "db/output_validator.cc",
         "db/periodic_task_scheduler.cc",
         "db/range_del_aggregator.cc",
@@ -250,7 +249,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "trace_replay/trace_record_result.cc",
         "trace_replay/trace_replay.cc",
         "util/async_file_reader.cc",
-        "util/auto_tune_compressor.cc",
+        "util/auto_skip_compressor.cc",
         "util/build_version.cc",
         "util/cleanable.cc",
         "util/coding.cc",

CMakeLists.txt

@@ -721,7 +721,6 @@ set(SOURCES
         db/memtable_list.cc
         db/merge_helper.cc
         db/merge_operator.cc
-        db/multi_scan.cc
         db/output_validator.cc
         db/periodic_task_scheduler.cc
         db/range_del_aggregator.cc
@@ -875,7 +874,7 @@ set(SOURCES
         trace_replay/trace_record.cc
         trace_replay/trace_replay.cc
         util/async_file_reader.cc
-        util/auto_tune_compressor.cc
+        util/auto_skip_compressor.cc
         util/cleanable.cc
         util/coding.cc
         util/compaction_job_stats_impl.cc

HISTORY.md

@@ -1,42 +1,6 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
 
-## 10.5.0 (07/18/2025)
-### Public API Changes
-* DB option skip_checking_sst_file_sizes_on_db_open is deprecated, in favor of validating file size in parallel in a thread pool, when db is opened. When DB is opened, with paranoid check enabled, a file with the wrong size would fail the DB open. With paranoid check disabled, the DB open would succeed, the column family with the corrupted file would not be read or write, while the other healthy column families could be read and write normally. When max_open_files option is not set to -1, only a subset of the files will be opened and checked. The rest of the files will be opened and checked when they are accessed.
-
-### Behavior Changes
-* PessimisticTransaction::GetWaitingTxns now returns waiting transaction information even if the current transaction has timed out. This allows the information to be surfaced to users for debugging purposes once it is known that the timeout has occured.
-* A new API GetFileSize is added to FSRandomAccessFile interface class. It uses fstat vs stat on the posix implementation which is more efficient. Caller could use it to get file size faster. This function might be required in the future for FileSystem implementation outside of the RocksDB code base.
-* RocksDB now triggers eligible compactions every 12 hours when periodic compaction is configured. This solves a limitation of the compaction trigger mechanism, which would only trigger compaction after specific events like flush, compaction, or SetOptions.
-
-### Bug Fixes
-* Fix a bug in BackupEngine that can crash backup due to a null FSWritableFile passed to WritableFileWriter.
-* Fix DB::NewMultiScan iterator to respect the scan upper bound specified in ScanOptions
-
-### Performance Improvements
-* Optimized MultiScan using BlockBasedTable to coalesce I/Os and prefetch all data blocks.
-
-## 10.4.0 (06/20/2025)
-### New Features
-* Add a new CF option `memtable_avg_op_scan_flush_trigger` that supports triggering memtable flush when an iterator scans through an expensive range of keys, with the average number of skipped keys from the active memtable exceeding the threshold.
-* Vector based memtable now supports concurrent writers (DBOptions::allow_concurrent_memtable_write) #13675.
-* Add new experimental `TransactionOptions::large_txn_commit_optimize_byte_threshold` to enable optimizations for large transaction commit by transaction batch data size.
-* Add a new option `CompactionOptionsUniversal::reduce_file_locking` and if it's true, auto universal compaction picking will adjust to minimize locking of input files when bottom priority compactions are waiting to run. This can increase the likelihood of existing L0s being selected for compaction, thereby improving write stall and reducing read regression.
-* Add new `format_version=7` to aid experimental support of custom compression algorithms with CompressionManager and block-based table. This format version includes changing the format of `TableProperties::compression_name`.
-
-### Public API Changes
-* Change NewExternalTableFactory to return a unique_ptr instead of shared_ptr.
-* Add an optional min file size requirement for deletion triggered compaction. It can be specified when creating `CompactOnDeletionCollectorFactory`.
-
-### Behavior Changes
-* `TransactionOptions::large_txn_commit_optimize_threshold` now has default value 0 for disabled. `TransactionDBOptions::txn_commit_bypass_memtable_threshold` now has no effect on transactions.
-
-### Bug Fixes
-* Fix a bug where CreateColumnFamilyWithImport() could miss the SST file for the memtable flush it triggered. The exported CF then may not contain the updates in the memtable when CreateColumnFamilyWithImport() is called.
-* Fix iterator operations returning NotImplemented status if disallow_memtable_writes and paranoid_memory_checks CF options are both set.
-* Fixed handling of file checksums in IngestExternalFile() to allow providing checksums using recognized but not necessarily the DB's preferred checksum function, to ease migration between checksum functions.
-
 ## 10.3.0 (05/17/2025)
 ### New Features
 * Add new experimental `CompactionOptionsFIFO::allow_trivial_copy_when_change_temperature` along with `CompactionOptionsFIFO::trivial_copy_buffer_size` to allow optimizing FIFO compactions with tiering when kChangeTemperature to move files from source tier FileSystem to another tier FileSystem via trivial and direct copying raw sst file instead of reading thru the content of the SST file then rebuilding the table files.

Makefile

@@ -2147,14 +2147,14 @@ ZLIB_DOWNLOAD_BASE ?= http://zlib.net
 BZIP2_VER ?= 1.0.8
 BZIP2_SHA256 ?= ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
 BZIP2_DOWNLOAD_BASE ?= http://sourceware.org/pub/bzip2
-SNAPPY_VER ?= 1.2.2
-SNAPPY_SHA256 ?= 90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc
+SNAPPY_VER ?= 1.2.1
+SNAPPY_SHA256 ?= 736aeb64d86566d2236ddffa2865ee5d7a82d26c9016b36218fcc27ea4f09f86
 SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/archive
-LZ4_VER ?= 1.10.0
-LZ4_SHA256 ?= 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b
+LZ4_VER ?= 1.9.4
+LZ4_SHA256 ?= 0b0e3aa07c8c063ddf40b082bdf7e37a1562bda40a0ff5272957f3e987e0e54b
 LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive
-ZSTD_VER ?= 1.5.7
-ZSTD_SHA256 ?= 37d7284556b20954e56e1ca85b80226768902e2edabd3b649e9e72c0c9012ee3
+ZSTD_VER ?= 1.5.5
+ZSTD_SHA256 ?= 98e9c3d949d1b924e28e01eccb7deed865eefebf25c2f21c702e5cd5b63b85e1
 ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
 CURL_SSL_OPTS ?= --tlsv1
 
@@ -2375,27 +2375,27 @@ rocksdbjavastaticreleasedocker: rocksdbjavastaticosx rocksdbjavastaticdockerx86
 
 rocksdbjavastaticdockerx86:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x86-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_x86-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_x86-be --platform linux/386 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerx86_64:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x64-be --platform linux/amd64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerppc64le:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_ppc64le-be --platform linux/ppc64le --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerarm64v8:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_arm64v8-be --platform linux/aarch64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_arm64v8-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockers390x:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_s390x-be --platform linux/s390x --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu18_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_s390x-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu18_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerriscv64:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_riscv64-be --platform linux/riscv64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu20_riscv64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_riscv64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:ubuntu20_riscv64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerx86musl:
 	mkdir -p java/target
@@ -2403,19 +2403,19 @@ rocksdbjavastaticdockerx86musl:
 
 rocksdbjavastaticdockerx86_64musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x64-musl-be --platform linux/amd64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_x64-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_x64-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerppc64lemusl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_ppc64le-musl-be --platform linux/ppc64le --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_ppc64le-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockerarm64v8musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_arm64v8-musl-be --platform linux/aarch64 --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_arm64v8-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticdockers390xmusl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_s390x-musl-be --platform linux/s390x --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
+	docker run --rm --name rocksdb_linux_s390x-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) --env J=$(J) evolvedbinary/rocksjava:alpine3_s390x-be /rocksdb-host/java/crossbuild/docker-build-linux.sh
 
 rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral
 
@@ -2470,8 +2470,8 @@ jtest_run:
 jtest: rocksdbjava
 	cd java;$(MAKE) sample test
 
-jpmd: rocksdbjavageneratepom
-	cd java;$(MAKE) java java_test pmd
+jpmd: rocksdbjava rocksdbjavageneratepom
+	cd java;$(MAKE) pmd
 
 jdb_bench:
 	cd java;$(MAKE) db_bench;
@@ -2492,7 +2492,7 @@ checkout_folly:
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically
-	cd third-party/folly && git reset --hard e95383b7c8b5b1e46cf47acf2f317d54f93c8268
+	cd third-party/folly && git reset --hard d17bf897cb5bbf8f07b122a614e8cffdc38edcde
 	@# Apparently missing include
 	perl -pi -e 's/(#include <atomic>)/$$1\n#include <cstring>/' third-party/folly/folly/lang/Exception.h
 	@# Warning-as-error on memcpy

cache/compressed_secondary_cache.cc

@@ -80,16 +80,15 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
     ptr = reinterpret_cast<CacheAllocationPtr*>(handle_value);
     handle_value_charge = cache_->GetCharge(lru_handle);
     data_ptr = ptr->get();
-    const char* limit = ptr->get() + handle_value_charge;
-    data_ptr =
-        GetVarint32Ptr(data_ptr, limit, static_cast<uint32_t*>(&type_32));
+    data_ptr = GetVarint32Ptr(data_ptr, data_ptr + 1,
+                              static_cast<uint32_t*>(&type_32));
     type = static_cast<CompressionType>(type_32);
-    data_ptr =
-        GetVarint32Ptr(data_ptr, limit, static_cast<uint32_t*>(&source_32));
+    data_ptr = GetVarint32Ptr(data_ptr, data_ptr + 1,
+                              static_cast<uint32_t*>(&source_32));
     source = static_cast<CacheTier>(source_32);
     uint64_t data_size = 0;
-    data_ptr =
-        GetVarint64Ptr(data_ptr, limit, static_cast<uint64_t*>(&data_size));
+    data_ptr = GetVarint64Ptr(data_ptr, ptr->get() + handle_value_charge,
+                              static_cast<uint64_t*>(&data_size));
     assert(handle_value_charge > data_size);
     handle_value_charge = data_size;
   }

cache/secondary_cache_adapter.cc

@@ -130,7 +130,7 @@ CacheWithSecondaryAdapter::~CacheWithSecondaryAdapter() {
               "Secondary cache reserved: %zu\n",
               pri_cache_res_->GetTotalMemoryUsed(), sec_capacity,
               sec_reserved_);
-      assert(!pri_cache_res_mismatch);
+      assert(pri_cache_res_mismatch);
     }
   }
 #endif  // NDEBUG
@@ -489,10 +489,12 @@ const char* CacheWithSecondaryAdapter::Name() const {
 // as well. At the moment, we don't have a good way of handling the case
 // where the new capacity < total cache reservations.
 void CacheWithSecondaryAdapter::SetCapacity(size_t capacity) {
+  size_t sec_capacity = static_cast<size_t>(
+      capacity * (distribute_cache_res_ ? sec_cache_res_ratio_ : 0.0));
+  size_t old_sec_capacity = 0;
+
   if (distribute_cache_res_) {
     MutexLock m(&cache_res_mutex_);
-    size_t sec_capacity = static_cast<size_t>(capacity * sec_cache_res_ratio_);
-    size_t old_sec_capacity = 0;
 
     Status s = secondary_cache_->GetCapacity(old_sec_capacity);
     if (!s.ok()) {
@@ -611,7 +613,6 @@ Status CacheWithSecondaryAdapter::UpdateCacheReservationRatio(
     //    cache utilization (increase in capacity - increase in share of cache
     //    reservation)
     // 3. Increase secondary cache capacity
-    assert(new_sec_reserved >= sec_reserved_);
     s = secondary_cache_->Deflate(new_sec_reserved - sec_reserved_);
     assert(s.ok());
     s = pri_cache_res_->UpdateCacheReservation(

db/builder.cc

@@ -218,7 +218,8 @@ Status BuildTable(
       const Slice& key = c_iter.key();
       const Slice& value = c_iter.value();
       ParsedInternalKey ikey = c_iter.ikey();
-      Slice key_after_flush = key;
+      key_after_flush_buf.assign(key.data(), key.size());
+      Slice key_after_flush = key_after_flush_buf;
       Slice value_after_flush = value;
 
       if (ikey.type == kTypeValuePreferredSeqno) {
@@ -236,7 +237,6 @@ Status BuildTable(
               std::min(smallest_preferred_seqno, preferred_seqno);
         } else {
           // Cannot get a useful preferred seqno, convert it to a kTypeValue.
-          key_after_flush_buf.assign(key.data(), key.size());
           UpdateInternalKey(&key_after_flush_buf, ikey.sequence, kTypeValue);
           ikey = ParsedInternalKey(ikey.user_key, ikey.sequence, kTypeValue);
           key_after_flush = key_after_flush_buf;

db/c.cc

@@ -925,10 +925,6 @@ void rocksdb_backup_engine_options_destroy(
   delete options;
 }
 
-void rocksdb_status_ptr_get_error(rocksdb_status_ptr_t* status, char** errptr) {
-  SaveError(errptr, *(status->rep));
-}
-
 rocksdb_checkpoint_t* rocksdb_checkpoint_object_create(rocksdb_t* db,
                                                        char** errptr) {
   Checkpoint* checkpoint;
@@ -3065,10 +3061,6 @@ uint64_t rocksdb_flushjobinfo_smallest_seqno(
   return info->rep.smallest_seqno;
 }
 
-uint32_t rocksdb_flushjobinfo_flush_reason(const rocksdb_flushjobinfo_t* info) {
-  return static_cast<uint32_t>(info->rep.flush_reason);
-}
-
 void rocksdb_reset_status(rocksdb_status_ptr_t* status_ptr) {
   auto ptr = status_ptr->rep;
   *ptr = Status::OK();
@@ -3200,11 +3192,6 @@ int rocksdb_subcompactionjobinfo_output_level(
   return info->rep.output_level;
 }
 
-uint32_t rocksdb_subcompactionjobinfo_compaction_reason(
-    const rocksdb_subcompactionjobinfo_t* info) {
-  return static_cast<uint32_t>(info->rep.compaction_reason);
-}
-
 /* ExternalFileIngestionInfo */
 
 const char* rocksdb_externalfileingestioninfo_cf_name(

db/c_test.c

@@ -2230,6 +2230,10 @@ int main(int argc, char** argv) {
     rocksdb_options_set_skip_stats_update_on_db_open(o, 1);
     CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o));
 
+    rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open(o, 1);
+    CheckCondition(
+        1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(o));
+
     rocksdb_options_set_max_write_buffer_number(o, 97);
     CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(o));
 
@@ -2489,6 +2493,8 @@ int main(int argc, char** argv) {
     CheckCondition(2.0 ==
                    rocksdb_options_get_max_bytes_for_level_multiplier(copy));
     CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(copy));
+    CheckCondition(
+        1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(copy));
     CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(copy));
     CheckCondition(23 ==
                    rocksdb_options_get_min_write_buffer_number_to_merge(copy));
@@ -2675,6 +2681,12 @@ int main(int argc, char** argv) {
     CheckCondition(0 == rocksdb_options_get_skip_stats_update_on_db_open(copy));
     CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o));
 
+    rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open(copy, 0);
+    CheckCondition(
+        0 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(copy));
+    CheckCondition(
+        1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(o));
+
     rocksdb_options_set_max_write_buffer_number(copy, 2000);
     CheckCondition(2000 == rocksdb_options_get_max_write_buffer_number(copy));
     CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(o));

db/column_family.cc

@@ -110,48 +110,23 @@ void GetInternalTblPropCollFactory(
   }
 }
 
-Status CheckCompressionSupportedWithManager(
-    CompressionType type, UnownedPtr<CompressionManager> mgr) {
-  if (mgr) {
-    if (!mgr->SupportsCompressionType(type)) {
-      return Status::NotSupported("Compression type " +
-                                  CompressionTypeToString(type) +
-                                  " is not recognized/supported by this "
-                                  "version of CompressionManager " +
-                                  mgr->GetId());
-    }
-  } else {
-    if (!CompressionTypeSupported(type)) {
-      if (type <= kLastBuiltinCompression) {
-        return Status::InvalidArgument("Compression type " +
-                                       CompressionTypeToString(type) +
-                                       " is not linked with the binary.");
-      } else {
-        return Status::NotSupported(
-            "Compression type " + CompressionTypeToString(type) +
-            " is not recognized/supported by built-in CompressionManager.");
-      }
-    }
-  }
-  return Status::OK();
-}
-
 Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) {
   if (!cf_options.compression_per_level.empty()) {
     for (size_t level = 0; level < cf_options.compression_per_level.size();
          ++level) {
-      Status s = CheckCompressionSupportedWithManager(
-          cf_options.compression_per_level[level],
-          cf_options.compression_manager.get());
-      if (!s.ok()) {
-        return s;
+      if (!CompressionTypeSupported(cf_options.compression_per_level[level])) {
+        return Status::InvalidArgument(
+            "Compression type " +
+            CompressionTypeToString(cf_options.compression_per_level[level]) +
+            " is not linked with the binary.");
       }
     }
   } else {
-    Status s = CheckCompressionSupportedWithManager(
-        cf_options.compression, cf_options.compression_manager.get());
-    if (!s.ok()) {
-      return s;
+    if (!CompressionTypeSupported(cf_options.compression)) {
+      return Status::InvalidArgument(
+          "Compression type " +
+          CompressionTypeToString(cf_options.compression) +
+          " is not linked with the binary.");
     }
   }
   if (cf_options.compression_opts.zstd_max_train_bytes > 0) {
@@ -1633,8 +1608,6 @@ Status ColumnFamilyData::SetOptions(
   Status s = GetColumnFamilyOptionsFromMap(config_opts, cf_opts, options_map,
                                            &cf_opts);
   if (s.ok()) {
-    // FIXME: we should call SanitizeOptions() too or consolidate it with
-    // ValidateOptions().
     s = ValidateOptions(db_opts, cf_opts);
   }
   if (s.ok()) {

db/compaction/compaction.cc

@@ -284,9 +284,9 @@ Compaction::Compaction(
     CompressionOptions _compression_opts, Temperature _output_temperature,
     uint32_t _max_subcompactions, std::vector<FileMetaData*> _grandparents,
     std::optional<SequenceNumber> _earliest_snapshot,
-    const SnapshotChecker* _snapshot_checker,
-    CompactionReason _compaction_reason, const std::string& _trim_ts,
-    double _score, bool l0_files_might_overlap,
+    const SnapshotChecker* _snapshot_checker, bool _manual_compaction,
+    const std::string& _trim_ts, double _score, bool _deletion_compaction,
+    bool l0_files_might_overlap, CompactionReason _compaction_reason,
     BlobGarbageCollectionPolicy _blob_garbage_collection_policy,
     double _blob_garbage_collection_age_cutoff)
     : input_vstorage_(vstorage),
@@ -304,9 +304,7 @@ Compaction::Compaction(
       output_compression_(_compression),
       output_compression_opts_(_compression_opts),
       output_temperature_(_output_temperature),
-      deletion_compaction_(_compaction_reason == CompactionReason::kFIFOTtl ||
-                           _compaction_reason ==
-                               CompactionReason::kFIFOMaxSize),
+      deletion_compaction_(_deletion_compaction),
       l0_files_might_overlap_(l0_files_might_overlap),
       inputs_(PopulateWithAtomicBoundaries(vstorage, std::move(_inputs))),
       grandparents_(std::move(_grandparents)),
@@ -323,8 +321,7 @@ Compaction::Compaction(
               ? false
               : IsBottommostLevel(output_level_, vstorage, inputs_)),
       is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
-      is_manual_compaction_(_compaction_reason ==
-                            CompactionReason::kManualCompaction),
+      is_manual_compaction_(_manual_compaction),
       trim_ts_(_trim_ts),
       is_trivial_move_(false),
       compaction_reason_(_compaction_reason),
@@ -352,6 +349,9 @@ Compaction::Compaction(
                                       immutable_options_, start_level_,
                                       output_level_)) {
   MarkFilesBeingCompacted(true);
+  if (is_manual_compaction_) {
+    compaction_reason_ = CompactionReason::kManualCompaction;
+  }
   if (max_subcompactions_ == 0) {
     max_subcompactions_ = _mutable_db_options.max_subcompactions;
   }

db/compaction/compaction.h

@@ -94,9 +94,10 @@ class Compaction {
              std::vector<FileMetaData*> grandparents,
              std::optional<SequenceNumber> earliest_snapshot,
              const SnapshotChecker* snapshot_checker,
-             CompactionReason compaction_reason,
-             const std::string& trim_ts = "", double score = -1,
+             bool manual_compaction = false, const std::string& trim_ts = "",
+             double score = -1, bool deletion_compaction = false,
              bool l0_files_might_overlap = true,
+             CompactionReason compaction_reason = CompactionReason::kUnknown,
              BlobGarbageCollectionPolicy blob_garbage_collection_policy =
                  BlobGarbageCollectionPolicy::kUseDefault,
              double blob_garbage_collection_age_cutoff = -1);

db/compaction/compaction_job_test.cc

@@ -458,10 +458,9 @@ class CompactionJobTestBase : public testing::Test {
       ReadOptions read_opts;
       Status s = cf_options_.table_factory->NewTableReader(
           read_opts,
-          TableReaderOptions(cfd->ioptions(), /*prefix_extractor=*/nullptr,
-                             /*compression_manager=*/nullptr, FileOptions(),
+          TableReaderOptions(cfd->ioptions(), nullptr, FileOptions(),
                              cfd_->internal_comparator(),
-                             /*block_protection_bytes_per_key=*/0),
+                             0 /* block_protection_bytes_per_key */),
           std::move(freader), file_size, &table_reader, false);
       ASSERT_OK(s);
       assert(table_reader);
@@ -651,8 +650,7 @@ class CompactionJobTestBase : public testing::Test {
         mutable_cf_options_.max_compaction_bytes, 0, kNoCompression,
         cfd->GetLatestMutableCFOptions().compression_opts,
         Temperature::kUnknown, max_subcompactions, grandparents,
-        /*earliest_snapshot*/ std::nullopt, /*snapshot_checker*/ nullptr,
-        CompactionReason::kManualCompaction);
+        /*earliest_snapshot*/ std::nullopt, /*snapshot_checker*/ nullptr, true);
     compaction.FinalizeInputInfo(cfd->current());
 
     assert(db_options_.info_log);

db/compaction/compaction_picker.cc

@@ -381,7 +381,7 @@ Compaction* CompactionPicker::CompactFiles(
       mutable_cf_options.default_write_temperature,
       compact_options.max_subcompactions,
       /* grandparents */ {}, /* earliest_snapshot */ std::nullopt,
-      /* snapshot_checker */ nullptr, CompactionReason::kManualCompaction);
+      /* snapshot_checker */ nullptr, true);
   RegisterCompaction(c);
   return c;
 }
@@ -680,9 +680,10 @@ Compaction* CompactionPicker::CompactRange(
         mutable_cf_options.default_write_temperature,
         compact_range_options.max_subcompactions,
         /* grandparents */ {}, /* earliest_snapshot */ std::nullopt,
-        /* snapshot_checker */ nullptr, CompactionReason::kManualCompaction,
-        trim_ts, /* score */ -1,
-        /* l0_files_might_overlap */ true,
+        /* snapshot_checker */ nullptr,
+        /* is manual */ true, trim_ts, /* score */ -1,
+        /* deletion_compaction */ false, /* l0_files_might_overlap */ true,
+        CompactionReason::kUnknown,
         compact_range_options.blob_garbage_collection_policy,
         compact_range_options.blob_garbage_collection_age_cutoff);
 
@@ -872,8 +873,9 @@ Compaction* CompactionPicker::CompactRange(
       mutable_cf_options.default_write_temperature,
       compact_range_options.max_subcompactions, std::move(grandparents),
       /* earliest_snapshot */ std::nullopt, /* snapshot_checker */ nullptr,
-      CompactionReason::kManualCompaction, trim_ts, /* score */ -1,
-      /* l0_files_might_overlap */ true,
+      /* is manual */ true, trim_ts, /* score */ -1,
+      /* deletion_compaction */ false, /* l0_files_might_overlap */ true,
+      CompactionReason::kUnknown,
       compact_range_options.blob_garbage_collection_policy,
       compact_range_options.blob_garbage_collection_age_cutoff);
 

db/compaction/compaction_picker_fifo.cc

@@ -127,9 +127,11 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
       mutable_cf_options.compression_opts,
       mutable_cf_options.default_write_temperature,
       /* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
-      /* snapshot_checker */ nullptr, CompactionReason::kFIFOTtl,
+      /* snapshot_checker */ nullptr,
+      /* is manual */ false,
       /* trim_ts */ "", vstorage->CompactionScore(0),
-      /* l0_files_might_overlap */ true);
+      /* is deletion compaction */ true, /* l0_files_might_overlap */ true,
+      CompactionReason::kFIFOTtl);
   return c;
 }
 
@@ -198,10 +200,11 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
             mutable_cf_options.default_write_temperature,
             0 /* max_subcompactions */, {},
             /* earliest_snapshot */ std::nullopt,
-            /* snapshot_checker */ nullptr,
-            CompactionReason::kFIFOReduceNumFiles,
+            /* snapshot_checker */ nullptr, /* is manual */ false,
             /* trim_ts */ "", vstorage->CompactionScore(0),
-            /* l0_files_might_overlap */ true);
+            /* is deletion compaction */ false,
+            /* l0_files_might_overlap */ true,
+            CompactionReason::kFIFOReduceNumFiles);
         return c;
       }
     }
@@ -294,9 +297,11 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
       mutable_cf_options.compression_opts,
       mutable_cf_options.default_write_temperature,
       /* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
-      /* snapshot_checker */ nullptr, CompactionReason::kFIFOMaxSize,
+      /* snapshot_checker */ nullptr,
+      /* is manual */ false,
       /* trim_ts */ "", vstorage->CompactionScore(0),
-      /* l0_files_might_overlap */ true);
+      /* is deletion compaction */ true,
+      /* l0_files_might_overlap */ true, CompactionReason::kFIFOMaxSize);
   return c;
 }
 
@@ -411,9 +416,10 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction(
       mutable_cf_options.compression, mutable_cf_options.compression_opts,
       compaction_target_temp,
       /* max_subcompactions */ 0, {}, /* earliest_snapshot */ std::nullopt,
-      /* snapshot_checker */ nullptr, CompactionReason::kChangeTemperature,
-      /* trim_ts */ "", vstorage->CompactionScore(0),
-      /* l0_files_might_overlap */ true);
+      /* snapshot_checker */ nullptr,
+      /* is manual */ false, /* trim_ts */ "", vstorage->CompactionScore(0),
+      /* is deletion compaction */ false, /* l0_files_might_overlap */ true,
+      CompactionReason::kChangeTemperature);
   return c;
 }
 

db/compaction/compaction_picker_level.cc

@@ -145,6 +145,7 @@ class LevelCompactionBuilder {
   int parent_index_ = -1;
   int base_index_ = -1;
   double start_level_score_ = 0;
+  bool is_manual_ = false;
   bool is_l0_trivial_move_ = false;
   CompactionInputFiles start_level_inputs_;
   std::vector<CompactionInputFiles> compaction_inputs_;
@@ -560,8 +561,9 @@ Compaction* LevelCompactionBuilder::GetCompaction() {
       mutable_cf_options_.default_write_temperature,
       /* max_subcompactions */ 0, std::move(grandparents_),
       /* earliest_snapshot */ std::nullopt, /* snapshot_checker */ nullptr,
-      compaction_reason_,
-      /* trim_ts */ "", start_level_score_, l0_files_might_overlap);
+      is_manual_,
+      /* trim_ts */ "", start_level_score_, false /* deletion_compaction */,
+      l0_files_might_overlap, compaction_reason_);
 
   // If it's level 0 compaction, make sure we don't execute any other level 0
   // compactions in parallel

db/compaction/compaction_picker_universal.cc

@@ -1097,9 +1097,10 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(
                         mutable_cf_options_.default_write_temperature,
                         /* max_subcompactions */ 0, grandparents,
                         /* earliest_snapshot */ std::nullopt,
-                        /* snapshot_checker */ nullptr, compaction_reason,
-                        /* trim_ts */ "", score_,
-                        /* l0_files_might_overlap */ true);
+                        /* snapshot_checker */ nullptr,
+                        /* is manual */ false, /* trim_ts */ "", score_,
+                        false /* deletion_compaction */,
+                        /* l0_files_might_overlap */ true, compaction_reason);
 }
 
 // Look at overall size amplification. If size amplification
@@ -1446,9 +1447,10 @@ Compaction* UniversalCompactionBuilder::PickIncrementalForReduceSizeAmp(
       /* max_subcompactions */ 0, /* grandparents */ {},
       /* earliest_snapshot */ std::nullopt,
       /* snapshot_checker */ nullptr,
-      CompactionReason::kUniversalSizeAmplification,
-      /* trim_ts */ "", score_,
-      /* l0_files_might_overlap */ true);
+      /* is manual */ false,
+      /* trim_ts */ "", score_, false /* deletion_compaction */,
+      /* l0_files_might_overlap */ true,
+      CompactionReason::kUniversalSizeAmplification);
 }
 
 // Pick files marked for compaction. Typically, files are marked by
@@ -1598,9 +1600,11 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
       GetCompressionOptions(mutable_cf_options_, vstorage_, output_level),
       mutable_cf_options_.default_write_temperature,
       /* max_subcompactions */ 0, grandparents, earliest_snapshot_,
-      snapshot_checker_, CompactionReason::kFilesMarkedForCompaction,
-      /* trim_ts */ "", score_,
-      /* l0_files_might_overlap */ true);
+      snapshot_checker_,
+      /* is manual */ false,
+      /* trim_ts */ "", score_, false /* deletion_compaction */,
+      /* l0_files_might_overlap */ true,
+      CompactionReason::kFilesMarkedForCompaction);
 }
 
 Compaction* UniversalCompactionBuilder::PickCompactionToOldest(
@@ -1696,9 +1700,10 @@ Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange(
       mutable_cf_options_.default_write_temperature,
       /* max_subcompactions */ 0, /* grandparents */ {},
       /* earliest_snapshot */ std::nullopt,
-      /* snapshot_checker */ nullptr, compaction_reason,
-      /* trim_ts */ "", score_,
-      /* l0_files_might_overlap */ true);
+      /* snapshot_checker */ nullptr,
+      /* is manual */ false,
+      /* trim_ts */ "", score_, false /* deletion_compaction */,
+      /* l0_files_might_overlap */ true, compaction_reason);
 }
 
 Compaction* UniversalCompactionBuilder::PickPeriodicCompaction() {

db/convenience.cc

@@ -93,10 +93,9 @@ Status VerifySstFileChecksumInternal(const Options& options,
           nullptr /* file_read_hist */, ioptions.rate_limiter.get()));
   const bool kImmortal = true;
   auto reader_options = TableReaderOptions(
-      ioptions, options.prefix_extractor, options.compression_manager.get(),
-      env_options, internal_comparator, options.block_protection_bytes_per_key,
-      false /* skip_filters */, !kImmortal, false /* force_direct_prefetch */,
-      -1 /* level */);
+      ioptions, options.prefix_extractor, env_options, internal_comparator,
+      options.block_protection_bytes_per_key, false /* skip_filters */,
+      !kImmortal, false /* force_direct_prefetch */, -1 /* level */);
   reader_options.largest_seqno = largest_seqno;
   s = options.table_factory->NewTableReader(
       read_options, reader_options, std::move(file_reader), file_size,

db/corruption_test.cc

@@ -556,74 +556,6 @@ TEST_F(CorruptionTest, TableFileFooterNotMagic) {
   ASSERT_TRUE(s.ToString().find(".sst") != std::string::npos);
 }
 
-TEST_F(CorruptionTest, DBOpenWithWrongFileSize) {
-  // Validate that when paranoid flag is true, DB::Open() fails if one of the
-  // file corrupted. Validate that when paranoid flag is false, DB::Open()
-  // succeed if one of the file corrupted, and the healthy file is readable.
-  CloseDb();
-
-  const std::string test_cf_name = "test_cf";
-  std::vector<ColumnFamilyDescriptor> cf_descs;
-  cf_descs.emplace_back(kDefaultColumnFamilyName, ColumnFamilyOptions());
-  cf_descs.emplace_back(test_cf_name, ColumnFamilyOptions());
-
-  {
-    options_.create_missing_column_families = true;
-    std::vector<ColumnFamilyHandle*> cfhs;
-    ASSERT_OK(DB::Open(options_, dbname_, cf_descs, &cfhs, &db_));
-    assert(db_ != nullptr);  // suppress false clang-analyze report
-
-    ASSERT_OK(db_->Put(WriteOptions(), cfhs[0], "k", "v"));
-    ASSERT_OK(db_->Put(WriteOptions(), cfhs[1], "k1", "v1"));
-    ASSERT_OK(db_->Put(WriteOptions(), cfhs[0], "k2", "v2"));
-    for (auto* cfh : cfhs) {
-      delete cfh;
-    }
-    DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
-    ASSERT_OK(dbi->TEST_FlushMemTable());
-
-    // ********************************************
-    // Corrupt the file by making the file bigger
-    std::vector<LiveFileMetaData> metadata;
-    db_->GetLiveFilesMetaData(&metadata);
-    std::string filename = dbname_ + metadata[0].name;
-    const auto& fs = options_.env->GetFileSystem();
-    {
-      std::unique_ptr<FSWritableFile> f;
-      ASSERT_OK(fs->ReopenWritableFile(filename, FileOptions(), &f, nullptr));
-      ASSERT_OK(f->Append("blahblah", IOOptions(), nullptr));
-      ASSERT_OK(f->Close(IOOptions(), nullptr));
-    }
-    CloseDb();
-  }
-
-  // DB failed to open due to one of the file is corrupted, as paranoid flag is
-  // true
-  options_.paranoid_checks = true;
-  std::vector<ColumnFamilyHandle*> cfhs;
-  auto s = DB::Open(options_, dbname_, cf_descs, &cfhs, &db_);
-  ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(s.ToString().find("file size mismatch") != std::string::npos);
-
-  // DB opened successfully, as paranoid flag is false, validate the one that is
-  // healthy is still accessible
-  options_.paranoid_checks = false;
-  ASSERT_OK(DB::Open(options_, dbname_, cf_descs, &cfhs, &db_));
-  assert(db_ != nullptr);  // suppress false clang-analyze report
-
-  std::string v;
-  ASSERT_OK(db_->Get(ReadOptions(), cfhs[1], "k1", &v));
-  ASSERT_EQ(v, "v1");
-
-  // Validate the default column family is corrupted
-  Check(0, 0);
-  s = db_->Get(ReadOptions(), cfhs[0], "k1", &v);
-  ASSERT_TRUE(s.IsCorruption());
-
-  delete cfhs[1];
-  delete cfhs[0];
-}
-
 TEST_F(CorruptionTest, TableFileWrongSize) {
   Build(100);
   DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
@@ -647,16 +579,13 @@ TEST_F(CorruptionTest, TableFileWrongSize) {
   // DB actually accepts this without paranoid checks, relying on size
   // recorded in manifest to locate the SST footer.
   options_.paranoid_checks = false;
+  options_.skip_checking_sst_file_sizes_on_db_open = false;
   Reopen();
-  // As footer could not be extraced, file is completely unreadable
-  Check(0, 0);
-  std::string v;
-  auto s = db_->Get(ReadOptions(), "k1", &v);
-  ASSERT_TRUE(s.IsCorruption());
+  Check(100, 100);
 
   // But reports the issue with paranoid checks
   options_.paranoid_checks = true;
-  s = TryReopen();
+  Status s = TryReopen();
   ASSERT_TRUE(s.IsCorruption());
   ASSERT_TRUE(s.ToString().find("file size mismatch") != std::string::npos);
 

db/db_block_cache_test.cc

@@ -829,78 +829,68 @@ TEST_F(DBBlockCacheTest, CacheCompressionDict) {
   const int kNumEntriesPerFile = 128;
   const int kNumBytesPerEntry = 1024;
 
-  std::vector<CompressionType> dict_compressions =
-      GetSupportedDictCompressions();
+  // Try all the available libraries that support dictionary compression
+  std::vector<CompressionType> compression_types;
+  if (Zlib_Supported()) {
+    compression_types.push_back(kZlibCompression);
+  }
+  if (LZ4_Supported()) {
+    compression_types.push_back(kLZ4Compression);
+    compression_types.push_back(kLZ4HCCompression);
+  }
+  if (ZSTD_Supported()) {
+    compression_types.push_back(kZSTD);
+  }
   Random rnd(301);
-  // Format version before and after compression handling changes
-  for (int format_version : {6, 7}) {
-    // Test all supported compression types because (at least historically)
-    // dictionary compression could be enabled and a dictionary block saved
-    // but ignored by some compression types. Ensure we at least don't crash
-    // or return corruption for those.
-    for (auto compression_type : GetSupportedCompressions()) {
-      // Extra handling checks only for types actually supporting dictionary
-      // compression.
-      bool dict_supported =
-          std::count(dict_compressions.begin(), dict_compressions.end(),
-                     compression_type) > 0;
+  for (auto compression_type : compression_types) {
+    Options options = CurrentOptions();
+    options.bottommost_compression = compression_type;
+    options.bottommost_compression_opts.max_dict_bytes = 4096;
+    options.bottommost_compression_opts.enabled = true;
+    options.create_if_missing = true;
+    options.num_levels = 2;
+    options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+    options.target_file_size_base = kNumEntriesPerFile * kNumBytesPerEntry;
+    BlockBasedTableOptions table_options;
+    table_options.cache_index_and_filter_blocks = true;
+    table_options.block_cache.reset(new MockCache());
+    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+    DestroyAndReopen(options);
 
-      Options options = CurrentOptions();
-      options.bottommost_compression = compression_type;
-      options.bottommost_compression_opts.max_dict_bytes = 4096;
-      options.bottommost_compression_opts.enabled = true;
-      options.create_if_missing = true;
-      options.num_levels = 2;
-      options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
-      options.target_file_size_base = kNumEntriesPerFile * kNumBytesPerEntry;
-      BlockBasedTableOptions table_options;
-      table_options.cache_index_and_filter_blocks = true;
-      table_options.block_cache.reset(new MockCache());
-      table_options.format_version = format_version;
-      options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-      DestroyAndReopen(options);
+    RecordCacheCountersForCompressionDict(options);
 
-      RecordCacheCountersForCompressionDict(options);
-
-      for (int i = 0; i < kNumFiles; ++i) {
-        ASSERT_EQ(i, NumTableFilesAtLevel(0, 0));
-        for (int j = 0; j < kNumEntriesPerFile; ++j) {
-          std::string value = rnd.RandomString(kNumBytesPerEntry);
-          ASSERT_OK(Put(Key(j * kNumFiles + i), value.c_str()));
-        }
-        ASSERT_OK(Flush());
-      }
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
-      ASSERT_EQ(0, NumTableFilesAtLevel(0));
-      ASSERT_EQ(kNumFiles, NumTableFilesAtLevel(1));
-
-      if (dict_supported) {
-        // Compression dictionary blocks are preloaded.
-        CheckCacheCountersForCompressionDict(
-            options, kNumFiles /* expected_compression_dict_misses */,
-            0 /* expected_compression_dict_hits */,
-            kNumFiles /* expected_compression_dict_inserts */);
-      }
-
-      // Seek to a key in a file. It should cause the SST's dictionary
-      // meta-block to be read.
-      RecordCacheCounters(options);
-      RecordCacheCountersForCompressionDict(options);
-      ReadOptions read_options;
-      ASSERT_NE("NOT_FOUND", Get(Key(kNumFiles * kNumEntriesPerFile - 1)));
-
-      if (dict_supported) {
-        // Two block hits: index and dictionary since they are prefetched
-        // One block missed/added: data block
-        CheckCacheCounters(options, 1 /* expected_misses */,
-                           2 /* expected_hits */, 1 /* expected_inserts */,
-                           0 /* expected_failures */);
-        CheckCacheCountersForCompressionDict(
-            options, 0 /* expected_compression_dict_misses */,
-            1 /* expected_compression_dict_hits */,
-            0 /* expected_compression_dict_inserts */);
+    for (int i = 0; i < kNumFiles; ++i) {
+      ASSERT_EQ(i, NumTableFilesAtLevel(0, 0));
+      for (int j = 0; j < kNumEntriesPerFile; ++j) {
+        std::string value = rnd.RandomString(kNumBytesPerEntry);
+        ASSERT_OK(Put(Key(j * kNumFiles + i), value.c_str()));
       }
+      ASSERT_OK(Flush());
     }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_EQ(0, NumTableFilesAtLevel(0));
+    ASSERT_EQ(kNumFiles, NumTableFilesAtLevel(1));
+
+    // Compression dictionary blocks are preloaded.
+    CheckCacheCountersForCompressionDict(
+        options, kNumFiles /* expected_compression_dict_misses */,
+        0 /* expected_compression_dict_hits */,
+        kNumFiles /* expected_compression_dict_inserts */);
+
+    // Seek to a key in a file. It should cause the SST's dictionary meta-block
+    // to be read.
+    RecordCacheCounters(options);
+    RecordCacheCountersForCompressionDict(options);
+    ReadOptions read_options;
+    ASSERT_NE("NOT_FOUND", Get(Key(kNumFiles * kNumEntriesPerFile - 1)));
+    // Two block hits: index and dictionary since they are prefetched
+    // One block missed/added: data block
+    CheckCacheCounters(options, 1 /* expected_misses */, 2 /* expected_hits */,
+                       1 /* expected_inserts */, 0 /* expected_failures */);
+    CheckCacheCountersForCompressionDict(
+        options, 0 /* expected_compression_dict_misses */,
+        1 /* expected_compression_dict_hits */,
+        0 /* expected_compression_dict_inserts */);
   }
 }
 

db/db_compaction_test.cc

@@ -11093,59 +11093,6 @@ TEST_F(DBCompactionTest, RecordNewestKeyTimeForTtlCompaction) {
   ASSERT_OK(dbfull()->TEST_WaitForCompact());
   ASSERT_EQ(NumTableFilesAtLevel(0), 0);
 }
-
-class PeriodicCompactionListener : public EventListener {
- public:
-  explicit PeriodicCompactionListener() {}
-  void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& ci) override {
-    if (ci.compaction_reason == CompactionReason::kPeriodicCompaction) {
-      ++num_periodic_compactions;
-    }
-  }
-
-  std::atomic<int> num_periodic_compactions = 0;
-};
-
-TEST_F(DBCompactionTest, PeriodicTask) {
-  // Tests that when no trigger event is fired (flush/compaction/setoptions),
-  // periodic compaction is still triggered by a scheduled periodic function.
-  auto mock_clock = std::make_shared<MockSystemClock>(env_->GetSystemClock());
-  mock_clock->SetCurrentTime(100);
-  mock_clock->InstallTimedWaitFixCallback();
-  auto mock_env = std::make_unique<CompositeEnvWrapper>(env_, mock_clock);
-  SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::StartPeriodicTaskScheduler:Init", [&](void* arg) {
-        auto periodic_task_scheduler_ptr =
-            static_cast<PeriodicTaskScheduler*>(arg);
-        periodic_task_scheduler_ptr->TEST_OverrideTimer(mock_clock.get());
-      });
-
-  Options options;
-  options.env = mock_env.get();
-  options.compaction_style = kCompactionStyleUniversal;
-  options.statistics = CreateDBStatistics();
-  int kPeriodicCompactionSeconds = 7 * 24 * 60 * 60;  // 1 week
-  options.periodic_compaction_seconds = kPeriodicCompactionSeconds;
-  options.num_levels = 50;
-  auto listener = std::make_shared<PeriodicCompactionListener>();
-  options.listeners.push_back(listener);
-  ASSERT_OK(TryReopen(options));
-
-  Random* rnd = Random::GetTLSInstance();
-  for (int k = 0; k < 10; ++k) {
-    ASSERT_OK(Put(Key(k), rnd->RandomString(100)));
-  }
-  ASSERT_OK(Flush());
-  ASSERT_OK(db_->CompactRange({}, nullptr, nullptr));
-  ASSERT_EQ(1, NumTableFilesAtLevel(49));
-
-  dbfull()->TEST_WaitForPeriodicTaskRun(
-      [&] { mock_clock->MockSleepForSeconds(kPeriodicCompactionSeconds + 1); });
-  ASSERT_OK(db_->WaitForCompact({}));
-
-  ASSERT_EQ(listener->num_periodic_compactions, 1);
-  Close();
-}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_encryption_test.cc

@@ -17,10 +17,9 @@ class DBEncryptionTest : public DBTestBase {
  public:
   DBEncryptionTest()
       : DBTestBase("db_encryption_test", /*env_do_fsync=*/true) {}
-  Env* GetNonEncryptedEnv() {
+  Env* GetTargetEnv() {
     if (encrypted_env_ != nullptr) {
-      return (static_cast_with_check<CompositeEnvWrapper>(encrypted_env_))
-          ->env_target();
+      return (static_cast<EnvWrapper*>(encrypted_env_))->target();
     } else {
       return env_;
     }
@@ -39,7 +38,7 @@ TEST_F(DBEncryptionTest, CheckEncrypted) {
   auto status = env_->GetChildren(dbname_, &fileNames);
   ASSERT_OK(status);
 
-  Env* target = GetNonEncryptedEnv();
+  Env* target = GetTargetEnv();
   int hits = 0;
   for (auto it = fileNames.begin(); it != fileNames.end(); ++it) {
     if (*it == "LOCK") {
@@ -90,7 +89,7 @@ TEST_F(DBEncryptionTest, CheckEncrypted) {
 }
 
 TEST_F(DBEncryptionTest, ReadEmptyFile) {
-  auto defaultEnv = GetNonEncryptedEnv();
+  auto defaultEnv = GetTargetEnv();
 
   // create empty file for reading it back in later
   auto envOptions = EnvOptions(CurrentOptions());
@@ -117,40 +116,6 @@ TEST_F(DBEncryptionTest, ReadEmptyFile) {
   ASSERT_TRUE(data.empty());
 }
 
-TEST_F(DBEncryptionTest, NotSupportedGetFileSize) {
-  // Validate envrypted env does not support GetFileSize.
-  // The goal of the test is to validate the encrypted env/fs does not support
-  // GetFileSize API on FSRandomAccessFile interface.
-  // This test combined with the rest of the integration tests validate that
-  // the new API GetFileSize on FSRandomAccessFile interface is not required to
-  // be supported for database to work properly.
-  // The GetFileSize API is used in ReadFooterFromFile() API to get the file
-  // size. When GetFileSize API is not supported, the ReadFooterFromFile() API
-  // will use FileSystem GetFileSize API as fallback. Refer to the
-  // EncryptedRandomAccessFile class definition for more details.
-  if (!encrypted_env_) {
-    return;
-  }
-
-  auto fs = encrypted_env_->GetFileSystem();
-
-  // create empty file for reading it back in later
-  auto filePath = dbname_ + "/empty.empty";
-
-  // Create empty file
-  CreateFile(fs.get(), filePath, "", false);
-
-  // Open it for reading footer
-  std::unique_ptr<FSRandomAccessFile> randomAccessFile;
-  auto status = fs->NewRandomAccessFile(filePath, FileOptions(),
-                                        &randomAccessFile, nullptr);
-  ASSERT_OK(status);
-
-  uint64_t fileSize;
-  status = randomAccessFile->GetFileSize(&fileSize);
-  ASSERT_TRUE(status.IsNotSupported());
-}
-
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_impl/db_impl.cc

@@ -253,9 +253,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
   periodic_task_functions_.emplace(
       PeriodicTaskType::kRecordSeqnoTime,
       [this]() { this->RecordSeqnoToTimeMapping(); });
-  periodic_task_functions_.emplace(
-      PeriodicTaskType::kTriggerCompaction,
-      [this]() { this->TriggerPeriodicCompaction(); });
 
   versions_.reset(new VersionSet(
       dbname_, &immutable_db_options_, file_options_, table_cache_.get(),
@@ -790,8 +787,7 @@ Status DBImpl::StartPeriodicTaskScheduler() {
     Status s = periodic_task_scheduler_.Register(
         PeriodicTaskType::kDumpStats,
         periodic_task_functions_.at(PeriodicTaskType::kDumpStats),
-        mutable_db_options_.stats_dump_period_sec,
-        /*run_immediately=*/true);
+        mutable_db_options_.stats_dump_period_sec);
     if (!s.ok()) {
       return s;
     }
@@ -800,8 +796,7 @@ Status DBImpl::StartPeriodicTaskScheduler() {
     Status s = periodic_task_scheduler_.Register(
         PeriodicTaskType::kPersistStats,
         periodic_task_functions_.at(PeriodicTaskType::kPersistStats),
-        mutable_db_options_.stats_persist_period_sec,
-        /*run_immediately=*/true);
+        mutable_db_options_.stats_persist_period_sec);
     if (!s.ok()) {
       return s;
     }
@@ -809,15 +804,7 @@ Status DBImpl::StartPeriodicTaskScheduler() {
 
   Status s = periodic_task_scheduler_.Register(
       PeriodicTaskType::kFlushInfoLog,
-      periodic_task_functions_.at(PeriodicTaskType::kFlushInfoLog),
-      /*run_immediately=*/true);
-
-  if (s.ok()) {
-    s = periodic_task_scheduler_.Register(
-        PeriodicTaskType::kTriggerCompaction,
-        periodic_task_functions_.at(PeriodicTaskType::kTriggerCompaction),
-        /*run_immediately=*/false);
-  }
+      periodic_task_functions_.at(PeriodicTaskType::kFlushInfoLog));
 
   return s;
 }
@@ -868,7 +855,7 @@ Status DBImpl::RegisterRecordSeqnoTimeWorker() {
     s = periodic_task_scheduler_.Register(
         PeriodicTaskType::kRecordSeqnoTime,
         periodic_task_functions_.at(PeriodicTaskType::kRecordSeqnoTime),
-        seqno_time_cadence, /*run_immediately=*/true);
+        seqno_time_cadence);
   }
 
   return s;
@@ -1378,7 +1365,7 @@ Status DBImpl::SetDBOptions(
         s = periodic_task_scheduler_.Register(
             PeriodicTaskType::kDumpStats,
             periodic_task_functions_.at(PeriodicTaskType::kDumpStats),
-            new_options.stats_dump_period_sec, /*run_immediately=*/true);
+            new_options.stats_dump_period_sec);
       }
       if (new_options.max_total_wal_size !=
           mutable_db_options_.max_total_wal_size) {
@@ -1393,7 +1380,7 @@ Status DBImpl::SetDBOptions(
           s = periodic_task_scheduler_.Register(
               PeriodicTaskType::kPersistStats,
               periodic_task_functions_.at(PeriodicTaskType::kPersistStats),
-              new_options.stats_persist_period_sec, /*run_immediately=*/true);
+              new_options.stats_persist_period_sec);
         }
       }
       mutex_.Lock();
@@ -3836,8 +3823,10 @@ bool DBImpl::KeyMayExist(const ReadOptions& read_options,
 std::unique_ptr<MultiScan> DBImpl::NewMultiScan(
     const ReadOptions& _read_options, ColumnFamilyHandle* column_family,
     const std::vector<ScanOptions>& scan_opts) {
-  std::unique_ptr<MultiScan> ms_iter = std::make_unique<MultiScan>(
-      _read_options, scan_opts, this, column_family);
+  std::unique_ptr<Iterator> iter(NewIterator(_read_options, column_family));
+  iter->Prepare(scan_opts);
+  std::unique_ptr<MultiScan> ms_iter =
+      std::make_unique<MultiScan>(scan_opts, std::move(iter));
   return ms_iter;
 }
 
@@ -5055,6 +5044,85 @@ void DBImpl::GetAllColumnFamilyMetaData(
   }
 }
 
+Status DBImpl::CheckConsistency() {
+  mutex_.AssertHeld();
+  std::vector<LiveFileMetaData> metadata;
+  versions_->GetLiveFilesMetaData(&metadata);
+  TEST_SYNC_POINT("DBImpl::CheckConsistency:AfterGetLiveFilesMetaData");
+
+  std::string corruption_messages;
+
+  if (immutable_db_options_.skip_checking_sst_file_sizes_on_db_open) {
+    // Instead of calling GetFileSize() for each expected file, call
+    // GetChildren() for the DB directory and check that all expected files
+    // are listed, without checking their sizes.
+    // Since sst files might be in different directories, do it for each
+    // directory separately.
+    std::map<std::string, std::vector<std::string>> files_by_directory;
+    for (const auto& md : metadata) {
+      // md.name has a leading "/". Remove it.
+      std::string fname = md.name;
+      if (!fname.empty() && fname[0] == '/') {
+        fname = fname.substr(1);
+      }
+      files_by_directory[md.db_path].push_back(fname);
+    }
+
+    IOOptions io_opts;
+    io_opts.do_not_recurse = true;
+    for (const auto& dir_files : files_by_directory) {
+      std::string directory = dir_files.first;
+      std::vector<std::string> existing_files;
+      Status s = fs_->GetChildren(directory, io_opts, &existing_files,
+                                  /*IODebugContext*=*/nullptr);
+      if (!s.ok()) {
+        corruption_messages +=
+            "Can't list files in " + directory + ": " + s.ToString() + "\n";
+        continue;
+      }
+      std::sort(existing_files.begin(), existing_files.end());
+
+      for (const std::string& fname : dir_files.second) {
+        if (!std::binary_search(existing_files.begin(), existing_files.end(),
+                                fname) &&
+            !std::binary_search(existing_files.begin(), existing_files.end(),
+                                Rocks2LevelTableFileName(fname))) {
+          corruption_messages +=
+              "Missing sst file " + fname + " in " + directory + "\n";
+        }
+      }
+    }
+  } else {
+    for (const auto& md : metadata) {
+      // md.name has a leading "/".
+      std::string file_path = md.db_path + md.name;
+
+      uint64_t fsize = 0;
+      TEST_SYNC_POINT("DBImpl::CheckConsistency:BeforeGetFileSize");
+      Status s = env_->GetFileSize(file_path, &fsize);
+      if (!s.ok() &&
+          env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok()) {
+        s = Status::OK();
+      }
+      if (!s.ok()) {
+        corruption_messages +=
+            "Can't access " + md.name + ": " + s.ToString() + "\n";
+      } else if (fsize != md.size) {
+        corruption_messages += "Sst file size mismatch: " + file_path +
+                               ". Size recorded in manifest " +
+                               std::to_string(md.size) + ", actual size " +
+                               std::to_string(fsize) + "\n";
+      }
+    }
+  }
+
+  if (corruption_messages.size() == 0) {
+    return Status::OK();
+  } else {
+    return Status::Corruption(corruption_messages);
+  }
+}
+
 Status DBImpl::GetDbIdentity(std::string& identity) const {
   identity.assign(db_id_);
   return Status::OK();
@@ -6816,35 +6884,6 @@ void DBImpl::RecordSeqnoToTimeMapping() {
   sv_context.Clean();
 }
 
-void DBImpl::TriggerPeriodicCompaction() {
-  TEST_SYNC_POINT("DBImpl::TriggerPeriodicCompaction:StartRunning");
-  {
-    InstrumentedMutexLock l(&mutex_);
-    ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                   "Running the periodic task to trigger compactions.");
-
-    for (ColumnFamilyData* cfd : *versions_->GetColumnFamilySet()) {
-      if (cfd->IsDropped()) {
-        continue;
-      }
-      if (cfd->GetLatestCFOptions().periodic_compaction_seconds &&
-          !cfd->queued_for_compaction()) {
-        cfd->current()->storage_info()->ComputeCompactionScore(
-            cfd->ioptions(), cfd->GetLatestMutableCFOptions());
-        EnqueuePendingCompaction(cfd);
-        if (cfd->queued_for_compaction()) {
-          ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                         "Periodic task to trigger compaction queued Column "
-                         "family [%s] for compaction.",
-                         cfd->GetName().c_str());
-        }
-      }
-    }
-    MaybeScheduleFlushOrCompaction();
-    bg_cv_.SignalAll();
-  }
-}
-
 void DBImpl::TrackOrUntrackFiles(
     const std::vector<std::string>& existing_data_files, bool track) {
   auto sfm = static_cast_with_check<SstFileManagerImpl>(

db/db_impl/db_impl.h

@@ -804,6 +804,10 @@ class DBImpl : public DB {
   // being detected.
   const Snapshot* GetSnapshotForWriteConflictBoundary();
 
+  // checks if all live files exist on file system and that their file sizes
+  // match to our in-memory records
+  virtual Status CheckConsistency();
+
   // max_file_num_to_ignore allows bottom level compaction to filter out newly
   // compacted SST files. Setting max_file_num_to_ignore to kMaxUint64 will
   // disable the filtering
@@ -1282,12 +1286,6 @@ class DBImpl : public DB {
   // For the background timer job
   void RecordSeqnoToTimeMapping();
 
-  // Compactions rely on an event triggers like flush/compaction/SetOptions.
-  // We need to trigger periodic compactions even when there is no such trigger.
-  // This function checks and schedules available compactions and will run
-  // periodically.
-  void TriggerPeriodicCompaction();
-
   // REQUIRES: DB mutex held
   std::pair<SequenceNumber, uint64_t> GetSeqnoToTimeSample() const;
 

db/db_impl/db_impl_compaction_flush.cc

@@ -1849,9 +1849,10 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
         0 /* max_subcompactions, not applicable */,
         {} /* grandparents, not applicable */,
         std::nullopt /* earliest_snapshot */, nullptr /* snapshot_checker */,
-        CompactionReason::kRefitLevel, "" /* trim_ts */,
-        -1 /* score, not applicable */,
-        false /* l0_files_might_overlap, not applicable */));
+        false /* is manual */, "" /* trim_ts */, -1 /* score, not applicable */,
+        false /* is deletion compaction, not applicable */,
+        false /* l0_files_might_overlap, not applicable */,
+        CompactionReason::kRefitLevel));
     cfd->compaction_picker()->RegisterCompaction(c.get());
     TEST_SYNC_POINT("DBImpl::ReFitLevel:PostRegisterCompaction");
     VersionEdit edit;
@@ -4423,14 +4424,13 @@ Compaction* DBImpl::CreateIntendedCompactionForwardedToBottomPriorityPool(
 
   c->ReleaseCompactionFiles(Status::OK());
 
-  Compaction* intended_compaction =
-      new Compaction(vstorage, io, mo, mutable_db_options_, std::move(inputs),
-                     c->output_level(), c->target_output_file_size(),
-                     c->max_compaction_bytes(), c->output_path_id(),
-                     c->output_compression(), c->output_compression_opts(),
-                     c->output_temperature(), c->max_subcompactions(),
-                     c->grandparents(), std::nullopt /* earliest_snapshot */,
-                     nullptr /* snapshot_checker */, c->compaction_reason());
+  Compaction* intended_compaction = new Compaction(
+      vstorage, io, mo, mutable_db_options_, std::move(inputs),
+      c->output_level(), c->target_output_file_size(),
+      c->max_compaction_bytes(), c->output_path_id(), c->output_compression(),
+      c->output_compression_opts(), c->output_temperature(),
+      c->max_subcompactions(), c->grandparents(),
+      std::nullopt /* earliest_snapshot */, nullptr /* snapshot_checker */);
 
   cfd->compaction_picker()->RegisterCompaction(intended_compaction);
   vstorage->ComputeCompactionScore(io, mo);

db/db_impl/db_impl_follower.cc

@@ -70,6 +70,9 @@ Status DBImplFollower::Recover(
     }
     return s;
   }
+  if (immutable_db_options_.paranoid_checks && s.ok()) {
+    s = CheckConsistency();
+  }
   if (s.ok()) {
     default_cf_handle_ = new ColumnFamilyHandleImpl(
         versions_->GetColumnFamilySet()->GetDefault(), this, &mutex_);

db/db_impl/db_impl_open.cc

@@ -191,6 +191,12 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src,
                    "wal_compression is disabled since only zstd is supported");
   }
 
+  if (!result.paranoid_checks) {
+    result.skip_checking_sst_file_sizes_on_db_open = true;
+    ROCKS_LOG_INFO(result.info_log,
+                   "file size check will be skipped during open.");
+  }
+
   return result;
 }
 
@@ -688,6 +694,9 @@ Status DBImpl::Recover(
     s = MaybeUpdateNextFileNumber(recovery_ctx);
   }
 
+  if (immutable_db_options_.paranoid_checks && s.ok()) {
+    s = CheckConsistency();
+  }
   if (s.ok() && !read_only) {
     // TODO: share file descriptors (FSDirectory) with SetDirectories above
     std::map<std::string, std::shared_ptr<FSDirectory>> created_dirs;

db/db_impl/db_impl_secondary.cc

@@ -49,6 +49,9 @@ Status DBImplSecondary::Recover(
     }
     return s;
   }
+  if (immutable_db_options_.paranoid_checks && s.ok()) {
+    s = CheckConsistency();
+  }
   // Initial max_total_in_memory_state_ before recovery logs.
   max_total_in_memory_state_ = 0;
   for (auto cfd : *versions_->GetColumnFamilySet()) {
@@ -650,6 +653,49 @@ Status DBImplSecondary::NewIterators(
   return Status::OK();
 }
 
+Status DBImplSecondary::CheckConsistency() {
+  mutex_.AssertHeld();
+  Status s = DBImpl::CheckConsistency();
+  // If DBImpl::CheckConsistency() which is stricter returns success, then we
+  // do not need to give a second chance.
+  if (s.ok()) {
+    return s;
+  }
+  // It's possible that DBImpl::CheckConssitency() can fail because the primary
+  // may have removed certain files, causing the GetFileSize(name) call to
+  // fail and returning a PathNotFound. In this case, we take a best-effort
+  // approach and just proceed.
+  TEST_SYNC_POINT_CALLBACK(
+      "DBImplSecondary::CheckConsistency:AfterFirstAttempt", &s);
+
+  if (immutable_db_options_.skip_checking_sst_file_sizes_on_db_open) {
+    return Status::OK();
+  }
+
+  std::vector<LiveFileMetaData> metadata;
+  versions_->GetLiveFilesMetaData(&metadata);
+
+  std::string corruption_messages;
+  for (const auto& md : metadata) {
+    // md.name has a leading "/".
+    std::string file_path = md.db_path + md.name;
+
+    uint64_t fsize = 0;
+    s = env_->GetFileSize(file_path, &fsize);
+    if (!s.ok() &&
+        (env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok() ||
+         s.IsPathNotFound())) {
+      s = Status::OK();
+    }
+    if (!s.ok()) {
+      corruption_messages +=
+          "Can't access " + md.name + ": " + s.ToString() + "\n";
+    }
+  }
+  return corruption_messages.empty() ? Status::OK()
+                                     : Status::Corruption(corruption_messages);
+}
+
 Status DBImplSecondary::TryCatchUpWithPrimary() {
   assert(versions_.get() != nullptr);
   Status s;

db/db_impl/db_impl_secondary.h

@@ -248,6 +248,12 @@ class DBImplSecondary : public DBImpl {
   Status MaybeInitLogReader(uint64_t log_number,
                             log::FragmentBufferedReader** log_reader);
 
+  // Check if all live files exist on file system and that their file sizes
+  // matche to the in-memory records. It is possible that some live files may
+  // have been deleted by the primary. In this case, CheckConsistency() does
+  // not flag the missing file as inconsistency.
+  Status CheckConsistency() override;
+
 #ifndef NDEBUG
   Status TEST_CompactWithoutInstallation(const OpenAndCompactOptions& options,
                                          ColumnFamilyHandle* cfh,

db/db_impl/db_impl_write.cc

@@ -856,7 +856,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
             write_group, current_sequence, column_family_memtables_.get(),
             &flush_scheduler_, &trim_history_scheduler_,
             write_options.ignore_missing_column_families,
-            0 /*recovery_log_number*/, this, seq_per_batch_, batch_per_txn_);
+            0 /*recovery_log_number*/, this, parallel, seq_per_batch_,
+            batch_per_txn_);
       } else {
         write_group.last_sequence = last_sequence;
         write_thread_.LaunchParallelMemTableWriters(&write_group);
@@ -1114,7 +1115,7 @@ Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
           memtable_write_group, w.sequence, column_family_memtables_.get(),
           &flush_scheduler_, &trim_history_scheduler_,
           write_options.ignore_missing_column_families, 0 /*log_number*/, this,
-          seq_per_batch_, batch_per_txn_);
+          false /*concurrent_memtable_writes*/, seq_per_batch_, batch_per_txn_);
       if (memtable_write_group.status
               .ok()) {  // Don't publish a partial batch write
         versions_->SetLastSequence(memtable_write_group.last_sequence);

db/db_iter.cc

@@ -511,6 +511,7 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
 
             valid_ = true;
             return true;
+            break;
           case kTypeMerge:
             if (!PrepareValueInternal()) {
               return false;
@@ -522,6 +523,7 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
             current_entry_is_merged_ = true;
             valid_ = true;
             return MergeValuesNewToOld();  // Go to a different state machine
+            break;
           default:
             valid_ = false;
             status_ = Status::Corruption(
@@ -1122,6 +1124,7 @@ bool DBIter::FindValueForCurrentKey() {
         }
         return true;
       }
+      break;
     case kTypeValue:
     case kTypeValuePreferredSeqno:
       SetValueAndColumnsFromPlain(pinned_value_);

db/db_iterator_test.cc

@@ -8,8 +8,6 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include <functional>
-#include <iomanip>
-#include <iostream>
 
 #include "db/arena_wrapped_db_iter.h"
 #include "db/db_iter.h"
@@ -4141,190 +4139,6 @@ TEST_P(DBIteratorTest, AverageMemtableOpsScanFlushTriggerByOverwrites) {
   ASSERT_OK(db_->WaitForCompact({}));
   ASSERT_EQ(1, NumTableFilesAtLevel(0));
 }
-
-class DBMultiScanIteratorTest : public DBTestBase {
- public:
-  DBMultiScanIteratorTest()
-      : DBTestBase("db_multi_scan_iterator_test", /*env_do_fsync=*/true) {}
-};
-
-TEST_F(DBMultiScanIteratorTest, BasicTest) {
-  // Create a file
-  for (int i = 0; i < 100; ++i) {
-    std::stringstream ss;
-    ss << std::setw(2) << std::setfill('0') << i;
-    ASSERT_OK(Put("k" + ss.str(), "val" + ss.str()));
-  }
-  ASSERT_OK(Flush());
-
-  std::vector<std::string> key_ranges({"k03", "k10", "k25", "k50"});
-  ReadOptions ro;
-  std::vector<ScanOptions> scan_options(
-      {ScanOptions(key_ranges[0], key_ranges[1]),
-       ScanOptions(key_ranges[2], key_ranges[3])});
-  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
-  std::unique_ptr<MultiScan> iter =
-      dbfull()->NewMultiScan(ro, cfh, scan_options);
-  try {
-    int idx = 0;
-    int count = 0;
-    for (auto range : *iter) {
-      for (auto it : range) {
-        ASSERT_GE(it.first.ToString().compare(key_ranges[idx]), 0);
-        ASSERT_LT(it.first.ToString().compare(key_ranges[idx + 1]), 0);
-        count++;
-      }
-      idx += 2;
-    }
-    ASSERT_EQ(count, 32);
-  } catch (MultiScanException& ex) {
-    // Make sure exception contains the status
-    ASSERT_NOK(ex.status());
-    std::cerr << "Iterator returned status " << ex.what();
-    abort();
-  } catch (std::logic_error& ex) {
-    std::cerr << "Iterator returned logic error " << ex.what();
-    abort();
-  }
-  iter.reset();
-
-  // Test the overlapping scan case
-  key_ranges[1] = "k30";
-  scan_options[0] = ScanOptions(key_ranges[0], key_ranges[1]);
-  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
-  try {
-    int idx = 0;
-    int count = 0;
-    for (auto range : *iter) {
-      for (auto it : range) {
-        ASSERT_GE(it.first.ToString().compare(key_ranges[idx]), 0);
-        ASSERT_LT(it.first.ToString().compare(key_ranges[idx + 1]), 0);
-        count++;
-      }
-      idx += 2;
-    }
-    ASSERT_EQ(count, 52);
-  } catch (MultiScanException& ex) {
-    // Make sure exception contains the status
-    ASSERT_NOK(ex.status());
-    std::cerr << "Iterator returned status " << ex.what();
-    abort();
-  } catch (std::logic_error& ex) {
-    std::cerr << "Iterator returned logic error " << ex.what();
-    abort();
-  }
-  iter.reset();
-
-  // Test the no limit scan case
-  scan_options[0] = ScanOptions(key_ranges[0]);
-  scan_options[1] = ScanOptions(key_ranges[2]);
-  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
-  try {
-    int idx = 0;
-    int count = 0;
-    for (auto range : *iter) {
-      for (auto it : range) {
-        ASSERT_GE(it.first.ToString().compare(key_ranges[idx]), 0);
-        if (it.first.ToString().compare(key_ranges[idx + 1]) == 0) {
-          break;
-        }
-        count++;
-      }
-      idx += 2;
-    }
-    ASSERT_EQ(count, 52);
-  } catch (MultiScanException& ex) {
-    // Make sure exception contains the status
-    ASSERT_NOK(ex.status());
-    std::cerr << "Iterator returned status " << ex.what();
-    abort();
-  } catch (std::logic_error& ex) {
-    std::cerr << "Iterator returned logic error " << ex.what();
-    abort();
-  }
-  iter.reset();
-}
-
-TEST_F(DBMultiScanIteratorTest, MixedBoundsTest) {
-  // Create a file
-  for (int i = 0; i < 100; ++i) {
-    std::stringstream ss;
-    ss << std::setw(2) << std::setfill('0') << i;
-    ASSERT_OK(Put("k" + ss.str(), "val" + ss.str()));
-  }
-  ASSERT_OK(Flush());
-
-  std::vector<std::string> key_ranges(
-      {"k03", "k10", "k25", "k50", "k75", "k90"});
-  ReadOptions ro;
-  std::vector<ScanOptions> scan_options(
-      {ScanOptions(key_ranges[0], key_ranges[1]), ScanOptions(key_ranges[2]),
-       ScanOptions(key_ranges[4], key_ranges[5])});
-  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
-  std::unique_ptr<MultiScan> iter =
-      dbfull()->NewMultiScan(ro, cfh, scan_options);
-  try {
-    int idx = 0;
-    int count = 0;
-    for (auto range : *iter) {
-      for (auto it : range) {
-        ASSERT_GE(it.first.ToString().compare(
-                      scan_options[idx].range.start->ToString()),
-                  0);
-        if (scan_options[idx].range.limit) {
-          ASSERT_LT(it.first.ToString().compare(
-                        scan_options[idx].range.limit->ToString()),
-                    0);
-        }
-        count++;
-      }
-      idx++;
-    }
-    ASSERT_EQ(count, 97);
-  } catch (MultiScanException& ex) {
-    // Make sure exception contains the status
-    ASSERT_NOK(ex.status());
-    std::cerr << "Iterator returned status " << ex.what();
-    abort();
-  } catch (std::logic_error& ex) {
-    std::cerr << "Iterator returned logic error " << ex.what();
-    abort();
-  }
-  iter.reset();
-
-  scan_options[0] = ScanOptions(key_ranges[0]);
-  scan_options[1] = ScanOptions(key_ranges[2], key_ranges[3]);
-  scan_options[2] = ScanOptions(key_ranges[4]);
-  iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
-  try {
-    int idx = 0;
-    int count = 0;
-    for (auto range : *iter) {
-      for (auto it : range) {
-        ASSERT_GE(it.first.ToString().compare(
-                      scan_options[idx].range.start->ToString()),
-                  0);
-        if (scan_options[idx].range.limit) {
-          ASSERT_LT(it.first.ToString().compare(
-                        scan_options[idx].range.limit->ToString()),
-                    0);
-        }
-        count++;
-      }
-      idx++;
-    }
-    ASSERT_EQ(count, 147);
-  } catch (MultiScanException& ex) {
-    // Make sure exception contains the status
-    ASSERT_NOK(ex.status());
-    std::cerr << "Iterator returned status " << ex.what();
-    abort();
-  } catch (std::logic_error& ex) {
-    std::cerr << "Iterator returned logic error " << ex.what();
-    abort();
-  }
-  iter.reset();
-}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_memtable_test.cc

@@ -424,96 +424,6 @@ TEST_F(DBMemTableTest, IntegrityChecks) {
     ASSERT_FALSE(iter->Valid());
   }
 }
-
-TEST_F(DBMemTableTest, VectorConcurrentInsert) {
-  Options options;
-  options.create_if_missing = true;
-  options.create_missing_column_families = true;
-  options.allow_concurrent_memtable_write = true;
-  options.memtable_factory.reset(new VectorRepFactory());
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"cf1"}, options);
-
-  // Multi-threaded writes
-  {
-    WriteOptions write_options;
-    std::vector<port::Thread> threads;
-    for (int i = 0; i < 10; ++i) {
-      threads.emplace_back([&, i]() {
-        int start = i * 100;
-        int end = start + 100;
-        WriteBatch batch;
-        for (int j = start; j < end; ++j) {
-          ASSERT_OK(
-              batch.Put(handles_[0], Key(j), "value" + std::to_string(j)));
-        }
-        ASSERT_OK(db_->Write(write_options, &batch));
-      });
-    }
-    for (auto& t : threads) {
-      t.join();
-    }
-
-    std::unique_ptr<Iterator> iter(
-        db_->NewIterator(ReadOptions(), handles_[0]));
-    iter->SeekToFirst();
-    for (int i = 0; i < 1000; ++i) {
-      ASSERT_TRUE(iter->Valid());
-      ASSERT_EQ(iter->key().ToString(), Key(i));
-      ASSERT_EQ(iter->value().ToString(), "value" + std::to_string(i));
-      iter->Next();
-    }
-    ASSERT_FALSE(iter->Valid());
-    ASSERT_OK(iter->status());
-  }
-
-  // Multi-threaded writes, multi CF
-  {
-    WriteOptions write_options;
-    std::vector<port::Thread> threads;
-    for (int i = 0; i < 10; ++i) {
-      threads.emplace_back([&, i]() {
-        int start = i * 100;
-        int end = start + 100;
-        WriteBatch batch;
-        for (int j = start; j < end; ++j) {
-          ASSERT_OK(batch.Put(handles_[0], Key(j), "CF0" + std::to_string(j)));
-          ASSERT_OK(batch.Put(handles_[1], Key(j), "CF1" + std::to_string(j)));
-        }
-        ASSERT_OK(db_->Write(write_options, &batch));
-      });
-    }
-
-    for (auto& t : threads) {
-      t.join();
-    }
-
-    std::unique_ptr<Iterator> iter0(
-        db_->NewIterator(ReadOptions(), handles_[0]));
-    std::unique_ptr<Iterator> iter1(
-        db_->NewIterator(ReadOptions(), handles_[1]));
-    iter0->SeekToFirst();
-    iter1->SeekToFirst();
-    for (int i = 0; i < 1000; ++i) {
-      ASSERT_TRUE(iter0->Valid());
-      ASSERT_EQ(iter0->key().ToString(), Key(i));
-      ASSERT_EQ(iter0->value().ToString(), "CF0" + std::to_string(i));
-      iter0->Next();
-
-      ASSERT_TRUE(iter1->Valid());
-      ASSERT_EQ(iter1->key().ToString(), Key(i));
-      ASSERT_EQ(iter1->value().ToString(), "CF1" + std::to_string(i));
-      iter1->Next();
-    }
-    ASSERT_FALSE(iter0->Valid());
-    ASSERT_OK(iter0->status());
-    ASSERT_FALSE(iter1->Valid());
-    ASSERT_OK(iter1->status());
-  }
-
-  ASSERT_OK(Flush(0));
-  ASSERT_OK(Flush(1));
-}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_secondary_test.cc

@@ -605,9 +605,6 @@ class TraceFileEnv : public EnvWrapper {
                   char* scratch) const override {
         return target_->Read(offset, n, result, scratch);
       }
-      Status GetFileSize(uint64_t* file_size) override {
-        return target_->GetFileSize(file_size);
-      }
 
      private:
       std::unique_ptr<RandomAccessFile> target_;
@@ -1294,6 +1291,46 @@ TEST_F(DBSecondaryTest, CatchUpAfterFlush) {
   ASSERT_OK(iter3->status());
 }
 
+TEST_F(DBSecondaryTest, CheckConsistencyWhenOpen) {
+  bool called = false;
+  Options options;
+  options.env = env_;
+  options.disable_auto_compactions = true;
+  Reopen(options);
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImplSecondary::CheckConsistency:AfterFirstAttempt", [&](void* arg) {
+        ASSERT_NE(nullptr, arg);
+        called = true;
+        auto* s = static_cast<Status*>(arg);
+        ASSERT_NOK(*s);
+      });
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::CheckConsistency:AfterGetLiveFilesMetaData",
+        "BackgroundCallCompaction:0"},
+       {"DBImpl::BackgroundCallCompaction:PurgedObsoleteFiles",
+        "DBImpl::CheckConsistency:BeforeGetFileSize"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(Put("a", "value0"));
+  ASSERT_OK(Put("c", "value0"));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Put("b", "value1"));
+  ASSERT_OK(Put("d", "value1"));
+  ASSERT_OK(Flush());
+  port::Thread thread([this]() {
+    Options opts;
+    opts.env = env_;
+    opts.max_open_files = -1;
+    OpenSecondary(opts);
+  });
+  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  thread.join();
+  ASSERT_TRUE(called);
+}
+
 TEST_F(DBSecondaryTest, StartFromInconsistent) {
   Options options = CurrentOptions();
   DestroyAndReopen(options);

db/db_sst_test.cc

@@ -135,6 +135,21 @@ TEST_F(DBSSTTest, SSTsWithLdbSuffixHandling) {
   Destroy(options);
 }
 
+// Check that we don't crash when opening DB with
+// DBOptions::skip_checking_sst_file_sizes_on_db_open = true.
+TEST_F(DBSSTTest, SkipCheckingSSTFileSizesOnDBOpen) {
+  ASSERT_OK(Put("pika", "choo"));
+  ASSERT_OK(Flush());
+
+  // Just open the DB with the option set to true and check that we don't crash.
+  Options options;
+  options.env = env_;
+  options.skip_checking_sst_file_sizes_on_db_open = true;
+  Reopen(options);
+
+  ASSERT_EQ("choo", Get("pika"));
+}
+
 TEST_F(DBSSTTest, DontDeleteMovedFile) {
   // This test triggers move compaction and verifies that the file is not
   // deleted when it's part of move compaction
@@ -1733,6 +1748,45 @@ TEST_F(DBSSTTest, GetTotalSstFilesSize) {
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
 
+TEST_F(DBSSTTest, OpenDBWithoutGetFileSizeInvocations) {
+  Options options = CurrentOptions();
+  std::unique_ptr<MockEnv> env{MockEnv::Create(Env::Default())};
+  options.env = env.get();
+  options.disable_auto_compactions = true;
+  options.compression = kNoCompression;
+  options.enable_blob_files = true;
+  options.blob_file_size = 32;  // create one blob per file
+  options.skip_checking_sst_file_sizes_on_db_open = true;
+
+  DestroyAndReopen(options);
+  // Generate 5 files in L0
+  for (int i = 0; i < 5; i++) {
+    for (int j = 0; j < 10; j++) {
+      std::string val = "val_file_" + std::to_string(i);
+      ASSERT_OK(Put(Key(j), val));
+    }
+    ASSERT_OK(Flush());
+  }
+  Close();
+
+  bool is_get_file_size_called = false;
+  SyncPoint::GetInstance()->SetCallBack(
+      "MockFileSystem::GetFileSize:CheckFileType", [&](void* arg) {
+        std::string* filename = static_cast<std::string*>(arg);
+        if (filename->find(".blob") != std::string::npos) {
+          is_get_file_size_called = true;
+        }
+      });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+  Reopen(options);
+  ASSERT_FALSE(is_get_file_size_called);
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  Destroy(options);
+}
+
 TEST_F(DBSSTTest, GetTotalSstFilesSizeVersionsFilesShared) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;

db/db_test.cc

@@ -1278,6 +1278,12 @@ class DelayFilterFactory : public CompactionFilterFactory {
 };
 }  // anonymous namespace
 
+static std::string CompressibleString(Random* rnd, int len) {
+  std::string r;
+  test::CompressibleString(rnd, 0.8, len, &r);
+  return r;
+}
+
 TEST_F(DBTest, FailMoreDbPaths) {
   Options options = CurrentOptions();
   options.db_paths.emplace_back(dbname_, 10000000);
@@ -5401,6 +5407,271 @@ TEST_F(DBTest, FlushOnDestroy) {
   CancelAllBackgroundWork(db_);
 }
 
+TEST_F(DBTest, DynamicLevelCompressionPerLevel) {
+  if (!Snappy_Supported()) {
+    return;
+  }
+  const int kNKeys = 120;
+  int keys[kNKeys];
+  for (int i = 0; i < kNKeys; i++) {
+    keys[i] = i;
+  }
+
+  Random rnd(301);
+  Options options;
+  options.env = env_;
+  options.create_if_missing = true;
+  options.db_write_buffer_size = 20480;
+  options.write_buffer_size = 20480;
+  options.max_write_buffer_number = 2;
+  options.level0_file_num_compaction_trigger = 2;
+  options.level0_slowdown_writes_trigger = 2;
+  options.level0_stop_writes_trigger = 2;
+  options.target_file_size_base = 20480;
+  options.level_compaction_dynamic_level_bytes = true;
+  options.max_bytes_for_level_base = 102400;
+  options.max_bytes_for_level_multiplier = 4;
+  options.max_background_compactions = 1;
+  options.num_levels = 5;
+  options.statistics = CreateDBStatistics();
+
+  options.compression_per_level.resize(3);
+  // No compression for L0
+  options.compression_per_level[0] = kNoCompression;
+  // No compression for the Ln whre L0 is compacted to
+  options.compression_per_level[1] = kNoCompression;
+  // Snappy compression for Ln+1
+  options.compression_per_level[2] = kSnappyCompression;
+
+  OnFileDeletionListener* listener = new OnFileDeletionListener();
+  options.listeners.emplace_back(listener);
+
+  DestroyAndReopen(options);
+
+  // Insert more than 80K. L4 should be base level. Neither L0 nor L4 should
+  // be compressed, so there shouldn't be any compression.
+  for (int i = 0; i < 20; i++) {
+    ASSERT_OK(Put(Key(keys[i]), CompressibleString(&rnd, 4000)));
+    ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+  }
+  ASSERT_OK(Flush());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
+  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
+  ASSERT_EQ(NumTableFilesAtLevel(3), 0);
+  ASSERT_TRUE(NumTableFilesAtLevel(0) > 0 || NumTableFilesAtLevel(4) > 0);
+
+  // Verify there was no compression
+  auto num_block_compressed =
+      options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
+  ASSERT_EQ(num_block_compressed, 0);
+
+  // Insert 400KB and there will be some files end up in L3. According to the
+  // above compression settings for each level, there will be some compression.
+  ASSERT_OK(options.statistics->Reset());
+  ASSERT_EQ(num_block_compressed, 0);
+  for (int i = 20; i < 120; i++) {
+    ASSERT_OK(Put(Key(keys[i]), CompressibleString(&rnd, 4000)));
+    ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+  }
+  ASSERT_OK(Flush());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
+  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
+  ASSERT_GE(NumTableFilesAtLevel(3), 1);
+  ASSERT_GE(NumTableFilesAtLevel(4), 1);
+
+  // Verify there was compression
+  num_block_compressed =
+      options.statistics->getTickerCount(NUMBER_BLOCK_COMPRESSED);
+  ASSERT_GT(num_block_compressed, 0);
+
+  // Make sure data in files in L3 is not compacted by removing all files
+  // in L4 and calculate number of rows
+  ASSERT_OK(dbfull()->SetOptions({
+      {"disable_auto_compactions", "true"},
+  }));
+  ColumnFamilyMetaData cf_meta;
+  db_->GetColumnFamilyMetaData(&cf_meta);
+
+  // Ensure that L1+ files are non-overlapping and together with L0 encompass
+  // full key range between smallestkey and largestkey from CF file metadata.
+  int largestkey_in_prev_level = -1;
+  int keys_found = 0;
+  for (int level = (int)cf_meta.levels.size() - 1; level >= 0; level--) {
+    int files_in_level = (int)cf_meta.levels[level].files.size();
+    int largestkey_in_prev_file = -1;
+    for (int j = 0; j < files_in_level; j++) {
+      int smallestkey = IdFromKey(cf_meta.levels[level].files[j].smallestkey);
+      int largestkey = IdFromKey(cf_meta.levels[level].files[j].largestkey);
+      int num_entries = (int)cf_meta.levels[level].files[j].num_entries;
+      ASSERT_EQ(num_entries, largestkey - smallestkey + 1);
+      keys_found += num_entries;
+      if (level > 0) {
+        if (j == 0) {
+          ASSERT_GT(smallestkey, largestkey_in_prev_level);
+        }
+        if (j > 0) {
+          ASSERT_GT(smallestkey, largestkey_in_prev_file);
+        }
+        if (j == files_in_level - 1) {
+          largestkey_in_prev_level = largestkey;
+        }
+      }
+      largestkey_in_prev_file = largestkey;
+    }
+  }
+  ASSERT_EQ(keys_found, kNKeys);
+
+  for (const auto& file : cf_meta.levels[4].files) {
+    listener->SetExpectedFileName(dbname_ + file.name);
+    const RangeOpt ranges(file.smallestkey, file.largestkey);
+    // Given verification from above, we're guaranteed that by deleting all the
+    // files in [<smallestkey>, <largestkey>] range, we're effectively deleting
+    // that very single file and nothing more.
+    EXPECT_OK(dbfull()->DeleteFilesInRanges(dbfull()->DefaultColumnFamily(),
+                                            &ranges, true /* include_end */));
+  }
+  listener->VerifyMatchedCount(cf_meta.levels[4].files.size());
+
+  int num_keys = 0;
+  std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    num_keys++;
+  }
+  ASSERT_OK(iter->status());
+
+  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
+  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
+  ASSERT_GE(NumTableFilesAtLevel(3), 1);
+  ASSERT_EQ(NumTableFilesAtLevel(4), 0);
+
+  ASSERT_GT(SizeAtLevel(0) + SizeAtLevel(3), num_keys * 4000U + num_keys * 10U);
+}
+
+TEST_F(DBTest, DynamicLevelCompressionPerLevel2) {
+  if (!Snappy_Supported() || !LZ4_Supported() || !Zlib_Supported()) {
+    return;
+  }
+  const int kNKeys = 500;
+  int keys[kNKeys];
+  for (int i = 0; i < kNKeys; i++) {
+    keys[i] = i;
+  }
+  RandomShuffle(std::begin(keys), std::end(keys));
+
+  Random rnd(301);
+  Options options;
+  options.create_if_missing = true;
+  options.db_write_buffer_size = 6000000;
+  options.write_buffer_size = 600000;
+  options.max_write_buffer_number = 2;
+  options.level0_file_num_compaction_trigger = 2;
+  options.level0_slowdown_writes_trigger = 2;
+  options.level0_stop_writes_trigger = 2;
+  options.soft_pending_compaction_bytes_limit = 1024 * 1024;
+  options.target_file_size_base = 20;
+  options.env = env_;
+  options.level_compaction_dynamic_level_bytes = true;
+  options.max_bytes_for_level_base = 200;
+  options.max_bytes_for_level_multiplier = 8;
+  options.max_background_compactions = 1;
+  options.num_levels = 5;
+  std::shared_ptr<mock::MockTableFactory> mtf(new mock::MockTableFactory);
+  options.table_factory = mtf;
+
+  options.compression_per_level.resize(3);
+  options.compression_per_level[0] = kNoCompression;
+  options.compression_per_level[1] = kLZ4Compression;
+  options.compression_per_level[2] = kZlibCompression;
+
+  DestroyAndReopen(options);
+  // When base level is L4, L4 is LZ4.
+  std::atomic<int> num_zlib(0);
+  std::atomic<int> num_lz4(0);
+  std::atomic<int> num_no(0);
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
+        Compaction* compaction = static_cast<Compaction*>(arg);
+        if (compaction->output_level() == 4) {
+          ASSERT_TRUE(compaction->output_compression() == kLZ4Compression);
+          num_lz4.fetch_add(1);
+        }
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table:output_compression", [&](void* arg) {
+        auto* compression = static_cast<CompressionType*>(arg);
+        ASSERT_TRUE(*compression == kNoCompression);
+        num_no.fetch_add(1);
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  for (int i = 0; i < 100; i++) {
+    std::string value = rnd.RandomString(200);
+    ASSERT_OK(Put(Key(keys[i]), value));
+    if (i % 25 == 24) {
+      ASSERT_OK(Flush());
+      ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    }
+  }
+
+  ASSERT_OK(Flush());
+  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
+  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
+  ASSERT_EQ(NumTableFilesAtLevel(3), 0);
+  ASSERT_GT(NumTableFilesAtLevel(4), 0);
+  ASSERT_GT(num_no.load(), 2);
+  ASSERT_GT(num_lz4.load(), 0);
+  int prev_num_files_l4 = NumTableFilesAtLevel(4);
+
+  // After base level turn L4->L3, L3 becomes LZ4 and L4 becomes Zlib
+  num_lz4.store(0);
+  num_no.store(0);
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "LevelCompactionPicker::PickCompaction:Return", [&](void* arg) {
+        Compaction* compaction = static_cast<Compaction*>(arg);
+        if (compaction->output_level() == 4 && compaction->start_level() == 3) {
+          ASSERT_TRUE(compaction->output_compression() == kZlibCompression);
+          num_zlib.fetch_add(1);
+        } else {
+          ASSERT_TRUE(compaction->output_compression() == kLZ4Compression);
+          num_lz4.fetch_add(1);
+        }
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "FlushJob::WriteLevel0Table:output_compression", [&](void* arg) {
+        auto* compression = static_cast<CompressionType*>(arg);
+        ASSERT_TRUE(*compression == kNoCompression);
+        num_no.fetch_add(1);
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  for (int i = 101; i < 500; i++) {
+    std::string value = rnd.RandomString(200);
+    ASSERT_OK(Put(Key(keys[i]), value));
+    if (i % 100 == 99) {
+      ASSERT_OK(Flush());
+      ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    }
+  }
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ASSERT_EQ(NumTableFilesAtLevel(1), 0);
+  ASSERT_EQ(NumTableFilesAtLevel(2), 0);
+  ASSERT_GT(NumTableFilesAtLevel(3), 0);
+  ASSERT_GT(NumTableFilesAtLevel(4), prev_num_files_l4);
+  ASSERT_GT(num_no.load(), 2);
+  ASSERT_GT(num_lz4.load(), 0);
+  ASSERT_GT(num_zlib.load(), 0);
+}
+
 TEST_F(DBTest, DynamicCompactionOptions) {
   // minimum write buffer size is enforced at 64KB
   const uint64_t k32KB = 1 << 15;
@@ -5938,8 +6209,8 @@ TEST_F(DBTest, L0L1L2AndUpHitCounter) {
 }
 
 TEST_F(DBTest, EncodeDecompressedBlockSizeTest) {
-  // Allow testing format_version=1
-  bool& allow_unsupported_fv = TEST_AllowUnsupportedFormatVersion();
+  bool& allow_unsupported_fv =
+      BlockBasedTableFactory::AllowUnsupportedFormatVersion();
   SaveAndRestore guard(&allow_unsupported_fv);
   ASSERT_FALSE(allow_unsupported_fv);
 
@@ -7078,6 +7349,27 @@ TEST_F(DBTest, LastWriteBufferDelay) {
 }
 #endif  // !defined(ROCKSDB_DISABLE_STALL_NOTIFICATION)
 
+TEST_F(DBTest, FailWhenCompressionNotSupportedTest) {
+  CompressionType compressions[] = {kZlibCompression, kBZip2Compression,
+                                    kLZ4Compression, kLZ4HCCompression,
+                                    kXpressCompression};
+  for (auto comp : compressions) {
+    if (!CompressionTypeSupported(comp)) {
+      // not supported, we should fail the Open()
+      Options options = CurrentOptions();
+      options.compression = comp;
+      ASSERT_TRUE(!TryReopen(options).ok());
+      // Try if CreateColumnFamily also fails
+      options.compression = kNoCompression;
+      ASSERT_OK(TryReopen(options));
+      ColumnFamilyOptions cf_options(options);
+      cf_options.compression = comp;
+      ColumnFamilyHandle* handle;
+      ASSERT_TRUE(!db_->CreateColumnFamily(cf_options, "name", &handle).ok());
+    }
+  }
+}
+
 TEST_F(DBTest, CreateColumnFamilyShouldFailOnIncompatibleOptions) {
   Options options = CurrentOptions();
   options.max_open_files = 100;

db/db_test2.cc

@@ -29,6 +29,7 @@
 #include "test_util/testutil.h"
 #include "util/defer.h"
 #include "util/random.h"
+#include "util/simple_mixed_compressor.h"
 #include "utilities/fault_injection_env.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -1185,6 +1186,922 @@ TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
   ASSERT_EQ(index, keys_cf.size());
 }
 
+TEST_F(DBTest2, PresetCompressionDict) {
+  // Verifies that compression ratio improves when dictionary is enabled, and
+  // improves even further when the dictionary is trained by ZSTD.
+  const size_t kBlockSizeBytes = 4 << 10;
+  const size_t kL0FileBytes = 128 << 10;
+  const size_t kApproxPerBlockOverheadBytes = 50;
+  const int kNumL0Files = 5;
+
+  Options options;
+  // Make sure to use any custom env that the test is configured with.
+  options.env = CurrentOptions().env;
+  options.allow_concurrent_memtable_write = false;
+  options.arena_block_size = kBlockSizeBytes;
+  options.create_if_missing = true;
+  options.disable_auto_compactions = true;
+  options.level0_file_num_compaction_trigger = kNumL0Files;
+  options.memtable_factory.reset(
+      test::NewSpecialSkipListFactory(kL0FileBytes / kBlockSizeBytes));
+  options.num_levels = 2;
+  options.target_file_size_base = kL0FileBytes;
+  options.target_file_size_multiplier = 2;
+  options.write_buffer_size = kL0FileBytes;
+  BlockBasedTableOptions table_options;
+  table_options.block_size = kBlockSizeBytes;
+  std::vector<CompressionType> compression_types;
+  if (Zlib_Supported()) {
+    compression_types.push_back(kZlibCompression);
+  }
+#if LZ4_VERSION_NUMBER >= 10400  // r124+
+  compression_types.push_back(kLZ4Compression);
+  compression_types.push_back(kLZ4HCCompression);
+#endif  // LZ4_VERSION_NUMBER >= 10400
+  if (ZSTD_Supported()) {
+    compression_types.push_back(kZSTD);
+  }
+
+  enum DictionaryTypes : int {
+    kWithoutDict,
+    kWithDict,
+    kWithZSTDfinalizeDict,
+    kWithZSTDTrainedDict,
+    kDictEnd,
+  };
+
+  for (auto compression_type : compression_types) {
+    options.compression = compression_type;
+    size_t bytes_without_dict = 0;
+    size_t bytes_with_dict = 0;
+    size_t bytes_with_zstd_finalize_dict = 0;
+    size_t bytes_with_zstd_trained_dict = 0;
+    for (int i = kWithoutDict; i < kDictEnd; i++) {
+      // First iteration: compress without preset dictionary
+      // Second iteration: compress with preset dictionary
+      // Third iteration (zstd only): compress with zstd-trained dictionary
+      //
+      // To make sure the compression dictionary has the intended effect, we
+      // verify the compressed size is smaller in successive iterations. Also in
+      // the non-first iterations, verify the data we get out is the same data
+      // we put in.
+      switch (i) {
+        case kWithoutDict:
+          options.compression_opts.max_dict_bytes = 0;
+          options.compression_opts.zstd_max_train_bytes = 0;
+          break;
+        case kWithDict:
+          options.compression_opts.max_dict_bytes = kBlockSizeBytes;
+          options.compression_opts.zstd_max_train_bytes = 0;
+          break;
+        case kWithZSTDfinalizeDict:
+          if (compression_type != kZSTD ||
+              !ZSTD_FinalizeDictionarySupported()) {
+            continue;
+          }
+          options.compression_opts.max_dict_bytes = kBlockSizeBytes;
+          options.compression_opts.zstd_max_train_bytes = kL0FileBytes;
+          options.compression_opts.use_zstd_dict_trainer = false;
+          break;
+        case kWithZSTDTrainedDict:
+          if (compression_type != kZSTD || !ZSTD_TrainDictionarySupported()) {
+            continue;
+          }
+          options.compression_opts.max_dict_bytes = kBlockSizeBytes;
+          options.compression_opts.zstd_max_train_bytes = kL0FileBytes;
+          options.compression_opts.use_zstd_dict_trainer = true;
+          break;
+        default:
+          assert(false);
+      }
+
+      options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+      options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+      CreateAndReopenWithCF({"pikachu"}, options);
+      Random rnd(301);
+      std::string seq_datas[10];
+      for (int j = 0; j < 10; ++j) {
+        seq_datas[j] =
+            rnd.RandomString(kBlockSizeBytes - kApproxPerBlockOverheadBytes);
+      }
+
+      ASSERT_EQ(0, NumTableFilesAtLevel(0, 1));
+      for (int j = 0; j < kNumL0Files; ++j) {
+        for (size_t k = 0; k < kL0FileBytes / kBlockSizeBytes + 1; ++k) {
+          auto key_num = j * (kL0FileBytes / kBlockSizeBytes) + k;
+          ASSERT_OK(Put(1, Key(static_cast<int>(key_num)),
+                        seq_datas[(key_num / 10) % 10]));
+        }
+        ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[1]));
+        ASSERT_EQ(j + 1, NumTableFilesAtLevel(0, 1));
+      }
+      ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1],
+                                            true /* disallow_trivial_move */));
+      ASSERT_EQ(0, NumTableFilesAtLevel(0, 1));
+      ASSERT_GT(NumTableFilesAtLevel(1, 1), 0);
+
+      // Get the live sst files size
+      size_t total_sst_bytes = TotalSize(1);
+      if (i == kWithoutDict) {
+        bytes_without_dict = total_sst_bytes;
+      } else if (i == kWithDict) {
+        bytes_with_dict = total_sst_bytes;
+      } else if (i == kWithZSTDfinalizeDict) {
+        bytes_with_zstd_finalize_dict = total_sst_bytes;
+      } else if (i == kWithZSTDTrainedDict) {
+        bytes_with_zstd_trained_dict = total_sst_bytes;
+      }
+
+      for (size_t j = 0; j < kNumL0Files * (kL0FileBytes / kBlockSizeBytes);
+           j++) {
+        ASSERT_EQ(seq_datas[(j / 10) % 10], Get(1, Key(static_cast<int>(j))));
+      }
+      if (i == kWithDict) {
+        ASSERT_GT(bytes_without_dict, bytes_with_dict);
+      } else if (i == kWithZSTDTrainedDict) {
+        // In zstd compression, it is sometimes possible that using a finalized
+        // dictionary does not get as good a compression ratio as raw content
+        // dictionary. But using a dictionary should always get better
+        // compression ratio than not using one.
+        ASSERT_TRUE(bytes_with_dict > bytes_with_zstd_finalize_dict ||
+                    bytes_without_dict > bytes_with_zstd_finalize_dict);
+      } else if (i == kWithZSTDTrainedDict) {
+        // In zstd compression, it is sometimes possible that using a trained
+        // dictionary does not get as good a compression ratio as without
+        // training.
+        // But using a dictionary (with or without training) should always get
+        // better compression ratio than not using one.
+        ASSERT_TRUE(bytes_with_dict > bytes_with_zstd_trained_dict ||
+                    bytes_without_dict > bytes_with_zstd_trained_dict);
+      }
+
+      DestroyAndReopen(options);
+    }
+  }
+}
+
+TEST_F(DBTest2, PresetCompressionDictLocality) {
+  if (!ZSTD_Supported()) {
+    return;
+  }
+  // Verifies that compression dictionary is generated from local data. The
+  // verification simply checks all output SSTs have different compression
+  // dictionaries. We do not verify effectiveness as that'd likely be flaky in
+  // the future.
+  const int kNumEntriesPerFile = 1 << 10;  // 1KB
+  const int kNumBytesPerEntry = 1 << 10;   // 1KB
+  const int kNumFiles = 4;
+  Options options = CurrentOptions();
+  options.compression = kZSTD;
+  options.compression_opts.max_dict_bytes = 1 << 14;        // 16KB
+  options.compression_opts.zstd_max_train_bytes = 1 << 18;  // 256KB
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+  options.target_file_size_base = kNumEntriesPerFile * kNumBytesPerEntry;
+  BlockBasedTableOptions table_options;
+  table_options.cache_index_and_filter_blocks = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  Reopen(options);
+
+  Random rnd(301);
+  for (int i = 0; i < kNumFiles; ++i) {
+    for (int j = 0; j < kNumEntriesPerFile; ++j) {
+      ASSERT_OK(Put(Key(i * kNumEntriesPerFile + j),
+                    rnd.RandomString(kNumBytesPerEntry)));
+    }
+    ASSERT_OK(Flush());
+    MoveFilesToLevel(1);
+    ASSERT_EQ(NumTableFilesAtLevel(1), i + 1);
+  }
+
+  // Store all the dictionaries generated during a full compaction.
+  std::vector<std::string> compression_dicts;
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "BlockBasedTableBuilder::WriteCompressionDictBlock:RawDict",
+      [&](void* arg) {
+        compression_dicts.emplace_back(static_cast<Slice*>(arg)->ToString());
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+  CompactRangeOptions compact_range_opts;
+  compact_range_opts.bottommost_level_compaction =
+      BottommostLevelCompaction::kForceOptimized;
+  ASSERT_OK(db_->CompactRange(compact_range_opts, nullptr, nullptr));
+
+  // Dictionary compression should not be so good as to compress four totally
+  // random files into one. If it does then there's probably something wrong
+  // with the test.
+  ASSERT_GT(NumTableFilesAtLevel(1), 1);
+
+  // Furthermore, there should be one compression dictionary generated per file.
+  // And they should all be different from each other.
+  ASSERT_EQ(NumTableFilesAtLevel(1),
+            static_cast<int>(compression_dicts.size()));
+  for (size_t i = 1; i < compression_dicts.size(); ++i) {
+    std::string& a = compression_dicts[i - 1];
+    std::string& b = compression_dicts[i];
+    size_t alen = a.size();
+    size_t blen = b.size();
+    ASSERT_TRUE(alen != blen || memcmp(a.data(), b.data(), alen) != 0);
+  }
+}
+
+class PresetCompressionDictTest
+    : public DBTestBase,
+      public testing::WithParamInterface<std::tuple<CompressionType, bool>> {
+ public:
+  PresetCompressionDictTest()
+      : DBTestBase("db_test2", false /* env_do_fsync */),
+        compression_type_(std::get<0>(GetParam())),
+        bottommost_(std::get<1>(GetParam())) {}
+
+ protected:
+  const CompressionType compression_type_;
+  const bool bottommost_;
+};
+
+INSTANTIATE_TEST_CASE_P(
+    DBTest2, PresetCompressionDictTest,
+    ::testing::Combine(::testing::ValuesIn(GetSupportedDictCompressions()),
+                       ::testing::Bool()));
+
+TEST_P(PresetCompressionDictTest, Flush) {
+  // Verifies that dictionary is generated and written during flush only when
+  // `ColumnFamilyOptions::compression` enables dictionary. Also verifies the
+  // size of the dictionary is within expectations according to the limit on
+  // buffering set by `CompressionOptions::max_dict_buffer_bytes`.
+  const size_t kValueLen = 256;
+  const size_t kKeysPerFile = 1 << 10;
+  const size_t kDictLen = 16 << 10;
+  const size_t kBlockLen = 4 << 10;
+
+  Options options = CurrentOptions();
+  if (bottommost_) {
+    options.bottommost_compression = compression_type_;
+    options.bottommost_compression_opts.enabled = true;
+    options.bottommost_compression_opts.max_dict_bytes = kDictLen;
+    options.bottommost_compression_opts.max_dict_buffer_bytes = kBlockLen;
+  } else {
+    options.compression = compression_type_;
+    options.compression_opts.max_dict_bytes = kDictLen;
+    options.compression_opts.max_dict_buffer_bytes = kBlockLen;
+  }
+  options.memtable_factory.reset(test::NewSpecialSkipListFactory(kKeysPerFile));
+  options.statistics = CreateDBStatistics();
+  BlockBasedTableOptions bbto;
+  bbto.block_size = kBlockLen;
+  bbto.cache_index_and_filter_blocks = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  Reopen(options);
+
+  Random rnd(301);
+  for (size_t i = 0; i <= kKeysPerFile; ++i) {
+    ASSERT_OK(Put(Key(static_cast<int>(i)), rnd.RandomString(kValueLen)));
+  }
+  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
+
+  // We can use `BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT` to detect whether a
+  // compression dictionary exists since dictionaries would be preloaded when
+  // the flush finishes.
+  if (bottommost_) {
+    // Flush is never considered bottommost. This should change in the future
+    // since flushed files may have nothing underneath them, like the one in
+    // this test case.
+    ASSERT_EQ(
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT),
+        0);
+  } else {
+    ASSERT_GT(
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT),
+        0);
+    // TODO(ajkr): fix the below assertion to work with ZSTD. The expectation on
+    // number of bytes needs to be adjusted in case the cached block is in
+    // ZSTD's digested dictionary format.
+    if (compression_type_ != kZSTD) {
+      // Although we limited buffering to `kBlockLen`, there may be up to two
+      // blocks of data included in the dictionary since we only check limit
+      // after each block is built.
+      ASSERT_LE(TestGetTickerCount(options,
+                                   BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT),
+                2 * kBlockLen);
+    }
+  }
+}
+
+TEST_P(PresetCompressionDictTest, CompactNonBottommost) {
+  // Verifies that dictionary is generated and written during compaction to
+  // non-bottommost level only when `ColumnFamilyOptions::compression` enables
+  // dictionary. Also verifies the size of the dictionary is within expectations
+  // according to the limit on buffering set by
+  // `CompressionOptions::max_dict_buffer_bytes`.
+  const size_t kValueLen = 256;
+  const size_t kKeysPerFile = 1 << 10;
+  const size_t kDictLen = 16 << 10;
+  const size_t kBlockLen = 4 << 10;
+
+  Options options = CurrentOptions();
+  if (bottommost_) {
+    options.bottommost_compression = compression_type_;
+    options.bottommost_compression_opts.enabled = true;
+    options.bottommost_compression_opts.max_dict_bytes = kDictLen;
+    options.bottommost_compression_opts.max_dict_buffer_bytes = kBlockLen;
+  } else {
+    options.compression = compression_type_;
+    options.compression_opts.max_dict_bytes = kDictLen;
+    options.compression_opts.max_dict_buffer_bytes = kBlockLen;
+  }
+  options.disable_auto_compactions = true;
+  options.statistics = CreateDBStatistics();
+  BlockBasedTableOptions bbto;
+  bbto.block_size = kBlockLen;
+  bbto.cache_index_and_filter_blocks = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  Reopen(options);
+
+  Random rnd(301);
+  for (size_t j = 0; j <= kKeysPerFile; ++j) {
+    ASSERT_OK(Put(Key(static_cast<int>(j)), rnd.RandomString(kValueLen)));
+  }
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(2);
+
+  for (int i = 0; i < 2; ++i) {
+    for (size_t j = 0; j <= kKeysPerFile; ++j) {
+      ASSERT_OK(Put(Key(static_cast<int>(j)), rnd.RandomString(kValueLen)));
+    }
+    ASSERT_OK(Flush());
+  }
+  ASSERT_EQ("2,0,1", FilesPerLevel(0));
+
+  uint64_t prev_compression_dict_bytes_inserted =
+      TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT);
+  // This L0->L1 compaction merges the two L0 files into L1. The produced L1
+  // file is not bottommost due to the existing L2 file covering the same key-
+  // range.
+  ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
+  ASSERT_EQ("0,1,1", FilesPerLevel(0));
+  // We can use `BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT` to detect whether a
+  // compression dictionary exists since dictionaries would be preloaded when
+  // the compaction finishes.
+  if (bottommost_) {
+    ASSERT_EQ(
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT),
+        prev_compression_dict_bytes_inserted);
+  } else {
+    ASSERT_GT(
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT),
+        prev_compression_dict_bytes_inserted);
+    // TODO(ajkr): fix the below assertion to work with ZSTD. The expectation on
+    // number of bytes needs to be adjusted in case the cached block is in
+    // ZSTD's digested dictionary format.
+    if (compression_type_ != kZSTD) {
+      // Although we limited buffering to `kBlockLen`, there may be up to two
+      // blocks of data included in the dictionary since we only check limit
+      // after each block is built.
+      ASSERT_LE(TestGetTickerCount(options,
+                                   BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT),
+                prev_compression_dict_bytes_inserted + 2 * kBlockLen);
+    }
+  }
+}
+
+TEST_P(PresetCompressionDictTest, CompactBottommost) {
+  // Verifies that dictionary is generated and written during compaction to
+  // non-bottommost level only when either `ColumnFamilyOptions::compression` or
+  // `ColumnFamilyOptions::bottommost_compression` enables dictionary. Also
+  // verifies the size of the dictionary is within expectations according to the
+  // limit on buffering set by `CompressionOptions::max_dict_buffer_bytes`.
+  const size_t kValueLen = 256;
+  const size_t kKeysPerFile = 1 << 10;
+  const size_t kDictLen = 16 << 10;
+  const size_t kBlockLen = 4 << 10;
+
+  Options options = CurrentOptions();
+  if (bottommost_) {
+    options.bottommost_compression = compression_type_;
+    options.bottommost_compression_opts.enabled = true;
+    options.bottommost_compression_opts.max_dict_bytes = kDictLen;
+    options.bottommost_compression_opts.max_dict_buffer_bytes = kBlockLen;
+  } else {
+    options.compression = compression_type_;
+    options.compression_opts.max_dict_bytes = kDictLen;
+    options.compression_opts.max_dict_buffer_bytes = kBlockLen;
+  }
+  options.disable_auto_compactions = true;
+  options.statistics = CreateDBStatistics();
+  BlockBasedTableOptions bbto;
+  bbto.block_size = kBlockLen;
+  bbto.cache_index_and_filter_blocks = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  Reopen(options);
+
+  Random rnd(301);
+  for (int i = 0; i < 2; ++i) {
+    for (size_t j = 0; j <= kKeysPerFile; ++j) {
+      ASSERT_OK(Put(Key(static_cast<int>(j)), rnd.RandomString(kValueLen)));
+    }
+    ASSERT_OK(Flush());
+  }
+  ASSERT_EQ("2", FilesPerLevel(0));
+
+  uint64_t prev_compression_dict_bytes_inserted =
+      TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT);
+  CompactRangeOptions cro;
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+  ASSERT_EQ("0,1", FilesPerLevel(0));
+  ASSERT_GT(
+      TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT),
+      prev_compression_dict_bytes_inserted);
+  // TODO(ajkr): fix the below assertion to work with ZSTD. The expectation on
+  // number of bytes needs to be adjusted in case the cached block is in ZSTD's
+  // digested dictionary format.
+  if (compression_type_ != kZSTD) {
+    // Although we limited buffering to `kBlockLen`, there may be up to two
+    // blocks of data included in the dictionary since we only check limit after
+    // each block is built.
+    ASSERT_LE(
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT),
+        prev_compression_dict_bytes_inserted + 2 * kBlockLen);
+  }
+}
+
+class CompactionCompressionListener : public EventListener {
+ public:
+  explicit CompactionCompressionListener(Options* db_options)
+      : db_options_(db_options) {}
+
+  void OnCompactionCompleted(DB* db, const CompactionJobInfo& ci) override {
+    // Figure out last level with files
+    int bottommost_level = 0;
+    for (int level = 0; level < db->NumberLevels(); level++) {
+      std::string files_at_level;
+      ASSERT_TRUE(
+          db->GetProperty("rocksdb.num-files-at-level" + std::to_string(level),
+                          &files_at_level));
+      if (files_at_level != "0") {
+        bottommost_level = level;
+      }
+    }
+
+    if (db_options_->bottommost_compression != kDisableCompressionOption &&
+        ci.output_level == bottommost_level) {
+      ASSERT_EQ(ci.compression, db_options_->bottommost_compression);
+    } else if (db_options_->compression_per_level.size() != 0) {
+      ASSERT_EQ(ci.compression,
+                db_options_->compression_per_level[ci.output_level]);
+    } else {
+      ASSERT_EQ(ci.compression, db_options_->compression);
+    }
+    max_level_checked = std::max(max_level_checked, ci.output_level);
+  }
+
+  int max_level_checked = 0;
+  const Options* db_options_;
+};
+
+enum CompressionFailureType {
+  kTestCompressionFail,
+  kTestDecompressionFail,
+  kTestDecompressionCorruption
+};
+
+class CompressionFailuresTest
+    : public DBTest2,
+      public testing::WithParamInterface<std::tuple<
+          CompressionFailureType, CompressionType, uint32_t, uint32_t>> {
+ public:
+  CompressionFailuresTest() {
+    std::tie(compression_failure_type_, compression_type_,
+             compression_max_dict_bytes_, compression_parallel_threads_) =
+        GetParam();
+  }
+
+  CompressionFailureType compression_failure_type_ = kTestCompressionFail;
+  CompressionType compression_type_ = kNoCompression;
+  uint32_t compression_max_dict_bytes_ = 0;
+  uint32_t compression_parallel_threads_ = 0;
+};
+
+INSTANTIATE_TEST_CASE_P(
+    DBTest2, CompressionFailuresTest,
+    ::testing::Combine(::testing::Values(kTestCompressionFail,
+                                         kTestDecompressionFail,
+                                         kTestDecompressionCorruption),
+                       ::testing::ValuesIn(GetSupportedCompressions()),
+                       ::testing::Values(0, 10), ::testing::Values(1, 4)));
+
+TEST_P(CompressionFailuresTest, CompressionFailures) {
+  if (compression_type_ == kNoCompression) {
+    return;
+  }
+
+  Options options = CurrentOptions();
+  options.level0_file_num_compaction_trigger = 2;
+  options.max_bytes_for_level_base = 1024;
+  options.max_bytes_for_level_multiplier = 2;
+  options.num_levels = 7;
+  options.max_background_compactions = 1;
+  options.target_file_size_base = 512;
+
+  BlockBasedTableOptions table_options;
+  table_options.block_size = 512;
+  table_options.verify_compression = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  options.compression = compression_type_;
+  options.compression_opts.parallel_threads = compression_parallel_threads_;
+  options.compression_opts.max_dict_bytes = compression_max_dict_bytes_;
+  options.bottommost_compression_opts.parallel_threads =
+      compression_parallel_threads_;
+  options.bottommost_compression_opts.max_dict_bytes =
+      compression_max_dict_bytes_;
+
+  if (compression_failure_type_ == kTestCompressionFail) {
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "CompressData:TamperWithReturnValue", [](void* arg) {
+          bool* ret = static_cast<bool*>(arg);
+          *ret = false;
+        });
+  } else if (compression_failure_type_ == kTestDecompressionFail) {
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "DecompressBlockData:TamperWithReturnValue", [](void* arg) {
+          Status* ret = static_cast<Status*>(arg);
+          ASSERT_OK(*ret);
+          *ret = Status::Corruption("kTestDecompressionFail");
+        });
+  } else if (compression_failure_type_ == kTestDecompressionCorruption) {
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "DecompressBlockData:TamperWithDecompressionOutput", [](void* arg) {
+          BlockContents* contents = static_cast<BlockContents*>(arg);
+          // Ensure uncompressed data != original data
+          const size_t len = contents->data.size() + 1;
+          std::unique_ptr<char[]> fake_data(new char[len]());
+          *contents = BlockContents(std::move(fake_data), len);
+        });
+  }
+
+  std::map<std::string, std::string> key_value_written;
+
+  const int kKeySize = 5;
+  const int kValUnitSize = 16;
+  const int kValSize = 256;
+  Random rnd(405);
+
+  Status s = Status::OK();
+
+  DestroyAndReopen(options);
+  // Write 10 random files
+  for (int i = 0; i < 10; i++) {
+    for (int j = 0; j < 5; j++) {
+      std::string key = rnd.RandomString(kKeySize);
+      // Ensure good compression ratio
+      std::string valueUnit = rnd.RandomString(kValUnitSize);
+      std::string value;
+      for (int k = 0; k < kValSize; k += kValUnitSize) {
+        value += valueUnit;
+      }
+      s = Put(key, value);
+      if (compression_failure_type_ == kTestCompressionFail) {
+        key_value_written[key] = value;
+        ASSERT_OK(s);
+      }
+    }
+    s = Flush();
+    if (compression_failure_type_ == kTestCompressionFail) {
+      ASSERT_OK(s);
+    }
+    s = dbfull()->TEST_WaitForCompact();
+    if (compression_failure_type_ == kTestCompressionFail) {
+      ASSERT_OK(s);
+    }
+    if (i == 4) {
+      // Make compression fail at the mid of table building
+      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+    }
+  }
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+
+  if (compression_failure_type_ == kTestCompressionFail) {
+    // Should be kNoCompression, check content consistency
+    std::unique_ptr<Iterator> db_iter(db_->NewIterator(ReadOptions()));
+    for (db_iter->SeekToFirst(); db_iter->Valid(); db_iter->Next()) {
+      std::string key = db_iter->key().ToString();
+      std::string value = db_iter->value().ToString();
+      ASSERT_NE(key_value_written.find(key), key_value_written.end());
+      ASSERT_EQ(key_value_written[key], value);
+      key_value_written.erase(key);
+    }
+    ASSERT_OK(db_iter->status());
+    ASSERT_EQ(0, key_value_written.size());
+  } else if (compression_failure_type_ == kTestDecompressionFail) {
+    ASSERT_EQ(std::string(s.getState()),
+              "Could not decompress: kTestDecompressionFail");
+  } else if (compression_failure_type_ == kTestDecompressionCorruption) {
+    ASSERT_EQ(std::string(s.getState()),
+              "Decompressed block did not match pre-compression block");
+  }
+}
+
+TEST_F(DBTest2, CompressionOptions) {
+  if (!Zlib_Supported() || !Snappy_Supported()) {
+    return;
+  }
+
+  Options options = CurrentOptions();
+  options.level0_file_num_compaction_trigger = 2;
+  options.max_bytes_for_level_base = 100;
+  options.max_bytes_for_level_multiplier = 2;
+  options.num_levels = 7;
+  options.max_background_compactions = 1;
+
+  CompactionCompressionListener* listener =
+      new CompactionCompressionListener(&options);
+  options.listeners.emplace_back(listener);
+
+  const int kKeySize = 5;
+  const int kValSize = 20;
+  Random rnd(301);
+
+  std::vector<uint32_t> compression_parallel_threads = {1, 4};
+
+  std::map<std::string, std::string> key_value_written;
+
+  for (int iter = 0; iter <= 2; iter++) {
+    listener->max_level_checked = 0;
+
+    if (iter == 0) {
+      // Use different compression algorithms for different levels but
+      // always use Zlib for bottommost level
+      options.compression_per_level = {kNoCompression,     kNoCompression,
+                                       kNoCompression,     kSnappyCompression,
+                                       kSnappyCompression, kSnappyCompression,
+                                       kZlibCompression};
+      options.compression = kNoCompression;
+      options.bottommost_compression = kZlibCompression;
+    } else if (iter == 1) {
+      // Use Snappy except for bottommost level use ZLib
+      options.compression_per_level = {};
+      options.compression = kSnappyCompression;
+      options.bottommost_compression = kZlibCompression;
+    } else if (iter == 2) {
+      // Use Snappy everywhere
+      options.compression_per_level = {};
+      options.compression = kSnappyCompression;
+      options.bottommost_compression = kDisableCompressionOption;
+    }
+
+    for (auto num_threads : compression_parallel_threads) {
+      options.compression_opts.parallel_threads = num_threads;
+      options.bottommost_compression_opts.parallel_threads = num_threads;
+
+      DestroyAndReopen(options);
+      // Write 10 random files
+      for (int i = 0; i < 10; i++) {
+        for (int j = 0; j < 5; j++) {
+          std::string key = rnd.RandomString(kKeySize);
+          std::string value = rnd.RandomString(kValSize);
+          key_value_written[key] = value;
+          ASSERT_OK(Put(key, value));
+        }
+        ASSERT_OK(Flush());
+        ASSERT_OK(dbfull()->TEST_WaitForCompact());
+      }
+
+      // Make sure that we wrote enough to check all 7 levels
+      ASSERT_EQ(listener->max_level_checked, 6);
+
+      // Make sure database content is the same as key_value_written
+      std::unique_ptr<Iterator> db_iter(db_->NewIterator(ReadOptions()));
+      for (db_iter->SeekToFirst(); db_iter->Valid(); db_iter->Next()) {
+        std::string key = db_iter->key().ToString();
+        std::string value = db_iter->value().ToString();
+        ASSERT_NE(key_value_written.find(key), key_value_written.end());
+        ASSERT_EQ(key_value_written[key], value);
+        key_value_written.erase(key);
+      }
+      ASSERT_OK(db_iter->status());
+      ASSERT_EQ(0, key_value_written.size());
+    }
+  }
+}
+
+TEST_F(DBTest2, RoundRobinManager) {
+  if (ZSTD_Supported()) {
+    auto mgr = std::make_shared<RoundRobinManager>(
+        GetDefaultBuiltinCompressionManager());
+
+    for (CompressionType type : {kZSTD}) {
+      std::vector<std::string> values;
+      for (bool use_wrapper : {true}) {
+        SCOPED_TRACE("Compression type: " + std::to_string(type) +
+                     (use_wrapper ? " with " : " no ") + "wrapper");
+
+        Options options = CurrentOptions();
+        options.compression = type;
+        options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+        options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex);
+        BlockBasedTableOptions bbto;
+        bbto.enable_index_compression = false;
+        options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+        options.compression_manager = use_wrapper ? mgr : nullptr;
+        DestroyAndReopen(options);
+
+        Random rnd(301);
+        constexpr int kCount = 13;
+
+        // Highly compressible blocks, except 1 non-compressible. Half of the
+        // compressible are morked for bypass and 1 marked for rejection. Values
+        // are large enough to ensure just 1 k-v per block.
+        for (int i = 0; i < kCount; ++i) {
+          std::string value;
+          if (i == 6) {
+            // One non-compressible block
+            value = rnd.RandomBinaryString(20000);
+          } else {
+            test::CompressibleString(&rnd, 0.1, 20000, &value);
+          }
+          values.push_back(value);
+          ASSERT_OK(Put(Key(i), value));
+          ASSERT_EQ(Get(Key(i)), value);
+        }
+        ASSERT_OK(Flush());
+
+        // Ensure well-formed for reads
+        for (int i = 0; i < kCount; ++i) {
+          ASSERT_NE(Get(Key(i)), "NOT_FOUND");
+          ASSERT_EQ(Get(Key(i)), values[i]);
+        }
+        ASSERT_EQ(Get(Key(kCount)), "NOT_FOUND");
+      }
+    }
+  }
+}
+
+TEST_F(DBTest2, RandomMixedCompressionManager) {
+  if (ZSTD_Supported()) {
+    auto mgr = std::make_shared<RandomMixedCompressionManager>(
+        GetDefaultBuiltinCompressionManager());
+    // Currently mixedmanager only supports with preffered compression manager
+    // zstd
+    for (CompressionType type : {kZSTD}) {
+      std::vector<std::string> values;
+      for (bool use_wrapper : {true}) {
+        SCOPED_TRACE("Compression type: " + std::to_string(type) +
+                     (use_wrapper ? " with " : " no ") + "wrapper");
+
+        Options options = CurrentOptions();
+        options.compression = type;
+        options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+        options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex);
+        BlockBasedTableOptions bbto;
+        bbto.enable_index_compression = false;
+        options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+        options.compression_manager = use_wrapper ? mgr : nullptr;
+        DestroyAndReopen(options);
+
+        Random rnd(301);
+        constexpr int kCount = 13;
+
+        // Highly compressible blocks, except 1 non-compressible. Half of the
+        // compressible are morked for bypass and 1 marked for rejection. Values
+        // are large enough to ensure just 1 k-v per block.
+        for (int i = 0; i < kCount; ++i) {
+          std::string value;
+          if (i == 6) {
+            // One non-compressible block
+            value = rnd.RandomBinaryString(20000);
+          } else {
+            test::CompressibleString(&rnd, 0.1, 20000, &value);
+          }
+          values.push_back(value);
+          ASSERT_OK(Put(Key(i), value));
+          ASSERT_EQ(Get(Key(i)), value);
+        }
+        ASSERT_OK(Flush());
+
+        // Ensure well-formed for reads
+        for (int i = 0; i < kCount; ++i) {
+          ASSERT_NE(Get(Key(i)), "NOT_FOUND");
+          ASSERT_EQ(Get(Key(i)), values[i]);
+        }
+        ASSERT_EQ(Get(Key(kCount)), "NOT_FOUND");
+      }
+    }
+  }
+}
+TEST_F(DBTest2, CompressionManagerWrapper) {
+  // Test that we can use a custom CompressionManager to wrap the built-in
+  // CompressionManager, thus adopting a custom *strategy* based on existing
+  // algorithms. This will "mark" some blocks (in their contents) as "do not
+  // compress", i.e. no attempt to compress, and some blocks as "reject
+  // compression", i.e. compression attempted but rejected because of ratio
+  // or otherwise. These cases are distinguishable for statistics that
+  // approximate "wasted effort".
+  static std::string kDoNotCompress = "do_not_compress";
+  static std::string kRejectCompression = "reject_compression";
+
+  struct MyCompressor : public CompressorWrapper {
+    using CompressorWrapper::CompressorWrapper;
+
+    Status CompressBlock(Slice uncompressed_data,
+                         std::string* compressed_output,
+                         CompressionType* out_compression_type,
+                         ManagedWorkingArea* working_area) override {
+      auto begin = uncompressed_data.data();
+      auto end = uncompressed_data.data() + uncompressed_data.size();
+      if (std::search(begin, end, kDoNotCompress.begin(),
+                      kDoNotCompress.end()) != end) {
+        // Do not attempt compression
+        EXPECT_EQ(*out_compression_type, kNoCompression);
+        return Status::OK();
+      } else if (std::search(begin, end, kRejectCompression.begin(),
+                             kRejectCompression.end()) != end) {
+        // Simulate attempted & rejected compression
+        *compressed_output = "blah";
+        EXPECT_EQ(*out_compression_type, kNoCompression);
+        return Status::OK();
+      } else {
+        return wrapped_->CompressBlock(uncompressed_data, compressed_output,
+                                       out_compression_type, working_area);
+      }
+    }
+  };
+  struct MyManager : public CompressionManagerWrapper {
+    using CompressionManagerWrapper::CompressionManagerWrapper;
+    const char* Name() const override { return wrapped_->Name(); }
+    std::unique_ptr<Compressor> GetCompressorForSST(
+        const FilterBuildingContext& context, const CompressionOptions& opts,
+        CompressionType preferred) override {
+      return std::make_unique<MyCompressor>(
+          wrapped_->GetCompressorForSST(context, opts, preferred));
+    }
+  };
+  auto mgr = std::make_shared<MyManager>(GetDefaultBuiltinCompressionManager());
+
+  for (CompressionType type : GetSupportedCompressions()) {
+    for (bool use_wrapper : {false, true}) {
+      if (type == kNoCompression) {
+        continue;
+      }
+      SCOPED_TRACE("Compression type: " + std::to_string(type) +
+                   (use_wrapper ? " with " : " no ") + "wrapper");
+
+      Options options = CurrentOptions();
+      options.compression = type;
+      options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+      options.statistics->set_stats_level(StatsLevel::kExceptTimeForMutex);
+      BlockBasedTableOptions bbto;
+      bbto.enable_index_compression = false;
+      options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+      options.compression_manager = use_wrapper ? mgr : nullptr;
+      DestroyAndReopen(options);
+
+      auto PopStat = [&](Tickers t) -> uint64_t {
+        return options.statistics->getAndResetTickerCount(t);
+      };
+
+      Random rnd(301);
+      constexpr int kCount = 13;
+
+      // Highly compressible blocks, except 1 non-compressible. Half of the
+      // compressible are morked for bypass and 1 marked for rejection. Values
+      // are large enough to ensure just 1 k-v per block.
+      for (int i = 0; i < kCount; ++i) {
+        std::string value;
+        if (i == 6) {
+          // One non-compressible block
+          value = rnd.RandomBinaryString(20000);
+        } else {
+          test::CompressibleString(&rnd, 0.1, 20000, &value);
+          if ((i % 2) == 0) {
+            // Half for bypass
+            value += kDoNotCompress;
+          } else if (i == 7) {
+            // One for rejection
+            value += kRejectCompression;
+          }
+        }
+        ASSERT_OK(Put(Key(i), value));
+      }
+      ASSERT_OK(Flush());
+
+      if (use_wrapper) {
+        EXPECT_EQ(kCount / 2 - 1, PopStat(NUMBER_BLOCK_COMPRESSED));
+        EXPECT_EQ(kCount / 2, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
+        EXPECT_EQ(1 + 1, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
+      } else {
+        EXPECT_EQ(kCount - 1, PopStat(NUMBER_BLOCK_COMPRESSED));
+        EXPECT_EQ(0, PopStat(NUMBER_BLOCK_COMPRESSION_BYPASSED));
+        EXPECT_EQ(1, PopStat(NUMBER_BLOCK_COMPRESSION_REJECTED));
+      }
+
+      // Ensure well-formed for reads
+      for (int i = 0; i < kCount; ++i) {
+        ASSERT_NE(Get(Key(i)), "NOT_FOUND");
+      }
+      ASSERT_EQ(Get(Key(kCount)), "NOT_FOUND");
+    }
+  }
+}
+
 class CompactionStallTestListener : public EventListener {
  public:
   CompactionStallTestListener()

db/db_test_util.h

@@ -452,10 +452,6 @@ class SpecialEnv : public EnvWrapper {
         return s;
       }
 
-      Status GetFileSize(uint64_t* s) override {
-        return target_->GetFileSize(s);
-      }
-
      private:
       std::unique_ptr<RandomAccessFile> target_;
       anon::AtomicCounter* counter_;
@@ -482,10 +478,6 @@ class SpecialEnv : public EnvWrapper {
         return target_->Prefetch(offset, n);
       }
 
-      Status GetFileSize(uint64_t* s) override {
-        return target_->GetFileSize(s);
-      }
-
      private:
       std::unique_ptr<RandomAccessFile> target_;
       std::atomic<uint64_t>* fail_cnt_;

db/db_with_timestamp_basic_test.cc

@@ -2400,6 +2400,7 @@ class DataVisibilityTest : public DBBasicTestWithTimestampBase {
     }
   }
 };
+constexpr int DataVisibilityTest::kTestDataSize;
 
 // Application specifies timestamp but not snapshot.
 //           reader              writer

db/external_sst_file_basic_test.cc

@@ -16,7 +16,6 @@
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
 #include "util/defer.h"
-#include "util/file_checksum_helper.h"
 #include "util/random.h"
 #include "utilities/fault_injection_env.h"
 
@@ -347,8 +346,7 @@ class ChecksumVerifyHelper {
 
   Status GetSingleFileChecksumAndFuncName(
       const std::string& file_path, std::string* file_checksum,
-      std::string* file_checksum_func_name,
-      const std::string& requested_func_name = {}) {
+      std::string* file_checksum_func_name) {
     Status s;
     EnvOptions soptions;
     std::unique_ptr<SequentialFile> file_reader;
@@ -366,8 +364,6 @@ class ChecksumVerifyHelper {
       return Status::OK();
     } else {
       FileChecksumGenContext gen_context;
-      gen_context.file_name = file_path;
-      gen_context.requested_checksum_func_name = requested_func_name;
       std::unique_ptr<FileChecksumGenerator> file_checksum_gen =
           file_checksum_gen_factory->CreateFileChecksumGenerator(gen_context);
       *file_checksum_func_name = file_checksum_gen->Name();
@@ -443,50 +439,10 @@ TEST_F(ExternalSSTFileBasicTest, BasicWithFileChecksumCrc32c) {
   DestroyAndRecreateExternalSSTFilesDir();
 }
 
-namespace {
-class VariousFileChecksumGenerator : public FileChecksumGenCrc32c {
- public:
-  explicit VariousFileChecksumGenerator(const std::string& name)
-      : FileChecksumGenCrc32c({}), name_(name) {}
-
-  const char* Name() const override { return name_.c_str(); }
-
-  std::string GetChecksum() const override {
-    return FileChecksumGenCrc32c::GetChecksum() + "_" + name_;
-  }
-
- private:
-  const std::string name_;
-};
-
-class VariousFileChecksumGenFactory : public FileChecksumGenFactory {
- public:
-  std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(
-      const FileChecksumGenContext& context) override {
-    static RelaxedAtomic<int> counter{0};
-    if (Slice(context.requested_checksum_func_name).starts_with("Various")) {
-      return std::make_unique<VariousFileChecksumGenerator>(
-          context.requested_checksum_func_name);
-    } else if (context.requested_checksum_func_name.empty()) {
-      // Lacking a specific request, use a different function name for each
-      // result.
-      return std::make_unique<VariousFileChecksumGenerator>(
-          "Various" + std::to_string(counter.FetchAddRelaxed(1)));
-    } else {
-      return nullptr;
-    }
-  }
-
-  static const char* kClassName() { return "VariousFileChecksumGenFactory"; }
-  const char* Name() const override { return kClassName(); }
-};
-}  // namespace
-
 TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
   Options old_options = CurrentOptions();
   Options options = CurrentOptions();
-  options.file_checksum_gen_factory =
-      std::make_shared<VariousFileChecksumGenFactory>();
+  options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
   const ImmutableCFOptions ioptions(options);
   ChecksumVerifyHelper checksum_helper(options);
 
@@ -507,8 +463,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
   ASSERT_EQ(file1_info.largest_key, Key(1099));
   std::string file_checksum1, file_checksum_func_name1;
   ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file1, &file_checksum1, &file_checksum_func_name1,
-      file1_info.file_checksum_func_name));
+      file1, &file_checksum1, &file_checksum_func_name1));
   ASSERT_EQ(file1_info.file_checksum, file_checksum1);
   ASSERT_EQ(file1_info.file_checksum_func_name, file_checksum_func_name1);
 
@@ -527,8 +482,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
   ASSERT_EQ(file2_info.largest_key, Key(1299));
   std::string file_checksum2, file_checksum_func_name2;
   ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file2, &file_checksum2, &file_checksum_func_name2,
-      file2_info.file_checksum_func_name));
+      file2, &file_checksum2, &file_checksum_func_name2));
   ASSERT_EQ(file2_info.file_checksum, file_checksum2);
   ASSERT_EQ(file2_info.file_checksum_func_name, file_checksum_func_name2);
 
@@ -547,8 +501,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
   ASSERT_EQ(file3_info.largest_key, Key(1499));
   std::string file_checksum3, file_checksum_func_name3;
   ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file3, &file_checksum3, &file_checksum_func_name3,
-      file3_info.file_checksum_func_name));
+      file3, &file_checksum3, &file_checksum_func_name3));
   ASSERT_EQ(file3_info.file_checksum, file_checksum3);
   ASSERT_EQ(file3_info.file_checksum_func_name, file_checksum_func_name3);
 
@@ -567,8 +520,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
   ASSERT_EQ(file4_info.largest_key, Key(1799));
   std::string file_checksum4, file_checksum_func_name4;
   ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file4, &file_checksum4, &file_checksum_func_name4,
-      file4_info.file_checksum_func_name));
+      file4, &file_checksum4, &file_checksum_func_name4));
   ASSERT_EQ(file4_info.file_checksum, file_checksum4);
   ASSERT_EQ(file4_info.file_checksum_func_name, file_checksum_func_name4);
 
@@ -587,8 +539,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
   ASSERT_EQ(file5_info.largest_key, Key(1999));
   std::string file_checksum5, file_checksum_func_name5;
   ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file5, &file_checksum5, &file_checksum_func_name5,
-      file5_info.file_checksum_func_name));
+      file5, &file_checksum5, &file_checksum_func_name5));
   ASSERT_EQ(file5_info.file_checksum, file_checksum5);
   ASSERT_EQ(file5_info.file_checksum_func_name, file_checksum_func_name5);
 
@@ -607,8 +558,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
   ASSERT_EQ(file6_info.largest_key, Key(2199));
   std::string file_checksum6, file_checksum_func_name6;
   ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file6, &file_checksum6, &file_checksum_func_name6,
-      file6_info.file_checksum_func_name));
+      file6, &file_checksum6, &file_checksum_func_name6));
   ASSERT_EQ(file6_info.file_checksum, file_checksum6);
   ASSERT_EQ(file6_info.file_checksum_func_name, file_checksum_func_name6);
 
@@ -678,23 +628,18 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
   }
   ASSERT_OK(env_->FileExists(file2));
 
-  // Enable verify_file_checksum option. No checksum information is provided,
-  // so it is generated when ingesting. The configured checksum factory will
-  // use a different function than before.
-  s = AddFileWithFileChecksum({file3}, {}, {}, true, false, false, false);
+  // Enable verify_file_checksum option
+  // No checksum information is provided, generate it when ingesting
+  std::vector<std::string> checksum, checksum_func;
+  s = AddFileWithFileChecksum({file3}, checksum, checksum_func, true, false,
+                              false, false);
   ASSERT_OK(s) << s.ToString();
   std::vector<LiveFileMetaData> live_files2;
   dbfull()->GetLiveFilesMetaData(&live_files2);
   for (const auto& f : live_files2) {
     if (set1.find(f.name) == set1.end()) {
-      // Recomputed checksum, different function
-      EXPECT_NE(f.file_checksum_func_name, file_checksum_func_name3);
-      std::string cur_checksum3, cur_checksum_func_name3;
-      ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-          dbname_ + f.name, &cur_checksum3, &cur_checksum_func_name3,
-          f.file_checksum_func_name));
-      EXPECT_EQ(f.file_checksum, cur_checksum3);
-      EXPECT_EQ(f.file_checksum_func_name, cur_checksum_func_name3);
+      ASSERT_EQ(f.file_checksum, file_checksum3);
+      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name3);
       set1.insert(f.name);
     }
   }
@@ -708,9 +653,8 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
   ASSERT_NOK(s) << s.ToString();
 
   // Does not enable verify_file_checksum options
-  // Checksum function name is recognized, so store the checksum being ingested.
-  std::string file_checksum_func_name4alt = "VariousABCD";
-  s = AddFileWithFileChecksum({file4}, {"asd"}, {file_checksum_func_name4alt},
+  // Checksum function name matches, store the checksum being ingested.
+  s = AddFileWithFileChecksum({file4}, {"asd"}, {file_checksum_func_name4},
                               false, false, false, false);
   ASSERT_OK(s) << s.ToString();
   std::vector<LiveFileMetaData> live_files3;
@@ -719,7 +663,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
     if (set1.find(f.name) == set1.end()) {
       ASSERT_FALSE(f.file_checksum == file_checksum4);
       ASSERT_EQ(f.file_checksum, "asd");
-      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name4alt);
+      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name4);
       set1.insert(f.name);
     }
   }
@@ -728,8 +672,7 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
 
   // enable verify_file_checksum options, DB enable checksum, and enable
   // write_global_seq. So the checksum stored is different from the one
-  // ingested due to the sequence number changes. The checksum function name
-  // may also change since the checksum is recomputed.
+  // ingested due to the sequence number changes.
   s = AddFileWithFileChecksum({file5}, {file_checksum5},
                               {file_checksum_func_name5}, true, false, false,
                               true);
@@ -738,14 +681,11 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
   dbfull()->GetLiveFilesMetaData(&live_files4);
   for (const auto& f : live_files4) {
     if (set1.find(f.name) == set1.end()) {
-      // Recomputed checksum, different function
-      EXPECT_NE(f.file_checksum_func_name, file_checksum_func_name5);
       std::string cur_checksum5, cur_checksum_func_name5;
       ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-          dbname_ + f.name, &cur_checksum5, &cur_checksum_func_name5,
-          f.file_checksum_func_name));
-      EXPECT_EQ(f.file_checksum, cur_checksum5);
-      EXPECT_EQ(f.file_checksum_func_name, cur_checksum_func_name5);
+          dbname_ + f.name, &cur_checksum5, &cur_checksum_func_name5));
+      ASSERT_EQ(f.file_checksum, cur_checksum5);
+      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name5);
       set1.insert(f.name);
     }
   }
@@ -753,22 +693,18 @@ TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
   ASSERT_OK(env_->FileExists(file5));
 
   // Does not enable verify_file_checksum options and also the ingested file
-  // checksum information is empty. DB will generate and store file checksum
-  // in Manifest, which could be different from the previous invocation.
-  s = AddFileWithFileChecksum({file6}, {}, {}, false, false, false, false);
+  // checksum information is empty. DB will generate and store the checksum
+  // in Manifest.
+  std::vector<std::string> files_c6, files_name6;
+  s = AddFileWithFileChecksum({file6}, files_c6, files_name6, false, false,
+                              false, false);
   ASSERT_OK(s) << s.ToString();
   std::vector<LiveFileMetaData> live_files6;
   dbfull()->GetLiveFilesMetaData(&live_files6);
   for (const auto& f : live_files6) {
     if (set1.find(f.name) == set1.end()) {
-      // Recomputed checksum, different function
-      EXPECT_NE(f.file_checksum_func_name, file_checksum_func_name6);
-      std::string cur_checksum6, cur_checksum_func_name6;
-      ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-          dbname_ + f.name, &cur_checksum6, &cur_checksum_func_name6,
-          f.file_checksum_func_name));
-      EXPECT_EQ(f.file_checksum, cur_checksum6);
-      EXPECT_EQ(f.file_checksum_func_name, cur_checksum_func_name6);
+      ASSERT_EQ(f.file_checksum, file_checksum6);
+      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name6);
       set1.insert(f.name);
     }
   }

db/external_sst_file_ingestion_job.cc

@@ -156,14 +156,6 @@ Status ExternalSstFileIngestionJob::Prepare(
         // It is unsafe to assume application had sync the file and file
         // directory before ingest the file. For integrity of RocksDB we need
         // to sync the file.
-
-        // TODO(xingbo), We should in general be moving away from production
-        // uses of ReuseWritableFile (except explicitly for WAL recycling),
-        // ReopenWritableFile, and NewRandomRWFile. We should create a
-        // FileSystem::SyncFile/FsyncFile API that by default does the
-        // re-open+sync+close combo but can (a) be reused easily, and (b) be
-        // overridden to do that more cleanly, e.g. in EncryptedEnv.
-        // https://github.com/facebook/rocksdb/issues/13741
         std::unique_ptr<FSWritableFile> file_to_sync;
         Status s = fs_->ReopenWritableFile(path_inside_db, env_options_,
                                            &file_to_sync, nullptr);
@@ -268,6 +260,10 @@ Status ExternalSstFileIngestionJob::Prepare(
     } else {
       need_generate_file_checksum_ = true;
     }
+    FileChecksumGenContext gen_context;
+    std::unique_ptr<FileChecksumGenerator> file_checksum_gen =
+        db_options_.file_checksum_gen_factory->CreateFileChecksumGenerator(
+            gen_context);
     std::vector<std::string> generated_checksums;
     std::vector<std::string> generated_checksum_func_names;
     // Step 1: generate the checksum for ingested sst file.
@@ -275,9 +271,7 @@ Status ExternalSstFileIngestionJob::Prepare(
       for (size_t i = 0; i < files_to_ingest_.size(); i++) {
         std::string generated_checksum;
         std::string generated_checksum_func_name;
-        std::string requested_checksum_func_name =
-            i < files_checksum_func_names.size() ? files_checksum_func_names[i]
-                                                 : "";
+        std::string requested_checksum_func_name;
         // TODO: rate limit file reads for checksum calculation during file
         // ingestion.
         // TODO: plumb Env::IOActivity
@@ -320,50 +314,40 @@ Status ExternalSstFileIngestionJob::Prepare(
             if (files_checksum_func_names[i] !=
                 generated_checksum_func_names[i]) {
               status = Status::InvalidArgument(
-                  "DB file checksum gen factory " +
-                  std::string(db_options_.file_checksum_gen_factory->Name()) +
-                  " generated checksum function name " +
-                  generated_checksum_func_names[i] + " for file " +
-                  external_files_paths[i] +
-                  " which does not match requested/provided " +
-                  files_checksum_func_names[i]);
+                  "Checksum function name does not match with the checksum "
+                  "function name of this DB");
+              ROCKS_LOG_WARN(
+                  db_options_.info_log,
+                  "Sst file checksum verification of file: %s failed: %s",
+                  external_files_paths[i].c_str(), status.ToString().c_str());
               break;
             }
             if (files_checksums[i] != generated_checksums[i]) {
               status = Status::Corruption(
-                  "Checksum verification mismatch for ingestion file " +
-                  external_files_paths[i] + " using function " +
-                  generated_checksum_func_names[i] + ". Expected: " +
-                  Slice(files_checksums[i]).ToString(/*hex=*/true) +
-                  " Computed: " +
-                  Slice(generated_checksums[i]).ToString(/*hex=*/true));
+                  "Ingested checksum does not match with the generated "
+                  "checksum");
+              ROCKS_LOG_WARN(
+                  db_options_.info_log,
+                  "Sst file checksum verification of file: %s failed: %s",
+                  files_to_ingest_[i].internal_file_path.c_str(),
+                  status.ToString().c_str());
               break;
             }
           }
         } else {
-          // If verify_file_checksum is not enabled, we only verify the factory
-          // recognizes the checksum function name. If it does not match, fail
-          // the ingestion. If matches, we trust the ingested checksum
-          // information and store in the Manifest.
+          // If verify_file_checksum is not enabled, we only verify the
+          // checksum function name. If it does not match, fail the ingestion.
+          // If matches, we trust the ingested checksum information and store
+          // in the Manifest.
           for (size_t i = 0; i < files_to_ingest_.size(); i++) {
-            FileChecksumGenContext gen_context;
-            gen_context.file_name = files_to_ingest_[i].internal_file_path;
-            gen_context.requested_checksum_func_name =
-                files_checksum_func_names[i];
-            auto file_checksum_gen =
-                db_options_.file_checksum_gen_factory
-                    ->CreateFileChecksumGenerator(gen_context);
-
-            if (file_checksum_gen == nullptr ||
-                files_checksum_func_names[i] != file_checksum_gen->Name()) {
+            if (files_checksum_func_names[i] != file_checksum_gen->Name()) {
               status = Status::InvalidArgument(
-                  "Checksum function name " + files_checksum_func_names[i] +
-                  " for file " + external_files_paths[i] +
-                  " not recognized by DB checksum gen factory" +
-                  db_options_.file_checksum_gen_factory->Name() +
-                  (file_checksum_gen ? (" Returned function " +
-                                        std::string(file_checksum_gen->Name()))
-                                     : ""));
+                  "Checksum function name does not match with the checksum "
+                  "function name of this DB");
+              ROCKS_LOG_WARN(
+                  db_options_.info_log,
+                  "Sst file checksum verification of file: %s failed: %s",
+                  external_files_paths[i].c_str(), status.ToString().c_str());
               break;
             }
             files_to_ingest_[i].file_checksum = files_checksums[i];
@@ -378,11 +362,12 @@ Status ExternalSstFileIngestionJob::Prepare(
         status = Status::InvalidArgument(
             "The checksum information of ingested sst files are nonempty and "
             "the size of checksums or the size of the checksum function "
-            "names does not match with the number of ingested sst files");
-      }
-      if (!status.ok()) {
-        ROCKS_LOG_WARN(db_options_.info_log, "Ingestion failed: %s",
-                       status.ToString().c_str());
+            "names "
+            "does not match with the number of ingested sst files");
+        ROCKS_LOG_WARN(
+            db_options_.info_log,
+            "The ingested sst files checksum information is incomplete: %s",
+            status.ToString().c_str());
       }
     }
   }
@@ -697,9 +682,10 @@ void ExternalSstFileIngestionJob::CreateEquivalentFileIngestingCompactions() {
         0 /* max_subcompaction, not applicable */,
         {} /* grandparents, not applicable */,
         std::nullopt /* earliest_snapshot */, nullptr /* snapshot_checker */,
-        CompactionReason::kExternalSstIngestion, "" /* trim_ts */,
-        -1 /* score, not applicable */,
-        files_overlap_ /* l0_files_might_overlap, not applicable */));
+        false /* is manual */, "" /* trim_ts */, -1 /* score, not applicable */,
+        false /* is deletion compaction, not applicable */,
+        files_overlap_ /* l0_files_might_overlap, not applicable */,
+        CompactionReason::kExternalSstIngestion));
   }
 }
 
@@ -848,8 +834,7 @@ Status ExternalSstFileIngestionJob::ResetTableReader(
       ro,
       TableReaderOptions(
           cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor,
-          sv->mutable_cf_options.compression_manager.get(), env_options_,
-          cfd_->internal_comparator(),
+          env_options_, cfd_->internal_comparator(),
           sv->mutable_cf_options.block_protection_bytes_per_key,
           /*skip_filters*/ false, /*immortal*/ false,
           /*force_direct_prefetch*/ false, /*level*/ -1,

db/import_column_family_job.cc

@@ -324,8 +324,7 @@ Status ImportColumnFamilyJob::GetIngestedFileInfo(
   status = sv->mutable_cf_options.table_factory->NewTableReader(
       TableReaderOptions(
           cfd_->ioptions(), sv->mutable_cf_options.prefix_extractor,
-          sv->mutable_cf_options.compression_manager.get(), env_options_,
-          cfd_->internal_comparator(),
+          env_options_, cfd_->internal_comparator(),
           sv->mutable_cf_options.block_protection_bytes_per_key,
           /*skip_filters*/ false, /*immortal*/ false,
           /*force_direct_prefetch*/ false, /*level*/ -1,

db/log_reader.cc

@@ -95,7 +95,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
   uint64_t prospective_record_offset = 0;
 
   Slice fragment;
-  for (;;) {
+  while (true) {
     uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
     size_t drop_size = 0;
     const uint8_t record_type =
@@ -140,7 +140,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
         prospective_record_offset = physical_record_offset;
         scratch->assign(fragment.data(), fragment.size());
         in_fragmented_record = true;
-        break;  // switch
+        break;
 
       case kMiddleType:
       case kRecyclableMiddleType:
@@ -153,7 +153,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
           }
           scratch->append(fragment.data(), fragment.size());
         }
-        break;  // switch
+        break;
 
       case kLastType:
       case kRecyclableLastType:
@@ -171,7 +171,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
           first_record_read_ = true;
           return true;
         }
-        break;  // switch
+        break;
 
       case kSetCompressionType: {
         if (compression_type_record_read_) {
@@ -193,7 +193,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
         } else {
           InitCompression(compression_record);
         }
-        break;  // switch
+        break;
       }
       case kPredecessorWALInfoType:
       case kRecyclePredecessorWALInfoType: {
@@ -210,7 +210,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
           MaybeVerifyPredecessorWALInfo(wal_recovery_mode, fragment,
                                         recorded_predecessor_wal_info);
         }
-        break;  // switch
+        break;
       }
       case kUserDefinedTimestampSizeType:
       case kRecyclableUserDefinedTimestampSizeType: {
@@ -235,7 +235,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
             ReportCorruption(fragment.size(), s.getState());
           }
         }
-        break;  // switch
+        break;
       }
 
       case kBadHeader:
@@ -304,7 +304,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
           in_fragmented_record = false;
           scratch->clear();
         }
-        break;  // switch
+        break;
 
       case kBadRecordLen:
         if (eof_) {
@@ -337,7 +337,7 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
           in_fragmented_record = false;
           scratch->clear();
         }
-        break;  // switch
+        break;
 
       default: {
         if ((record_type & kRecordTypeSafeIgnoreMask) == 0) {
@@ -349,11 +349,11 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch,
         }
         in_fragmented_record = false;
         scratch->clear();
-        break;  // switch
+        break;
       }
     }
   }
-  // unreachable
+  return false;
 }
 
 void Reader::MaybeVerifyPredecessorWALInfo(

db/memtable.cc

@@ -196,11 +196,10 @@ bool MemTable::ShouldFlushNow() {
   // allocate one more block.
   const double kAllowOverAllocationRatio = 0.6;
 
-  // range deletion use skip list which allocates all memeory through `arena_`
-  assert(range_del_table_->ApproximateMemoryUsage() == 0);
   // If arena still have room for new block allocation, we can safely say it
   // shouldn't flush.
   auto allocated_memory = table_->ApproximateMemoryUsage() +
+                          range_del_table_->ApproximateMemoryUsage() +
                           arena_.MemoryAllocatedBytes();
 
   approximate_memory_usage_.store(allocated_memory, std::memory_order_relaxed);

db/memtable.h

@@ -688,7 +688,6 @@ class MemTable final : public ReadOnlyMemTable {
   // Update counters and flush status after inserting a whole write batch
   // Used in concurrent memtable inserts.
   void BatchPostProcess(const MemTablePostProcessInfo& update_counters) {
-    table_->BatchPostProcess();
     num_entries_.fetch_add(update_counters.num_entries,
                            std::memory_order_relaxed);
     data_size_.fetch_add(update_counters.data_size, std::memory_order_relaxed);
@@ -699,10 +698,6 @@ class MemTable final : public ReadOnlyMemTable {
     if (update_counters.num_range_deletes > 0) {
       num_range_deletes_.fetch_add(update_counters.num_range_deletes,
                                    std::memory_order_relaxed);
-      // noop for skip-list memtable
-      // Besides correctness test in stress test, memtable flush record count
-      // check will catch this if it were not noop.
-      // range_del_table_->BatchPostProcess();
     }
     UpdateFlushState();
   }

db/merge_helper.h

@@ -250,7 +250,7 @@ class MergeHelper {
   // Parallel with keys_; stores the operands
   mutable MergeContext merge_context_;
 
-  StopWatchNano<> filter_timer_;
+  StopWatchNano filter_timer_;
   uint64_t total_filter_time_;
   Statistics* stats_;
 

db/multi_scan.cc

@@ -1,70 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "rocksdb/db.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-using MultiScanIterator = MultiScan::MultiScanIterator;
-
-MultiScan::MultiScan(const ReadOptions& read_options,
-                     const std::vector<ScanOptions>& scan_opts, DB* db,
-                     ColumnFamilyHandle* cfh)
-    : read_options_(read_options), scan_opts_(scan_opts), db_(db), cfh_(cfh) {
-  bool slow_path = false;
-  // Setup read_options with iterate_uuper_bound based on the first scan.
-  // Subsequent scans will update and allocate a new DB iterator as necessary
-  if (scan_opts[0].range.limit) {
-    upper_bound_ = *scan_opts[0].range.limit;
-    read_options_.iterate_upper_bound = &upper_bound_;
-  } else {
-    read_options_.iterate_upper_bound = nullptr;
-  }
-  for (auto opts : scan_opts) {
-    // Check that all the ScanOptions either specify an upper bound or not. If
-    // its mixed we take the slow path which avoids calling Prepare: we have to
-    // reallocate the Iterator with updated read_options everytime we switch
-    // between upper bound or no upper bound, which complicates Prepare.
-    if (opts.range.limit.has_value() != scan_opts[0].range.limit.has_value()) {
-      slow_path = true;
-      break;
-    }
-  }
-  db_iter_.reset(db->NewIterator(read_options_, cfh));
-  if (!slow_path) {
-    db_iter_->Prepare(scan_opts);
-  }
-}
-
-MultiScanIterator& MultiScanIterator::operator++() {
-  if (idx_ >= scan_opts_.size()) {
-    throw std::logic_error("Index out of range");
-  }
-  idx_++;
-  if (idx_ < scan_opts_.size()) {
-    // Check if we need to update read_options_
-    if (scan_opts_[idx_].range.limit.has_value() !=
-        (read_options_.iterate_upper_bound != nullptr)) {
-      if (scan_opts_[idx_].range.limit) {
-        *upper_bound_ = *scan_opts_[idx_].range.limit;
-        read_options_.iterate_upper_bound = upper_bound_;
-      } else {
-        read_options_.iterate_upper_bound = nullptr;
-      }
-      db_iter_.reset(db_->NewIterator(read_options_, cfh_));
-      scan_.Reset(db_iter_.get());
-    } else if (scan_opts_[idx_].range.limit) {
-      *upper_bound_ = *scan_opts_[idx_].range.limit;
-    }
-    db_iter_->Seek(*scan_opts_[idx_].range.start);
-    status_ = db_iter_->status();
-    if (!status_.ok()) {
-      throw MultiScanException(status_);
-    }
-  }
-  return *this;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/periodic_task_scheduler.cc

@@ -26,7 +26,6 @@ static const std::map<PeriodicTaskType, uint64_t> kDefaultPeriodSeconds = {
     {PeriodicTaskType::kPersistStats, kInvalidPeriodSec},
     {PeriodicTaskType::kFlushInfoLog, 10},
     {PeriodicTaskType::kRecordSeqnoTime, kInvalidPeriodSec},
-    {PeriodicTaskType::kTriggerCompaction, 12 * 60 * 60}  // 12 hours
 };
 
 static const std::map<PeriodicTaskType, std::string> kPeriodicTaskTypeNames = {
@@ -34,20 +33,16 @@ static const std::map<PeriodicTaskType, std::string> kPeriodicTaskTypeNames = {
     {PeriodicTaskType::kPersistStats, "pst_st"},
     {PeriodicTaskType::kFlushInfoLog, "flush_info_log"},
     {PeriodicTaskType::kRecordSeqnoTime, "record_seq_time"},
-    {PeriodicTaskType::kTriggerCompaction, "trigger_compaction"},
 };
 
 Status PeriodicTaskScheduler::Register(PeriodicTaskType task_type,
-                                       const PeriodicTaskFunc& fn,
-                                       bool run_immediately) {
-  return Register(task_type, fn, kDefaultPeriodSeconds.at(task_type),
-                  run_immediately);
+                                       const PeriodicTaskFunc& fn) {
+  return Register(task_type, fn, kDefaultPeriodSeconds.at(task_type));
 }
 
 Status PeriodicTaskScheduler::Register(PeriodicTaskType task_type,
                                        const PeriodicTaskFunc& fn,
-                                       uint64_t repeat_period_seconds,
-                                       bool run_immediately) {
+                                       uint64_t repeat_period_seconds) {
   MutexLock l(&timer_mutex);
   static std::atomic<uint64_t> initial_delay(0);
 
@@ -70,13 +65,10 @@ Status PeriodicTaskScheduler::Register(PeriodicTaskType task_type,
   std::string unique_id =
       kPeriodicTaskTypeNames.at(task_type) + std::to_string(id_++);
 
-  uint64_t initial_delay_micros =
-      (initial_delay.fetch_add(1) % repeat_period_seconds) * kMicrosInSecond;
-  if (!run_immediately) {
-    initial_delay_micros += repeat_period_seconds * kMicrosInSecond;
-  }
-  bool succeeded = timer_->Add(fn, unique_id, initial_delay_micros,
-                               repeat_period_seconds * kMicrosInSecond);
+  bool succeeded = timer_->Add(
+      fn, unique_id,
+      (initial_delay.fetch_add(1) % repeat_period_seconds) * kMicrosInSecond,
+      repeat_period_seconds * kMicrosInSecond);
   if (!succeeded) {
     return Status::Aborted("Failed to register periodic task");
   }

db/periodic_task_scheduler.h

@@ -21,7 +21,6 @@ enum class PeriodicTaskType : uint8_t {
   kPersistStats,
   kFlushInfoLog,
   kRecordSeqnoTime,
-  kTriggerCompaction,
   kMax,
 };
 
@@ -43,16 +42,13 @@ class PeriodicTaskScheduler {
   PeriodicTaskScheduler& operator=(PeriodicTaskScheduler&&) = delete;
 
   // Register a task with its default repeat period. Thread safe call.
-  // @param run_immediately If true, the task will run soon after it's
-  // scheduled, instead of waiting for the repeat period.
-  Status Register(PeriodicTaskType task_type, const PeriodicTaskFunc& fn,
-                  bool run_immediately);
+  Status Register(PeriodicTaskType task_type, const PeriodicTaskFunc& fn);
 
   // Register a task with specified repeat period. 0 is an invalid argument
   // (kInvalidPeriodSec). To stop the task, please use Unregister().
   // Thread safe call.
   Status Register(PeriodicTaskType task_type, const PeriodicTaskFunc& fn,
-                  uint64_t repeat_period_seconds, bool run_immediately);
+                  uint64_t repeat_period_seconds);
 
   // Unregister the task. Thread safe call.
   Status Unregister(PeriodicTaskType task_type);

db/periodic_task_scheduler_test.cc

@@ -56,12 +56,6 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
   SyncPoint::GetInstance()->SetCallBack(
       "DBImpl::FlushInfoLog:StartRunning",
       [&](void*) { flush_info_log_counter++; });
-
-  int trigger_compaction_counter = 0;
-  SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::TriggerPeriodicCompaction:StartRunning",
-      [&](void*) { trigger_compaction_counter++; });
-
   SyncPoint::GetInstance()->EnableProcessing();
 
   Reopen(options);
@@ -76,7 +70,7 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
 
   const PeriodicTaskScheduler& scheduler =
       dbfull()->TEST_GetPeriodicTaskScheduler();
-  ASSERT_EQ((int)PeriodicTaskType::kMax - 1, scheduler.TEST_GetValidTaskNum());
+  ASSERT_EQ(3, scheduler.TEST_GetValidTaskNum());
 
   ASSERT_EQ(1, dump_st_counter);
   ASSERT_EQ(1, pst_st_counter);
@@ -109,14 +103,14 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
   ASSERT_EQ(3, pst_st_counter);
   ASSERT_EQ(4, flush_info_log_counter);
 
-  ASSERT_EQ(2u, scheduler.TEST_GetValidTaskNum());
+  ASSERT_EQ(1u, scheduler.TEST_GetValidTaskNum());
 
   // Re-enable one task
   ASSERT_OK(dbfull()->SetDBOptions({{"stats_dump_period_sec", "5"}}));
   ASSERT_EQ(5u, dbfull()->GetDBOptions().stats_dump_period_sec);
   ASSERT_EQ(0u, dbfull()->GetDBOptions().stats_persist_period_sec);
 
-  ASSERT_EQ(3, scheduler.TEST_GetValidTaskNum());
+  ASSERT_EQ(2, scheduler.TEST_GetValidTaskNum());
 
   dbfull()->TEST_WaitForPeriodicTaskRun(
       [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kPeriodSec)); });
@@ -124,16 +118,6 @@ TEST_F(PeriodicTaskSchedulerTest, Basic) {
   ASSERT_EQ(3, pst_st_counter);
   ASSERT_EQ(5, flush_info_log_counter);
 
-  ASSERT_EQ(0, trigger_compaction_counter);
-  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
-    mock_clock_->MockSleepForSeconds(static_cast<int>(12 * 60 * 60));
-  });
-  ASSERT_EQ(1, trigger_compaction_counter);
-  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
-    mock_clock_->MockSleepForSeconds(static_cast<int>(12 * 60 * 60));
-  });
-  ASSERT_EQ(2, trigger_compaction_counter);
-
   Close();
 }
 
@@ -166,9 +150,7 @@ TEST_F(PeriodicTaskSchedulerTest, MultiInstances) {
   auto dbi = static_cast_with_check<DBImpl>(dbs[kInstanceNum - 1]);
 
   const PeriodicTaskScheduler& scheduler = dbi->TEST_GetPeriodicTaskScheduler();
-  // kRecordSeqnoTime is not registered since the feature is not enabled
-  ASSERT_EQ(kInstanceNum * ((int)PeriodicTaskType::kMax - 1),
-            scheduler.TEST_GetValidTaskNum());
+  ASSERT_EQ(kInstanceNum * 3, scheduler.TEST_GetValidTaskNum());
 
   int expected_run = kInstanceNum;
   dbi->TEST_WaitForPeriodicTaskRun(

db/table_cache.cc

@@ -146,8 +146,7 @@ Status TableCache::GetTableReader(
     s = mutable_cf_options.table_factory->NewTableReader(
         ro,
         TableReaderOptions(
-            ioptions_, mutable_cf_options.prefix_extractor,
-            mutable_cf_options.compression_manager.get(), file_options,
+            ioptions_, mutable_cf_options.prefix_extractor, file_options,
             internal_comparator,
             mutable_cf_options.block_protection_bytes_per_key, skip_filters,
             immortal_tables_, false /* force_direct_prefetch */, level,

db/version_set.cc

@@ -95,8 +95,6 @@ namespace ROCKSDB_NAMESPACE {
 
 namespace {
 
-using ScanOptionsMap = std::unordered_map<size_t, std::vector<ScanOptions>>;
-
 // Find File in LevelFilesBrief data structure
 // Within an index range defined by left and right
 int FindFileInRange(const InternalKeyComparator& icmp,
@@ -1102,47 +1100,9 @@ class LevelIterator final : public InternalIterator {
   }
 
   void Prepare(const std::vector<ScanOptions>* scan_opts) override {
-    // We assume here that scan_opts is sorted such that
-    // scan_opts[0].range.start < scan_opts[1].range.start, and non overlapping
     scan_opts_ = scan_opts;
-    if (scan_opts_ == nullptr) {
-      return;
-    }
-
-    file_to_scan_opts_ = std::make_unique<ScanOptionsMap>();
-    for (size_t k = 0; k < scan_opts_->size(); k++) {
-      const ScanOptions& opt = scan_opts_->at(k);
-      auto start = opt.range.start;
-      auto end = opt.range.limit;
-
-      if (!start.has_value()) {
-        continue;
-      }
-
-      // We can capture this case in the future, but for now lets skip this.
-      if (!end.has_value()) {
-        continue;
-      }
-
-      InternalKey istart(start.value(), kMaxSequenceNumber, kValueTypeForSeek);
-      InternalKey iend(end.value(), 0, kValueTypeForSeekForPrev);
-
-      // TODO: This needs to be optimized, right now we iterate twice, which
-      // we dont need to. We can do this in N rather than 2N.
-      size_t fstart = FindFile(icomparator_, *flevel_, istart.Encode());
-      size_t fend = FindFile(icomparator_, *flevel_, iend.Encode());
-
-      // We need to check the relevant cases
-      // Cases:
-      // 1. [  S        E  ]
-      // 2. [  S  ]  [  E  ]
-      // 3. [  S  ] ...... [  E  ]
-      for (auto i = fstart; i <= fend; i++) {
-        if (i < flevel_->num_files) {
-          (*file_to_scan_opts_)[i].emplace_back(start.value(), end.value());
-          (*file_to_scan_opts_)[i].back().property_bag = opt.property_bag;
-        }
-      }
+    if (file_iter_.iter()) {
+      file_iter_.Prepare(scan_opts_);
     }
   }
 
@@ -1273,9 +1233,6 @@ class LevelIterator final : public InternalIterator {
   bool to_return_sentinel_ = false;
   const std::vector<ScanOptions>* scan_opts_;
 
-  // Our stored scan_opts for each prefix
-  std::unique_ptr<ScanOptionsMap> file_to_scan_opts_ = nullptr;
-
   // Sets flags for if we should return the sentinel key next.
   // The condition for returning sentinel is reaching the end of current
   // file_iter_: !Valid() && status.().ok().
@@ -1537,18 +1494,7 @@ bool LevelIterator::SkipEmptyFileForward() {
     // LevelIterator::Seek*, it should also call Seek* into the corresponding
     // range tombstone iterator.
     if (file_iter_.iter() != nullptr) {
-      // If we are doing prepared scan opts then we should seek to the values
-      // specified by the scan opts
-      if (scan_opts_ && (*file_to_scan_opts_)[file_index_].size()) {
-        const ScanOptions& opts = file_to_scan_opts_->at(file_index_).front();
-        if (opts.range.start.has_value()) {
-          InternalKey target(*opts.range.start.AsPtr(), kMaxSequenceNumber,
-                             kValueTypeForSeek);
-          file_iter_.Seek(target.Encode());
-        }
-      } else {
-        file_iter_.SeekToFirst();
-      }
+      file_iter_.SeekToFirst();
       if (range_tombstone_iter_) {
         if (*range_tombstone_iter_) {
           (*range_tombstone_iter_)->SeekToFirst();
@@ -1596,15 +1542,10 @@ void LevelIterator::SetFileIterator(InternalIterator* iter) {
   }
 
   InternalIterator* old_iter = file_iter_.Set(iter);
+  // Since this is a new table iterator, no need to call Prepare() if
+  // scan_opts_ is null
   if (iter && scan_opts_) {
-    if (file_to_scan_opts_.get() &&
-        file_to_scan_opts_->find(file_index_) != file_to_scan_opts_->end()) {
-      const std::vector<ScanOptions>& opts =
-          file_to_scan_opts_->at(file_index_);
-      file_iter_.Prepare(&opts);
-    } else {
-      file_iter_.Prepare(scan_opts_);
-    }
+    file_iter_.Prepare(scan_opts_);
   }
 
   // Update the read pattern for PrefetchBuffer.
@@ -1641,7 +1582,6 @@ void LevelIterator::InitFileIterator(size_t new_file_index) {
     }
   }
 }
-
 }  // anonymous namespace
 
 Status Version::GetTableProperties(const ReadOptions& read_options,
@@ -2801,10 +2741,9 @@ void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
           RecordTick(db_statistics_, MULTIGET_COROUTINE_COUNT,
                      mget_tasks.size());
           // Collect all results so far
-          std::vector<Status> statuses =
-              folly::coro::blockingWait(co_withExecutor(
-                  &range->context()->executor(),
-                  folly::coro::collectAllRange(std::move(mget_tasks))));
+          std::vector<Status> statuses = folly::coro::blockingWait(
+              folly::coro::collectAllRange(std::move(mget_tasks))
+                  .scheduleOn(&range->context()->executor()));
           if (s.ok()) {
             for (Status stat : statuses) {
               if (!stat.ok()) {
@@ -3089,10 +3028,9 @@ Status Version::MultiGetAsync(
         assert(waiting.size());
         RecordTick(db_statistics_, MULTIGET_COROUTINE_COUNT, mget_tasks.size());
         // Collect all results so far
-        std::vector<Status> statuses =
-            folly::coro::blockingWait(co_withExecutor(
-                &range->context()->executor(),
-                folly::coro::collectAllRange(std::move(mget_tasks))));
+        std::vector<Status> statuses = folly::coro::blockingWait(
+            folly::coro::collectAllRange(std::move(mget_tasks))
+                .scheduleOn(&range->context()->executor()));
         mget_tasks.clear();
         if (s.ok()) {
           for (Status stat : statuses) {
@@ -3569,13 +3507,8 @@ void VersionStorageInfo::ComputeCompactionScore(
       }
 
       if (compaction_style_ == kCompactionStyleFIFO) {
-        auto max_table_files_size =
-            mutable_cf_options.compaction_options_fifo.max_table_files_size;
-        if (max_table_files_size == 0) {
-          // avoid divide 0
-          max_table_files_size = 1;
-        }
-        score = static_cast<double>(total_size) / max_table_files_size;
+        score = static_cast<double>(total_size) /
+                mutable_cf_options.compaction_options_fifo.max_table_files_size;
         if (score < 1 &&
             mutable_cf_options.compaction_options_fifo.allow_compaction) {
           score = std::max(

db/write_batch.cc

@@ -551,6 +551,9 @@ Status WriteBatchInternal::Iterate(const WriteBatch* wb,
 
     if (LIKELY(!s.IsTryAgain())) {
       last_was_try_again = false;
+      tag = 0;
+      column_family = 0;  // default
+
       s = ReadRecordFromWriteBatch(&input, &tag, &column_family, &key, &value,
                                    &blob, &xid, &write_unix_time);
       if (!s.ok()) {
@@ -1894,6 +1897,7 @@ Status WriteBatch::VerifyChecksum() const {
     // ReadRecordFromWriteBatch
     key.clear();
     value.clear();
+    column_family = 0;
     s = ReadRecordFromWriteBatch(&input, &tag, &column_family, &key, &value,
                                  &blob, &xid, /*write_unix_time=*/nullptr);
     if (!s.ok()) {
@@ -3210,11 +3214,11 @@ Status WriteBatchInternal::InsertInto(
     ColumnFamilyMemTables* memtables, FlushScheduler* flush_scheduler,
     TrimHistoryScheduler* trim_history_scheduler,
     bool ignore_missing_column_families, uint64_t recovery_log_number, DB* db,
-    bool seq_per_batch, bool batch_per_txn) {
+    bool concurrent_memtable_writes, bool seq_per_batch, bool batch_per_txn) {
   MemTableInserter inserter(
       sequence, memtables, flush_scheduler, trim_history_scheduler,
       ignore_missing_column_families, recovery_log_number, db,
-      /*concurrent_memtable_writes=*/false, nullptr /* prot_info */,
+      concurrent_memtable_writes, nullptr /* prot_info */,
       nullptr /*has_valid_writes*/, seq_per_batch, batch_per_txn);
   for (auto w : write_group) {
     if (w->CallbackFailed()) {

db/write_batch_internal.h

@@ -185,19 +185,18 @@ class WriteBatchInternal {
   // If flush_scheduler is non-null, it will be invoked if the memtable
   // should be flushed.
   //
-  // This overload is for non-concurrent insertion only.
+  // Under concurrent use, the caller is responsible for making sure that
+  // the memtables object itself is thread-local.
   static Status InsertInto(
       WriteThread::WriteGroup& write_group, SequenceNumber sequence,
       ColumnFamilyMemTables* memtables, FlushScheduler* flush_scheduler,
       TrimHistoryScheduler* trim_history_scheduler,
       bool ignore_missing_column_families = false, uint64_t log_number = 0,
-      DB* db = nullptr, bool seq_per_batch = false, bool batch_per_txn = true);
+      DB* db = nullptr, bool concurrent_memtable_writes = false,
+      bool seq_per_batch = false, bool batch_per_txn = true);
 
   // Convenience form of InsertInto when you have only one batch
   // next_seq returns the seq after last sequence number used in MemTable insert
-  //
-  // Under concurrent use, the caller is responsible for making sure that
-  // the memtables object itself is thread-local.
   static Status InsertInto(
       const WriteBatch* batch, ColumnFamilyMemTables* memtables,
       FlushScheduler* flush_scheduler,

db_stress_tool/db_stress_compression_manager.h

@@ -1,65 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include "test_util/testutil.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class DbStressCustomCompressionManager : public CompressionManager {
- public:
-  const char* Name() const override {
-    return "DbStressCustomCompressionManager";
-  }
-  const char* CompatibilityName() const override { return "DbStressCustom1"; }
-
-  bool SupportsCompressionType(CompressionType type) const override {
-    return default_->SupportsCompressionType(type) ||
-           type == kCustomCompressionAA || type == kCustomCompressionAB ||
-           type == kCustomCompressionAC;
-  }
-
-  std::unique_ptr<Compressor> GetCompressor(const CompressionOptions& opts,
-                                            CompressionType type) override {
-    // db_stress never specifies a custom type, so we randomly use them anyway
-    // when this compression manager is used.
-    std::array<CompressionType, 4> choices = {
-        type, kCustomCompressionAA, kCustomCompressionAB, kCustomCompressionAC};
-    type = choices[Random::GetTLSInstance()->Uniform(4)];
-    switch (static_cast<unsigned char>(type)) {
-      case kCustomCompressionAA:
-        return std::make_unique<
-            test::CompressorCustomAlg<kCustomCompressionAA>>();
-      case kCustomCompressionAB:
-        return std::make_unique<
-            test::CompressorCustomAlg<kCustomCompressionAB>>();
-      case kCustomCompressionAC:
-        return std::make_unique<
-            test::CompressorCustomAlg<kCustomCompressionAC>>();
-      // Also support built-in compression algorithms
-      default:
-        return GetBuiltinV2CompressionManager()->GetCompressor(opts, type);
-    }
-  }
-
-  std::shared_ptr<Decompressor> GetDecompressor() override {
-    return std::make_shared<test::DecompressorCustomAlg>();
-  }
-
-  std::shared_ptr<Decompressor> GetDecompressorForTypes(
-      const CompressionType* types_begin,
-      const CompressionType* types_end) override {
-    auto decomp = std::make_shared<test::DecompressorCustomAlg>();
-    decomp->SetAllowedTypes(types_begin, types_end);
-    return decomp;
-  }
-
- protected:
-  std::shared_ptr<CompressionManager> default_ =
-      GetBuiltinV2CompressionManager();
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db_stress_tool/db_stress_test_base.cc

@@ -11,7 +11,6 @@
 #include <ios>
 #include <thread>
 
-#include "db_stress_tool/db_stress_compression_manager.h"
 #include "db_stress_tool/db_stress_listener.h"
 #include "rocksdb/io_status.h"
 #include "rocksdb/options.h"
@@ -876,10 +875,6 @@ Status StressTest::CommitTxn(Transaction& txn, ThreadState* thread) {
     return Status::InvalidArgument("CommitTxn when FLAGS_use_txn is not set");
   }
   Status s = Status::OK();
-  // We don't issue write to transaction's underlying WriteBatch in stress test
-  assert(txn.GetWriteBatch()->GetWriteBatch()->Count());
-  assert(txn.GetWriteBatch()->GetWBWIOpCount() ==
-         txn.GetWriteBatch()->GetWriteBatch()->Count());
   if (FLAGS_use_optimistic_txn) {
     assert(optimistic_txn_db_);
     s = txn.Commit();
@@ -3417,37 +3412,32 @@ void StressTest::Open(SharedState* shared, bool reopen) {
     InitializeOptionsFromFlags(cache_, filter_policy_, options_);
   }
   InitializeOptionsGeneral(cache_, filter_policy_, sqfc_factory_, options_);
-  {
-    // We must register any compression managers with a custom
-    // CompatibilityName() so that if it was used in a past invocation but not
-    // the current invocation, we can still read the SST files requiring it.
-    static std::once_flag loaded;
-    std::call_once(loaded, [&]() {
-      TEST_AllowUnsupportedFormatVersion() = true;
-      auto& library = *ObjectLibrary::Default();
-      library.AddFactory<CompressionManager>(
-          DbStressCustomCompressionManager().CompatibilityName(),
-          [](const std::string& /*uri*/,
-             std::unique_ptr<CompressionManager>* guard,
-             std::string* /*errmsg*/) {
-            *guard = std::make_unique<DbStressCustomCompressionManager>();
-            return guard->get();
-          });
-    });
-  }
-  if (!strcasecmp(FLAGS_compression_manager.c_str(), "custom")) {
-    options_.compression_manager =
-        std::make_shared<DbStressCustomCompressionManager>();
-  } else if (!strcasecmp(FLAGS_compression_manager.c_str(), "mixed")) {
-    options_.compression_manager =
-        std::make_shared<RoundRobinManager>(GetBuiltinV2CompressionManager());
-  } else if (!strcasecmp(FLAGS_compression_manager.c_str(), "randommixed")) {
-    options_.compression_manager =
-        std::make_shared<RandomMixedCompressionManager>(
-            GetBuiltinV2CompressionManager());
+  if (!strcasecmp(FLAGS_compression_manager.c_str(), "mixed") ||
+      !strcasecmp(FLAGS_compression_manager.c_str(), "randommixed")) {
+    // Currently limited to ZSTD compression. Table property compression_name
+    // needs to set to zstd for now even when there can be more than one
+    // algorithm in the table under your compressor.
+    if (!ZSTD_Supported()) {
+      fprintf(stderr,
+              "ZSTD compression not supported thus mixed compression cannot be "
+              "used\n");
+      exit(1);
+    }
+    if (!strcasecmp(FLAGS_compression_manager.c_str(), "mixed")) {
+      auto mgr = std::make_shared<RoundRobinManager>(
+          GetDefaultBuiltinCompressionManager());
+      options_.compression_manager = mgr;
+    } else if (!strcasecmp(FLAGS_compression_manager.c_str(), "randommixed")) {
+      auto mgr = std::make_shared<RandomMixedCompressionManager>(
+          GetDefaultBuiltinCompressionManager());
+      options_.compression_manager = mgr;
+    }
+    options_.compression = kZSTD;
+    options_.bottommost_compression = kZSTD;
+
   } else if (!strcasecmp(FLAGS_compression_manager.c_str(), "autoskip")) {
     options_.compression_manager =
-        CreateAutoSkipCompressionManager(GetBuiltinV2CompressionManager());
+        CreateAutoSkipCompressionManager(GetDefaultBuiltinCompressionManager());
   } else if (!strcasecmp(FLAGS_compression_manager.c_str(), "none")) {
     // Nothing to do using default compression manager
   } else {

env/composite_env.cc

@@ -100,10 +100,6 @@ class CompositeRandomAccessFileWrapper : public RandomAccessFile {
     return target_->InvalidateCache(offset, length);
   }
 
-  Status GetFileSize(uint64_t* size) override {
-    return target_->GetFileSize(size);
-  }
-
  private:
   std::unique_ptr<FSRandomAccessFile> target_;
 };

env/env.cc

@@ -186,10 +186,6 @@ class LegacyRandomAccessFileWrapper : public FSRandomAccessFile {
   IOStatus InvalidateCache(size_t offset, size_t length) override {
     return status_to_io_status(target_->InvalidateCache(offset, length));
   }
-  IOStatus GetFileSize(uint64_t* result) override {
-    auto status = target_->GetFileSize(result);
-    return status_to_io_status(std::move(status));
-  }
 
  private:
   std::unique_ptr<RandomAccessFile> target_;

env/env_encryption.cc

@@ -665,52 +665,17 @@ class EncryptedFileSystemImpl : public EncryptedFileSystem {
                               std::unique_ptr<FSWritableFile>* result,
                               IODebugContext* dbg) override {
     result->reset();
-    if (options.use_mmap_reads || options.use_mmap_writes) {
+    if (options.use_mmap_writes) {
       return IOStatus::InvalidArgument();
     }
-
-    size_t prefix_length = 0;
-    std::unique_ptr<BlockAccessCipherStream> stream;
-
     // Open file using underlying Env implementation
     std::unique_ptr<FSWritableFile> underlying;
-    auto status =
+    IOStatus status =
         FileSystemWrapper::ReopenWritableFile(fname, options, &underlying, dbg);
     if (!status.ok()) {
       return status;
     }
-
-    if (underlying->GetFileSize(options.io_options, dbg) != 0) {
-      // read the cipher stream from file for non-empty file
-      std::unique_ptr<FSRandomAccessFile> underlying_file_reader;
-      status = FileSystemWrapper::NewRandomAccessFile(
-          fname, options, &underlying_file_reader, dbg);
-      if (!status.ok()) {
-        return status;
-      }
-
-      status = CreateRandomReadCipherStream(
-          fname, underlying_file_reader, options, &prefix_length, &stream, dbg);
-
-      if (!status.ok()) {
-        return status;
-      }
-    } else {
-      // create cipher stream for new or empty file
-      status = CreateWritableCipherStream(fname, underlying, options,
-                                          &prefix_length, &stream, dbg);
-      if (!status.ok()) {
-        return status;
-      }
-    }
-
-    if (stream) {
-      result->reset(new EncryptedWritableFile(
-          std::move(underlying), std::move(stream), prefix_length));
-    } else {
-      result->reset(underlying.release());
-    }
-    return status;
+    return CreateWritableEncryptedFile(fname, underlying, options, result, dbg);
   }
 
   IOStatus ReuseWritableFile(const std::string& fname,

env/env_posix.cc

@@ -169,9 +169,8 @@ class PosixClock : public SystemClock {
     struct timespec ts;
     clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts);
     return (static_cast<uint64_t>(ts.tv_sec) * 1000000000 + ts.tv_nsec) / 1000;
-#else
-    return 0;
 #endif
+    return 0;
   }
 
   uint64_t CPUNanos() override {
@@ -180,9 +179,8 @@ class PosixClock : public SystemClock {
     struct timespec ts;
     clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts);
     return static_cast<uint64_t>(ts.tv_sec) * 1000000000 + ts.tv_nsec;
-#else
-    return 0;
 #endif
+    return 0;
   }
 
   void SleepForMicroseconds(int micros) override { usleep(micros); }

env/env_test.cc

@@ -3467,6 +3467,7 @@ class ReadAsyncRandomAccessFile : public FSRandomAccessFileOwnerWrapper {
 
  private:
   ReadAsyncFS& fs_;
+  std::unique_ptr<FSRandomAccessFile> file_;
   int counter = 0;
 };
 
@@ -3656,42 +3657,6 @@ TEST(EnvTestMisc, StaticDestruction) {
   static_destruction_tester.activated = true;
 }
 
-// Test GetFileSize API
-class TestGetFileSize : public testing::Test {
- public:
-  TestGetFileSize() { env_ = Env::Default(); }
-  Env* env_;
-};
-
-// Validate GetFileSize API returns the right value.
-// Use the default implementation from env
-TEST_F(TestGetFileSize, GetFileSize) {
-  EnvOptions soptions;
-  auto fs = env_->GetFileSystem();
-
-  std::string fname = test::PerThreadDBPath(env_, "getFileSizeTestfile");
-
-  // randomize file size
-  auto rnd = Random::GetTLSInstance();
-  auto expectedFileSize = rnd->Uniform(256 * 1024) + 1;
-  auto content = rnd->RandomBinaryString(static_cast<int>(expectedFileSize));
-
-  ASSERT_OK(CreateFile(fs.get(), fname, content, false));
-
-  std::unique_ptr<FSRandomAccessFile> file;
-  ASSERT_OK(fs->NewRandomAccessFile(fname, FileOptions(), &file, nullptr));
-
-  uint64_t fileSizeFromFileSystemAPI;
-  ASSERT_OK(
-      fs->GetFileSize(fname, IOOptions(), &fileSizeFromFileSystemAPI, nullptr));
-  ASSERT_EQ(fileSizeFromFileSystemAPI, expectedFileSize);
-
-  uint64_t fileSizeFromFsRandomAccessFileAPI;
-  ASSERT_OK(file->GetFileSize(&fileSizeFromFsRandomAccessFileAPI));
-
-  ASSERT_EQ(fileSizeFromFsRandomAccessFileAPI, expectedFileSize);
-}
-
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

env/fs_posix.cc

@@ -243,7 +243,7 @@ class PosixFileSystem : public FileSystem {
       // Use mmap when virtual address-space is plentiful.
       uint64_t size;
       IOOptions opts;
-      s = GetFileSizeOnOpenedFile(fd, fname, &size);
+      s = GetFileSize(fname, opts, &size, nullptr);
       if (s.ok()) {
         void* base = mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0);
         if (base != MAP_FAILED) {
@@ -324,7 +324,7 @@ class PosixFileSystem : public FileSystem {
     }
     uint64_t initial_file_size = 0;
     if (reopen) {
-      s = GetFileSizeOnOpenedFile(fd, fname, &initial_file_size);
+      s = GetFileSize(fname, IOOptions(), &initial_file_size, nullptr);
       if (!s.ok()) {
         close(fd);
         return s;
@@ -509,7 +509,7 @@ class PosixFileSystem : public FileSystem {
     uint64_t size;
     if (status.ok()) {
       IOOptions opts;
-      status = GetFileSizeOnOpenedFile(fd, fname, &size);
+      status = GetFileSize(fname, opts, &size, nullptr);
     }
     void* base = nullptr;
     if (status.ok()) {
@@ -671,7 +671,7 @@ class PosixFileSystem : public FileSystem {
 
   IOStatus GetFileSize(const std::string& fname, const IOOptions& /*opts*/,
                        uint64_t* size, IODebugContext* /*dbg*/) override {
-    struct stat sbuf {};
+    struct stat sbuf;
     if (stat(fname.c_str(), &sbuf) != 0) {
       *size = 0;
       return IOError("while stat a file for size", fname, errno);
@@ -868,6 +868,7 @@ class PosixFileSystem : public FileSystem {
       IOOptions opts;
       return CreateDirIfMissing(*result, opts, nullptr);
     }
+    return IOStatus::OK();
   }
 
   IOStatus GetFreeSpace(const std::string& fname, const IOOptions& /*opts*/,
@@ -974,22 +975,6 @@ class PosixFileSystem : public FileSystem {
  private:
   bool forceMmapOff_ = false;  // do we override Env options?
 
-  // This is a faster API comparing to the public method that uses stat to get
-  // file size. However this API only works on opened file.
-  IOStatus GetFileSizeOnOpenedFile(const int fd, const std::string& name,
-                                   uint64_t* size) {
-    struct stat sb {};
-    *size = 0;
-    // Get file information using fstat
-    if (fstat(fd, &sb) == -1) {
-      return IOError(
-          "while fstat a file for size with fd " + std::to_string(fd), name,
-          errno);
-    }
-    *size = sb.st_size;
-    return IOStatus::OK();
-  }
-
 #ifdef OS_LINUX
   // Get the minimum "linux system limit" (i.e, the largest I/O size that the OS
   // can issue to block devices under a directory, also known as

env/io_posix.cc

@@ -607,17 +607,6 @@ PosixRandomAccessFile::PosixRandomAccessFile(
 
 PosixRandomAccessFile::~PosixRandomAccessFile() { close(fd_); }
 
-IOStatus PosixRandomAccessFile::GetFileSize(uint64_t* result) {
-  struct stat sbuf {};
-  if (fstat(fd_, &sbuf) != 0) {
-    *result = 0;
-    return IOError("While fstat with fd " + std::to_string(fd_), filename_,
-                   errno);
-  }
-  *result = sbuf.st_size;
-  return IOStatus::OK();
-}
-
 IOStatus PosixRandomAccessFile::Read(uint64_t offset, size_t n,
                                      const IOOptions& /*opts*/, Slice* result,
                                      char* scratch,
@@ -1067,11 +1056,6 @@ IOStatus PosixMmapReadableFile::InvalidateCache(size_t offset, size_t length) {
 #endif
 }
 
-IOStatus PosixMmapReadableFile::GetFileSize(uint64_t* result) {
-  *result = length_;
-  return IOStatus::OK();
-}
-
 /*
  * PosixMmapFile
  *

env/io_posix.h

@@ -352,8 +352,6 @@ class PosixRandomAccessFile : public FSRandomAccessFile {
                              void* cb_arg, void** io_handle,
                              IOHandleDeleter* del_fn,
                              IODebugContext* dbg) override;
-
-  virtual IOStatus GetFileSize(uint64_t* result) override;
 };
 
 class PosixWritableFile : public FSWritableFile {
@@ -439,7 +437,6 @@ class PosixMmapReadableFile : public FSRandomAccessFile {
                 char* scratch, IODebugContext* dbg) const override;
   void Hint(AccessPattern pattern) override;
   IOStatus InvalidateCache(size_t offset, size_t length) override;
-  virtual IOStatus GetFileSize(uint64_t* result) override;
 };
 
 class PosixMmapFile : public FSWritableFile {

env/mock_env.cc

@@ -322,11 +322,6 @@ class MockRandomAccessFile : public FSRandomAccessFile {
     }
   }
 
-  IOStatus GetFileSize(uint64_t* size) override {
-    *size = file_->Size();
-    return IOStatus::OK();
-  }
-
  private:
   MemFile* file_;
   bool use_direct_io_;

file/readahead_raf.cc

@@ -108,10 +108,6 @@ class ReadaheadRandomAccessFile : public FSRandomAccessFile {
 
   bool use_direct_io() const override { return file_->use_direct_io(); }
 
-  IOStatus GetFileSize(uint64_t* result) override {
-    return file_->GetFileSize(result);
-  }
-
  private:
   // Tries to read from buffer_ n bytes starting at offset. If anything was read
   // from the cache, it sets cached_len to the number of bytes actually read,

file/sst_file_manager_impl.h

@@ -162,6 +162,7 @@ class SstFileManagerImpl : public SstFileManager {
   void Close();
 
   void SetStatisticsPtr(const std::shared_ptr<Statistics>& stats) override {
+    stats_ = stats;
     delete_scheduler_.SetStatisticsPtr(stats);
   }
 
@@ -215,6 +216,7 @@ class SstFileManagerImpl : public SstFileManager {
   std::list<ErrorHandler*> error_handler_list_;
   // Pointer to ErrorHandler instance that is currently processing recovery
   ErrorHandler* cur_instance_;
+  std::shared_ptr<Statistics> stats_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

fuzz/sst_file_writer_fuzzer.cc

@@ -91,8 +91,7 @@ TableReader* NewTableReader(const std::string& sst_file_path,
   }
   if (s.ok()) {
     ImmutableOptions iopts(options, cf_ioptions);
-    TableReaderOptions t_opt(iopts, /*prefix_extractor=*/nullptr,
-                             /*compression_manager=*/nullptr, env_options,
+    TableReaderOptions t_opt(iopts, /*prefix_extractor=*/nullptr, env_options,
                              cf_ioptions.internal_comparator,
                              0 /* block_protection_bytes_per_key */);
     t_opt.largest_seqno = kMaxSequenceNumber;

include/rocksdb/advanced_compression.h

@@ -50,24 +50,11 @@ struct FilterBuildingContext;
 // a number of built-in CompressionTypes that ignore any dictionary block in
 // the file; therefore they cannot accommodate dictionary compression in the
 // future without a schema change / extension.)
-//
-// Exceptions MUST NOT propagate out of overridden functions into RocksDB,
-// because RocksDB is not exception-safe. This could cause undefined behavior
-// including data loss, unreported corruption, deadlocks, and more.
 class Compressor {
  public:
   Compressor() = default;
   virtual ~Compressor() = default;
 
-  // Class name for logging / debugging purposes
-  virtual const char* Name() const = 0;
-
-  // Potentially more elaborate identifier for logging / debugging purposes
-  virtual std::string GetId() const {
-    std::string id = Name();
-    return id;
-  }
-
   // Returns the max total bytes of for all sampled blocks for creating the data
   // dictionary, or zero indicating dictionary compression should not be
   // used/configured. This will typically be called after
@@ -138,15 +125,15 @@ class Compressor {
   // EXTENSIBLE or reinterpret_cast-able by custom Compressor implementations
   struct WorkingArea {};
 
+ protected:
   // To allow for flexible re-use / reclaimation, we have explicit Get and
   // Release functions, and usually wrap in a special RAII smart pointer.
   // For example, a WorkingArea could be saved/recycled in thread-local or
   // core-local storage, or heap managed, etc., though an explicit WorkingArea
   // is only advised for repeated compression (by a single thread).
-  // ReleaseWorkingArea() in not intended to be called directly, but used by
-  // ManagedWorkingArea.
   virtual void ReleaseWorkingArea(WorkingArea*) {}
 
+ public:
   using ManagedWorkingArea =
       ManagedPtr<WorkingArea, Compressor, &Compressor::ReleaseWorkingArea>;
 
@@ -225,10 +212,6 @@ class Compressor {
 // decompressed into part of a single buffer allocated to hold a block's
 // uncompressed contents along with an in-memory object representation of the
 // block (to reduce fragmentation and other overheads of separate objects).
-//
-// Exceptions MUST NOT propagate out of overridden functions into RocksDB,
-// because RocksDB is not exception-safe. This could cause undefined behavior
-// including data loss, unreported corruption, deadlocks, and more.
 class Decompressor {
  public:
   Decompressor() = default;
@@ -245,19 +228,21 @@ class Decompressor {
   // EXTENSIBLE or reinterpret_cast-able by custom Compressor implementations
   struct WorkingArea {};
 
+ protected:
   // To allow for flexible re-use / reclaimation, we have explicit Obtain and
   // Release functions, which are typically wrapped in a special RAII smart
   // pointer. For example, a WorkingArea could be saved/recycled in thread-local
   // or core-local storage, or heap managed, etc., though an explicit
   // WorkingArea is only advised for repeated decompression (by a single
-  // thread). ReleaseWorkingArea() in not intended to be called directly, but
-  // used by ManagedWorkingArea.
+  // thread).
+
   virtual void ReleaseWorkingArea(WorkingArea* wa) {
     // Default implementation: no working area
     (void)wa;
     assert(wa == nullptr);
   }
 
+ public:
   using ManagedWorkingArea =
       ManagedPtr<WorkingArea, Decompressor, &Decompressor::ReleaseWorkingArea>;
 
@@ -281,14 +266,11 @@ class Decompressor {
   // dictionary is processed into a form reusable by repeated compressions in
   // many threads, that happens within this call.
   //
-  // Must return OK if and only if storing a result in `out`. Otherwise, could
-  // return values like NotSupported - dictionary compression is not (yet)
-  // supported for this kind of Decompressor. Corruption - dictionary is
-  // malformed (though many implementations will accept any data as a
-  // dictionary)
-  //
-  // RocksDB promises not to call this function with an empty dictionary slice
-  // (equivalent to no dictionary).
+  // Must return OK if storing a result in `out`. Otherwise, could return values
+  // like NotSupported - dictionary compression is not (yet) supported for this
+  // kind of Decompressor.
+  // Corruption - dictionary is malformed (though many implementations will
+  // accept any data as a dictionary)
   virtual Status MaybeCloneForDict(const Slice& /*serialized_dict*/,
                                    std::unique_ptr<Decompressor>* /*out*/) {
     return Status::NotSupported(
@@ -350,10 +332,6 @@ class Decompressor {
 //   (because that would break backward compatibility, potential quiet
 //   corruption)
 // TODO: consider adding optional streaming compression support (low priority)
-//
-// Exceptions MUST NOT propagate out of overridden functions into RocksDB,
-// because RocksDB is not exception-safe. This could cause undefined behavior
-// including data loss, unreported corruption, deadlocks, and more.
 class CompressionManager
     : public std::enable_shared_from_this<CompressionManager>,
       public Customizable {
@@ -368,31 +346,22 @@ class CompressionManager
   // should have the same CompatibilityName(), so that a compatible
   // CompressionManager/Decompressor might be used if the original is
   // unavailable. (Name() can be useful in addition to CompatibilityName() for
-  // understanding what compression strategy was used.) This name should be
-  // limited to legal variable names in C++ (alphanumeric and underscores).
+  // understanding what compression strategy was used.)
   virtual const char* CompatibilityName() const = 0;
 
   // Default implementation checks the current compatibility name and returns
   // this CompressionManager (via `out`) if appropriate, and otherwise defers
-  // to CreateFromString(). Failure should simply be a matter of "not found" in
-  // which case nullptr is returned.
-  virtual std::shared_ptr<CompressionManager> FindCompatibleCompressionManager(
-      Slice compatibility_name);
+  // to CreateFromString().
+  virtual Status FindCompatibleCompressionManager(
+      Slice compatibility_name, std::shared_ptr<CompressionManager>* out);
 
-  // Create or find a CompressionManager from a string, including built-in
+  // Create a CompressionManager from a string, including built-in
   // CompressionManager types.
   // TODO: ObjectLibrary stuff
   static Status CreateFromString(const ConfigOptions& config_options,
                                  const std::string& id,
                                  std::shared_ptr<CompressionManager>* result);
 
-  // Returns false iff a configuration that would pass the given compression
-  // type to GetCompressor/GetCompressorForSST should be rejected (not
-  // supported)
-  virtual bool SupportsCompressionType(CompressionType type) const = 0;
-
-  // TODO: function to check compatibility with or sanitize CompressionOptions
-
   // ************************* Compressor creation *********************** //
   // Returning nullptr means compression is entirely disabled for the file,
   // which is valid at the discretion of the CompressionManager. Returning
@@ -441,14 +410,6 @@ class CompressionManager
     // Safe default implementation
     return GetDecompressor();
   }
-
-  // Get a decompressor that is allowed to have support only for the
-  // CompressionTypes used by the given Compressor.
-  virtual std::shared_ptr<Decompressor> GetDecompressorForCompressor(
-      const Compressor& compressor) {
-    // Reasonable default implementation
-    return GetDecompressorOptimizeFor(compressor.GetPreferredCompressionType());
-  }
 };
 
 // ************************* Utility wrappers etc. *********************** //
@@ -481,10 +442,6 @@ class CompressorWrapper : public Compressor {
     return wrapped_->ObtainWorkingArea();
   }
 
-  // NOTE: Don't need to override ReleaseWorkingArea() here because
-  // ManagedWorkingArea takes care of calling it on the Compressor that created
-  // the WorkingArea.
-
   Status CompressBlock(Slice uncompressed_data, std::string* compressed_output,
                        CompressionType* out_compression_type,
                        ManagedWorkingArea* working_area) override {
@@ -496,55 +453,6 @@ class CompressorWrapper : public Compressor {
   std::unique_ptr<Compressor> wrapped_;
 };
 
-class DecompressorWrapper : public Decompressor {
- public:
-  explicit DecompressorWrapper(std::shared_ptr<Decompressor> decompressor)
-      : wrapped_(std::move(decompressor)) {}
-  // No copies
-  DecompressorWrapper(const DecompressorWrapper&) = delete;
-  DecompressorWrapper& operator=(const DecompressorWrapper&) = delete;
-
-  const char* Name() const override { return wrapped_->Name(); }
-
-  void ReleaseWorkingArea(WorkingArea* wa) override {
-    wrapped_->ReleaseWorkingArea(wa);
-  }
-
-  // NOTE: Don't need to override ReleaseWorkingArea() here because
-  // ManagedWorkingArea takes care of calling it on the Decompressor that
-  // created the WorkingArea.
-
-  ManagedWorkingArea ObtainWorkingArea(CompressionType preferred) override {
-    return wrapped_->ObtainWorkingArea(preferred);
-  }
-
-  const Slice& GetSerializedDict() const override {
-    return wrapped_->GetSerializedDict();
-  }
-
-  Status MaybeCloneForDict(const Slice& serialized_dict,
-                           std::unique_ptr<Decompressor>* out) override {
-    // NOTE: derived class probably needs to override this to ensure a
-    // derived wrapper around the new Decompressor
-    return wrapped_->MaybeCloneForDict(serialized_dict, out);
-  }
-
-  size_t ApproximateOwnedMemoryUsage() const override {
-    return wrapped_->ApproximateOwnedMemoryUsage();
-  }
-
-  Status ExtractUncompressedSize(Args& args) override {
-    return wrapped_->ExtractUncompressedSize(args);
-  }
-
-  Status DecompressBlock(const Args& args, char* uncompressed_output) override {
-    return wrapped_->DecompressBlock(args, uncompressed_output);
-  }
-
- protected:
-  std::shared_ptr<Decompressor> wrapped_;
-};
-
 // TODO: CompressorBase, for custom compressions
 
 class CompressionManagerWrapper : public CompressionManager {
@@ -557,13 +465,10 @@ class CompressionManagerWrapper : public CompressionManager {
     return wrapped_->CompatibilityName();
   }
 
-  std::shared_ptr<CompressionManager> FindCompatibleCompressionManager(
-      Slice compatibility_name) override {
-    return wrapped_->FindCompatibleCompressionManager(compatibility_name);
-  }
-
-  bool SupportsCompressionType(CompressionType type) const override {
-    return wrapped_->SupportsCompressionType(type);
+  Status FindCompatibleCompressionManager(
+      Slice compatibility_name,
+      std::shared_ptr<CompressionManager>* out) override {
+    return wrapped_->FindCompatibleCompressionManager(compatibility_name, out);
   }
 
   std::unique_ptr<Compressor> GetCompressorForSST(
@@ -592,34 +497,19 @@ class CompressionManagerWrapper : public CompressionManager {
     return wrapped_->GetDecompressorForTypes(types_begin, types_end);
   }
 
-  std::shared_ptr<Decompressor> GetDecompressorForCompressor(
-      const Compressor& compressor) override {
-    return wrapped_->GetDecompressorForCompressor(compressor);
-  }
-
  protected:
   std::shared_ptr<CompressionManager> wrapped_;
 };
 
-// Compression manager that implements the second schema for RocksDB built-in
-// compression support. (The first schema is intentionally not provided here.)
-// *** CURRENT STATE ***
-// This is currently the latest schema for built-in compression, and the
-// compression manager used when compression_manager=nullptr.
-const std::shared_ptr<CompressionManager>& GetBuiltinV2CompressionManager();
-
-// NOTE: No GetLatestBuiltinCompressionManager() is provided because that could
-// lead to unexpected schema changes for user CompressionManagers building on
-// the built-in schema, in the unlikely/rare case of a new built-in schema.
-
-// Creates CompressionManager designed for the automated compression strategy.
+// Compression manager that implements built-in compression strategy. The
+// behavior of
+// compression_manager=nullptr with this
+const std::shared_ptr<CompressionManager>&
+GetDefaultBuiltinCompressionManager();
+// Gets CompressionManager designed for the automated compression strategy.
 // This may include deciding to compress or not.
-// EXPERIMENTAL
+// In future should be able to select compression algorithm based on the CPU
+// utilization and IO constraints.
 std::shared_ptr<CompressionManagerWrapper> CreateAutoSkipCompressionManager(
-    std::shared_ptr<CompressionManager> wrapped = nullptr);
-// Creates CompressionManager designed for the CPU and IO cost aware compression
-// strategy
-// EXPERIMENTAL
-std::shared_ptr<CompressionManagerWrapper> CreateCostAwareCompressionManager(
-    std::shared_ptr<CompressionManager> wrapped = nullptr);
+    std::shared_ptr<CompressionManager> wrapped);
 }  // namespace ROCKSDB_NAMESPACE

include/rocksdb/c.h

@@ -1171,8 +1171,6 @@ extern ROCKSDB_LIBRARY_API uint64_t
 rocksdb_flushjobinfo_smallest_seqno(const rocksdb_flushjobinfo_t*);
 extern ROCKSDB_LIBRARY_API void rocksdb_reset_status(
     rocksdb_status_ptr_t* status_ptr);
-extern ROCKSDB_LIBRARY_API uint32_t
-rocksdb_flushjobinfo_flush_reason(const rocksdb_flushjobinfo_t* info);
 
 /* Compaction job info */
 extern ROCKSDB_LIBRARY_API void rocksdb_compactionjobinfo_status(
@@ -1223,9 +1221,6 @@ extern ROCKSDB_LIBRARY_API int rocksdb_subcompactionjobinfo_base_input_level(
     const rocksdb_subcompactionjobinfo_t*);
 extern ROCKSDB_LIBRARY_API int rocksdb_subcompactionjobinfo_output_level(
     const rocksdb_subcompactionjobinfo_t*);
-extern ROCKSDB_LIBRARY_API uint32_t
-rocksdb_subcompactionjobinfo_compaction_reason(
-    const rocksdb_subcompactionjobinfo_t* info);
 
 /* External file ingestion info */
 extern ROCKSDB_LIBRARY_API const char*

include/rocksdb/compression_type.h

@@ -18,148 +18,16 @@ namespace ROCKSDB_NAMESPACE {
 enum CompressionType : unsigned char {
   // NOTE: do not change the values of existing entries, as these are
   // part of the persistent format on disk.
-  kNoCompression = 0x00,
-  kSnappyCompression = 0x01,
-  kZlibCompression = 0x02,
-  kBZip2Compression = 0x03,
-  kLZ4Compression = 0x04,
-  kLZ4HCCompression = 0x05,
-  kXpressCompression = 0x06,
-  kZSTD = 0x07,
-  kLastBuiltinCompression = kZSTD,
+  kNoCompression = 0x0,
+  kSnappyCompression = 0x1,
+  kZlibCompression = 0x2,
+  kBZip2Compression = 0x3,
+  kLZ4Compression = 0x4,
+  kLZ4HCCompression = 0x5,
+  kXpressCompression = 0x6,
+  kZSTD = 0x7,
 
-  // Reserved for future use: up to 0x7F
-
-  // For use by user custom CompressionManagers
-  kCustomCompression80 = 0x80,
-  kFirstCustomCompression = kCustomCompression80,
-  kCustomCompression81 = 0x81,
-  kCustomCompression82 = 0x82,
-  kCustomCompression83 = 0x83,
-  kCustomCompression84 = 0x84,
-  kCustomCompression85 = 0x85,
-  kCustomCompression86 = 0x86,
-  kCustomCompression87 = 0x87,
-  kCustomCompression88 = 0x88,
-  kCustomCompression89 = 0x89,
-  kCustomCompression8A = 0x8A,
-  kCustomCompression8B = 0x8B,
-  kCustomCompression8C = 0x8C,
-  kCustomCompression8D = 0x8D,
-  kCustomCompression8E = 0x8E,
-  kCustomCompression8F = 0x8F,
-  kCustomCompression90 = 0x90,
-  kCustomCompression91 = 0x91,
-  kCustomCompression92 = 0x92,
-  kCustomCompression93 = 0x93,
-  kCustomCompression94 = 0x94,
-  kCustomCompression95 = 0x95,
-  kCustomCompression96 = 0x96,
-  kCustomCompression97 = 0x97,
-  kCustomCompression98 = 0x98,
-  kCustomCompression99 = 0x99,
-  kCustomCompression9A = 0x9A,
-  kCustomCompression9B = 0x9B,
-  kCustomCompression9C = 0x9C,
-  kCustomCompression9D = 0x9D,
-  kCustomCompression9E = 0x9E,
-  kCustomCompression9F = 0x9F,
-  kCustomCompressionA0 = 0xA0,
-  kCustomCompressionA1 = 0xA1,
-  kCustomCompressionA2 = 0xA2,
-  kCustomCompressionA3 = 0xA3,
-  kCustomCompressionA4 = 0xA4,
-  kCustomCompressionA5 = 0xA5,
-  kCustomCompressionA6 = 0xA6,
-  kCustomCompressionA7 = 0xA7,
-  kCustomCompressionA8 = 0xA8,
-  kCustomCompressionA9 = 0xA9,
-  kCustomCompressionAA = 0xAA,
-  kCustomCompressionAB = 0xAB,
-  kCustomCompressionAC = 0xAC,
-  kCustomCompressionAD = 0xAD,
-  kCustomCompressionAE = 0xAE,
-  kCustomCompressionAF = 0xAF,
-  kCustomCompressionB0 = 0xB0,
-  kCustomCompressionB1 = 0xB1,
-  kCustomCompressionB2 = 0xB2,
-  kCustomCompressionB3 = 0xB3,
-  kCustomCompressionB4 = 0xB4,
-  kCustomCompressionB5 = 0xB5,
-  kCustomCompressionB6 = 0xB6,
-  kCustomCompressionB7 = 0xB7,
-  kCustomCompressionB8 = 0xB8,
-  kCustomCompressionB9 = 0xB9,
-  kCustomCompressionBA = 0xBA,
-  kCustomCompressionBB = 0xBB,
-  kCustomCompressionBC = 0xBC,
-  kCustomCompressionBD = 0xBD,
-  kCustomCompressionBE = 0xBE,
-  kCustomCompressionBF = 0xBF,
-  kCustomCompressionC0 = 0xC0,
-  kCustomCompressionC1 = 0xC1,
-  kCustomCompressionC2 = 0xC2,
-  kCustomCompressionC3 = 0xC3,
-  kCustomCompressionC4 = 0xC4,
-  kCustomCompressionC5 = 0xC5,
-  kCustomCompressionC6 = 0xC6,
-  kCustomCompressionC7 = 0xC7,
-  kCustomCompressionC8 = 0xC8,
-  kCustomCompressionC9 = 0xC9,
-  kCustomCompressionCA = 0xCA,
-  kCustomCompressionCB = 0xCB,
-  kCustomCompressionCC = 0xCC,
-  kCustomCompressionCD = 0xCD,
-  kCustomCompressionCE = 0xCE,
-  kCustomCompressionCF = 0xCF,
-  kCustomCompressionD0 = 0xD0,
-  kCustomCompressionD1 = 0xD1,
-  kCustomCompressionD2 = 0xD2,
-  kCustomCompressionD3 = 0xD3,
-  kCustomCompressionD4 = 0xD4,
-  kCustomCompressionD5 = 0xD5,
-  kCustomCompressionD6 = 0xD6,
-  kCustomCompressionD7 = 0xD7,
-  kCustomCompressionD8 = 0xD8,
-  kCustomCompressionD9 = 0xD9,
-  kCustomCompressionDA = 0xDA,
-  kCustomCompressionDB = 0xDB,
-  kCustomCompressionDC = 0xDC,
-  kCustomCompressionDD = 0xDD,
-  kCustomCompressionDE = 0xDE,
-  kCustomCompressionDF = 0xDF,
-  kCustomCompressionE0 = 0xE0,
-  kCustomCompressionE1 = 0xE1,
-  kCustomCompressionE2 = 0xE2,
-  kCustomCompressionE3 = 0xE3,
-  kCustomCompressionE4 = 0xE4,
-  kCustomCompressionE5 = 0xE5,
-  kCustomCompressionE6 = 0xE6,
-  kCustomCompressionE7 = 0xE7,
-  kCustomCompressionE8 = 0xE8,
-  kCustomCompressionE9 = 0xE9,
-  kCustomCompressionEA = 0xEA,
-  kCustomCompressionEB = 0xEB,
-  kCustomCompressionEC = 0xEC,
-  kCustomCompressionED = 0xED,
-  kCustomCompressionEE = 0xEE,
-  kCustomCompressionEF = 0xEF,
-  kCustomCompressionF0 = 0xF0,
-  kCustomCompressionF1 = 0xF1,
-  kCustomCompressionF2 = 0xF2,
-  kCustomCompressionF3 = 0xF3,
-  kCustomCompressionF4 = 0xF4,
-  kCustomCompressionF5 = 0xF5,
-  kCustomCompressionF6 = 0xF6,
-  kCustomCompressionF7 = 0xF7,
-  kCustomCompressionF8 = 0xF8,
-  kCustomCompressionF9 = 0xF9,
-  kCustomCompressionFA = 0xFA,
-  kCustomCompressionFB = 0xFB,
-  kCustomCompressionFC = 0xFC,
-  kCustomCompressionFD = 0xFD,
-  kCustomCompressionFE = 0xFE,
-  kLastCustomCompression = kCustomCompressionFE,
+  // TODO: add enum values for user custom compression types
 
   // kDisableCompressionOption is used to disable some compression options.
   kDisableCompressionOption = 0xff,

include/rocksdb/db.h

@@ -57,7 +57,6 @@ struct WaitForCompactOptions;
 class Env;
 class EventListener;
 class FileSystem;
-class MultiScan;
 class Replayer;
 class StatsHistoryIterator;
 class TraceReader;
@@ -1093,30 +1092,7 @@ class DB {
 
   // Get an iterator that scans multiple key ranges. The scan ranges should
   // be in increasing order of start key. See multi_scan_iterator.h for more
-  // details. For optimal performance, ensure that either all entries in
-  // scan_opts specify the range limit, or none of them do.
-  //
-  // NOTE: iterate_upper_bound in ReadOptions will be ignored. Instead, the
-  // range.limit in ScanOptions is consulted to determine the upper bound key,
-  // if specified.
-  //
-  // Example usage -
-  //  std::vector<ScanOptions> scans{{.start = Slice("bar")},
-  //                              {.start = Slice("foo")}};
-  //  std::unique_ptr<MultiScan> iter.reset(
-  //                                      db->NewMultiScan());
-  //  try {
-  //    for (auto scan : *iter) {
-  //      for (auto it : scan) {
-  //        // Do something with key - it.first
-  //        // Do something with value - it.second
-  //      }
-  //    }
-  //  } catch (MultiScanException& ex) {
-  //    // Check ex.status()
-  //  } catch (std::logic_error& ex) {
-  //    // Check ex.what()
-  //  }
+  // details.
   virtual std::unique_ptr<MultiScan> NewMultiScan(
       const ReadOptions& /*options*/, ColumnFamilyHandle* /*column_family*/,
       const std::vector<ScanOptions>& /*scan_opts*/) {

include/rocksdb/env.h

@@ -866,13 +866,6 @@ class RandomAccessFile {
         "RandomAccessFile::InvalidateCache not supported.");
   }
 
-  // The default implementation returns "not supported" so that user
-  // implementations of FSRandomAccessFile do not need to immediately implement
-  // this function.
-  virtual Status GetFileSize(uint64_t* /*result*/) {
-    return Status::NotSupported("RandomAccessFile::GetFileSize not supported.");
-  }
-
   // If you're adding methods here, remember to add them to
   // RandomAccessFileWrapper too.
 };
@@ -1757,9 +1750,6 @@ class RandomAccessFileWrapper : public RandomAccessFile {
   Status InvalidateCache(size_t offset, size_t length) override {
     return target_->InvalidateCache(offset, length);
   }
-  Status GetFileSize(uint64_t* file_size) override {
-    return target_->GetFileSize(file_size);
-  }
 
  private:
   RandomAccessFile* target_;

include/rocksdb/env_encryption.h

@@ -240,15 +240,6 @@ class EncryptedRandomAccessFile : public FSRandomAccessFile {
   size_t GetRequiredBufferAlignment() const override;
 
   IOStatus InvalidateCache(size_t offset, size_t length) override;
-
-  // Intentionally leave GetFileSize not overridden here, so that it inherits
-  // the default implementation from its parent class, which is Not Supported.
-  //
-  // As GetFileSize API is not required to be implemented yet, we use encrypted
-  // file system in unit test to validate the rest of the system could continue
-  // working with the Not Supported behavior.
-  //
-  // IOStatus GetFileSize(uint64_t* /*result*/) override;
 };
 
 class EncryptedWritableFile : public FSWritableFile {

include/rocksdb/external_table.h

@@ -269,7 +269,7 @@ class ExternalTableFactory : public Customizable {
 
 // Allocate a TableFactory that wraps around an ExternalTableFactory. Use this
 // to allocate and set in ColumnFamilyOptions::table_factory.
-std::unique_ptr<TableFactory> NewExternalTableFactory(
+std::shared_ptr<TableFactory> NewExternalTableFactory(
     std::shared_ptr<ExternalTableFactory> inner_factory);
 
 }  // namespace ROCKSDB_NAMESPACE

include/rocksdb/file_checksum.h

@@ -80,8 +80,7 @@ class FileChecksumGenFactory : public Customizable {
       const ConfigOptions& options, const std::string& value,
       std::shared_ptr<FileChecksumGenFactory>* result);
 
-  // Create a new FileChecksumGenerator. Recommended to return nullptr if the
-  // requested function name is not recognized.
+  // Create a new FileChecksumGenerator.
   virtual std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(
       const FileChecksumGenContext& context) = 0;
 

include/rocksdb/file_system.h

@@ -1051,14 +1051,6 @@ class FSRandomAccessFile {
   // open.
   virtual Temperature GetTemperature() const { return Temperature::kUnknown; }
 
-  // Get the file size on an open-for-reading file without re-seeking the file's
-  // path in the filesystem. The default implementation returns "not supported"
-  // so that user implementations of FSRandomAccessFile do not need to
-  // immediately implement this function.
-  virtual IOStatus GetFileSize(uint64_t* /*result*/) {
-    return IOStatus::NotSupported("GetFileSize Not Supported");
-  }
-
   // If you're adding methods here, remember to add them to
   // RandomAccessFileWrapper too.
 };
@@ -1780,10 +1772,6 @@ class FSRandomAccessFileWrapper : public FSRandomAccessFile {
     return target_->GetTemperature();
   }
 
-  virtual IOStatus GetFileSize(uint64_t* result) override {
-    return target_->GetFileSize(result);
-  }
-
  private:
   std::unique_ptr<FSRandomAccessFile> guard_;
   FSRandomAccessFile* target_;

include/rocksdb/iterator.h

@@ -95,18 +95,14 @@ class Iterator : public IteratorBase {
     return Slice();
   }
 
-  // Prepare the iterator to scan the ranges specified in scan_opts. This
-  // includes prefetching relevant blocks from disk. The upper bound and
-  // other table specific limits should be specified for each
-  // scan for best results. If an upper bound is not specified, Prepare may
-  // skip prefetching as it cannot accurately determine how much to prefetch.
-  //
-  // Prepare should typically be followed by Seeks to the start keys in the
-  // order they're specified in scan_opts. If the user does a Seek to some
-  // other target key, the iterator should disregard the scan_opts from that
-  // point onwards and behave like a normal iterator. Its the user's
-  // responsibility to again call Prepare().
-  //
+  // RocksDB Internal - DO NOT USE
+  // Prepare the iterator to scan the ranges specified in scan_opts. The
+  // upper bound and other table specific limits may be specified. This will
+  // typically be followed by Seeks to the start keys in the order they're
+  // specified in scan_opts. If the user does a Seek to some other target key,
+  // the iterator should disregard the scan_opts from that point onwards and
+  // behave like a normal iterator. Its the user's responsibility to again
+  // call Prepare().
   // If Prepare() is called, it overrides the iterate_upper_bound in
   // ReadOptions
   virtual void Prepare(const std::vector<ScanOptions>& /*scan_opts*/) {}

include/rocksdb/memtablerep.h

@@ -162,12 +162,6 @@ class MemTableRep {
     return true;
   }
 
-  // Only used after concurrent memtable inserts.
-  // This function will be called by each writer after all writes are done
-  // through InsertConcurrently().
-  // This is used by VectorRep to do batched writes for concurrent inserts.
-  virtual void BatchPostProcess() {}
-
   // Returns true iff an entry that compares equal to key is in the collection.
   virtual bool Contains(const char* key) const = 0;
 
@@ -403,11 +397,6 @@ class SkipListFactory : public MemTableRepFactory {
 // the vector is sorted. This is useful for workloads where iteration is very
 // rare and writes are generally not issued after reads begin.
 //
-// Concurrent inserts are supported by buffering writes in thread-local vectors
-// for each write batch. To optimize performance for concurrent inserts, it is
-// recommended to perform batched writes, and enable unordered_write (refer to
-// the option comment for its impact on read consistency).
-//
 // Parameters:
 //   count: Passed to the constructor of the underlying std::vector of each
 //     VectorRep. On initialization, the underlying array will be at least count
@@ -429,8 +418,6 @@ class VectorRepFactory : public MemTableRepFactory {
   MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&, Allocator*,
                                  const SliceTransform*,
                                  Logger* logger) override;
-
-  bool IsInsertConcurrentlySupported() const override { return true; }
 };
 
 // This class contains a fixed array of buckets, each

include/rocksdb/multi_scan.h

@@ -5,7 +5,6 @@
 
 #pragma once
 
-#include "rocksdb/db.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/options.h"
 
@@ -73,8 +72,6 @@ class Scan {
 
   explicit Scan(Iterator* db_iter) : db_iter_(db_iter) {}
 
-  void Reset(Iterator* db_iter) { db_iter_ = db_iter; }
-
   ScanIterator begin() { return ScanIterator(db_iter_); }
 
   std::nullptr_t end() { return nullptr; }
@@ -152,9 +149,9 @@ class Scan {
 // A Status exception is thrown if there is an error.
 class MultiScan {
  public:
-  MultiScan(const ReadOptions& read_options,
-            const std::vector<ScanOptions>& scan_opts, DB* db,
-            ColumnFamilyHandle* cfh);
+  MultiScan(const std::vector<ScanOptions>& scan_opts,
+            std::unique_ptr<Iterator>&& db_iter)
+      : scan_opts_(scan_opts), db_iter_(std::move(db_iter)) {}
 
   explicit MultiScan(std::unique_ptr<Iterator>&& db_iter)
       : db_iter_(std::move(db_iter)) {}
@@ -171,17 +168,9 @@ class MultiScan {
     using difference_type = int;
     using iterator_category = std::input_iterator_tag;
 
-    MultiScanIterator(const std::vector<ScanOptions>& scan_opts, DB* db,
-                      ColumnFamilyHandle* cfh, ReadOptions& read_options,
-                      Slice* upper_bound, std::unique_ptr<Iterator>& db_iter)
-        : scan_opts_(scan_opts),
-          db_(db),
-          cfh_(cfh),
-          read_options_(read_options),
-          upper_bound_(upper_bound),
-          idx_(0),
-          db_iter_(db_iter),
-          scan_(db_iter_.get()) {
+    MultiScanIterator(const std::vector<ScanOptions>& scan_opts,
+                      Iterator* db_iter)
+        : scan_opts_(scan_opts), idx_(0), db_iter_(db_iter), scan_(db_iter_) {
       if (scan_opts_.empty()) {
         throw std::logic_error("Zero scans in multi-scan");
       }
@@ -192,9 +181,28 @@ class MultiScan {
       }
     }
 
+    explicit MultiScanIterator(const std::vector<ScanOptions>& scan_opts)
+        : scan_opts_(scan_opts),
+          idx_(scan_opts_.size()),
+          db_iter_(nullptr),
+          scan_(nullptr) {}
+
     ~MultiScanIterator() { assert(status_.ok()); }
 
-    MultiScanIterator& operator++();
+    MultiScanIterator& operator++() {
+      if (idx_ >= scan_opts_.size()) {
+        throw std::logic_error("Index out of range");
+      }
+      idx_++;
+      if (idx_ < scan_opts_.size()) {
+        db_iter_->Seek(*scan_opts_[idx_].range.start);
+        status_ = db_iter_->status();
+        if (!status_.ok()) {
+          throw MultiScanException(status_);
+        }
+      }
+      return *this;
+    }
 
     bool operator==(std::nullptr_t /*other*/) const {
       return idx_ >= scan_opts_.size();
@@ -209,29 +217,20 @@ class MultiScan {
 
    private:
     const std::vector<ScanOptions>& scan_opts_;
-    DB* db_;
-    ColumnFamilyHandle* cfh_;
-    ReadOptions& read_options_;
-    Slice* upper_bound_;
     size_t idx_;
-    std::unique_ptr<Iterator>& db_iter_;
+    Iterator* db_iter_;
     Status status_;
     Scan scan_;
   };
 
   MultiScanIterator begin() {
-    return MultiScanIterator(scan_opts_, db_, cfh_, read_options_,
-                             &upper_bound_, db_iter_);
+    return MultiScanIterator(scan_opts_, db_iter_.get());
   }
 
   std::nullptr_t end() { return nullptr; }
 
  private:
-  ReadOptions read_options_;
   const std::vector<ScanOptions> scan_opts_;
-  DB* db_;
-  ColumnFamilyHandle* cfh_;
-  Slice upper_bound_;
   std::unique_ptr<Iterator> db_iter_;
 };
 

include/rocksdb/options.h

@@ -57,7 +57,6 @@ class Statistics;
 class InternalKeyComparator;
 class WalFilter;
 class FileSystem;
-class UserDefinedIndexFactory;
 
 struct Options;
 struct DbPath;
@@ -609,13 +608,6 @@ struct DBOptions {
   // checksums. True also enters a read-only mode when a DB write fails;
   // see DB::Resume().
   //
-  // When set to true, the DB will fail to open if any SST files fail to open
-  // e.g. due to incorrect file size or corrupted footer.
-  //
-  // When set to false, when there are files corrupted, the DB will still be
-  // opened, and the healthy ones could still be accessed, while corrupted one
-  // will not
-  //
   // As most workloads value data correctness over availability, this option
   // is on by default. Note that the name of this old option is potentially
   // misleading, and other options and operations go further in proactive
@@ -1305,13 +1297,12 @@ struct DBOptions {
   // Default: false
   bool skip_stats_update_on_db_open = false;
 
-  // This option is deprecated and marked as no-op. Kept for backward
-  // compatibility until usage is fully removed.
-  // File size check will be performed through a thread
-  // pool during DB Open, when max_open_files is set to -1.
-  // Therefore, the concern of DB Open slowness is eliminated.
-  // Note that when max_open_files is not set to -1, only a subset of files will
-  // be opened and checked during DB Open.
+  // If true, then DB::Open() will not fetch and check sizes of all sst files.
+  // This may significantly speed up startup if there are many sst files,
+  // especially when using non-default Env with expensive GetFileSize().
+  // We'll still check that all required sst files exist.
+  // If paranoid_checks is false, this option is ignored, and sst files are
+  // not checked at all.
   //
   // Default: false
   bool skip_checking_sst_file_sizes_on_db_open = false;
@@ -2070,17 +2061,6 @@ struct ReadOptions {
   // Default: false
   bool auto_refresh_iterator_with_snapshot = false;
 
-  // EXPERIMENTAL
-  //
-  // Specify an alternate index to use in the SST files instead of the native
-  // block based table index. The table_factory used for the column family
-  // must support building/reading this index.
-  //
-  // Currently, only forward scans are supported. For forward scans, only Seek()
-  // is supported. SeekToFirst() is not supported. If the caller wishes to scan
-  // from start to end, the native index must be used.
-  const UserDefinedIndexFactory* table_index_factory = nullptr;
-
   // *** END options only relevant to iterators or scans ***
 
   // *** BEGIN options for RocksDB internal use only ***

include/rocksdb/table.h

@@ -44,7 +44,6 @@ class TableReader;
 class WritableFileWriter;
 struct ConfigOptions;
 struct EnvOptions;
-class UserDefinedIndexFactory;
 
 // Types of checksums to use for checking integrity of logical blocks within
 // files. All checksums currently use 32 bits of checking power (1 in 4B
@@ -493,16 +492,8 @@ struct BlockBasedTableOptions {
   // Because filters only impact performance and are not data-critical, an
   // SST file can be opened and used without filters if (a) the filter
   // policy name or schema is unrecognized, or (b) filter_policy is nullptr.
-  // See filter_policy regarding filters.
   std::shared_ptr<const FilterPolicy> filter_policy = nullptr;
 
-  // EXPERIMENTAL
-  //
-  // If non-nullptr, use the specified factory to build user-defined index.
-  // This allows users to define their own index format and build the index
-  // during table building.
-  std::shared_ptr<UserDefinedIndexFactory> user_defined_index_factory = nullptr;
-
   // If true, place whole keys in the filter (not just prefixes).
   // This must generally be true for gets to be efficient.
   bool whole_key_filtering = true;
@@ -570,10 +561,6 @@ struct BlockBasedTableOptions {
   // misplaced within or between files is as likely to fail checksum
   // verification as random corruption. Also checksum-protects SST footer.
   // Can be read by RocksDB versions >= 8.6.0.
-  // 7 -- Support for custom compression algorithms with a CompressionManager
-  // using a non-built-in CompatibilityName(). See `compression_manager` in
-  // ColumnFamilyOptions. Also changes the format of TableProperties field
-  // `compression_name`. Can be read by RocksDB versions >= 10.4.0.
   //
   // Using the default setting of format_version is strongly recommended, so
   // that available enhancements are adopted eventually and automatically. The

include/rocksdb/table_properties.h

@@ -109,10 +109,6 @@ class TablePropertiesCollector {
   // table.
   // @params key    the user key that is inserted into the table.
   // @params value  the value that is inserted into the table.
-  // @params file_size the current file size. For BlockBasedTable, this
-  //         includes all the data blocks written so far, upto but not including
-  //         the current block being built. With parallel compression, data
-  //         blocks are written async so it depends on the compression progress.
   virtual Status AddUserKey(const Slice& key, const Slice& value,
                             EntryType /*type*/, SequenceNumber /*seq*/,
                             uint64_t /*file_size*/) {
@@ -147,7 +143,7 @@ class TablePropertiesCollector {
   // The name of the properties collector can be used for debugging purpose.
   virtual const char* Name() const = 0;
 
-  // Return whether the output file should be further compacted
+  // EXPERIMENTAL Return whether the output file should be further compacted
   virtual bool NeedCompact() const { return false; }
 
   // For internal use only.
@@ -348,20 +344,7 @@ struct TableProperties {
   // {collector_name[1]},{collector_name[2]},{collector_name[3]} ..
   std::string property_collectors_names;
 
-  // Identifies the compression algorithm or schema used in the file.
-  // Specifically:
-  // * For format_version < 7, it is one of several names for built-in
-  // compression types. Because of how some previous versions of RocksDB
-  // behave, this must be set to "ZSTD" if any blocks are compressed
-  // with zstd and must NOT be set to "NoCompression" if any blocks are
-  // compressed.
-  // * For format_version >= 7, the format is
-  //   <compatibility_name>;<hex-coded compression types>;<future use>
-  // where <compatibility_name> is the CompatibilityName() of the
-  // CompressionManager used for the file, or empty if compression was
-  // disabled; <hex-coded compression types> represents a sorted set of
-  // CompressionType values used in the file other than kNoCompression, each
-  // as 2-digit hex, e.g. 04 for LZ$, 07 for ZSTD, etc.
+  // The compression algo used to compress the SST files.
   std::string compression_name;
 
   // Compression options used to compress the SST files.

include/rocksdb/tool_hooks.h

@@ -69,7 +69,6 @@ class ToolHooks {
   virtual Status Open(const Options& options,
                       const blob_db::BlobDBOptions& bdb_options,
                       const std::string& dbname, blob_db::BlobDB** blob_db) = 0;
-  virtual void Exit(int status) = 0;
 };
 
 class DefaultHooks : public ToolHooks {
@@ -118,8 +117,6 @@ class DefaultHooks : public ToolHooks {
                       const blob_db::BlobDBOptions& bdb_options,
                       const std::string& dbname,
                       blob_db::BlobDB** blob_db) override;
-
-  virtual void Exit(int status) override { exit(status); }
 };
 
 extern DefaultHooks defaultHooks;

include/rocksdb/user_defined_index.h

@@ -1,142 +0,0 @@
-//  Copyright (c) Meta Platforms, Inc. and affiliates.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-//  *****************************************************************
-//  EXPERIMENTAL - subject to change while under development
-//  *****************************************************************
-
-#pragma once
-
-#include <string>
-
-#include "rocksdb/advanced_iterator.h"
-#include "rocksdb/customizable.h"
-#include "rocksdb/options.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Prefix for user-defined index block names
-inline const std::string kUserDefinedIndexPrefix =
-    "rocksdb.user_defined_index.";
-
-// This is a public API for user-defined index builders.
-// It allows users to define their own index format and build custom
-// indexes during table building. Currently, only a monolithic index
-// block is supported (no partitioned index).
-
-// The interface for building user-defined index.
-class UserDefinedIndexBuilder {
- public:
-  // Right now, we only support Puts. In the future, we may support merges,
-  // deletions etc.
-  enum ValueType {
-    kValue,
-    kTypeMax,
-  };
-
-  // File offset and size of the data block
-  struct BlockHandle {
-    uint64_t offset;
-    uint64_t size;
-  };
-
-  virtual ~UserDefinedIndexBuilder() = default;
-
-  // Add a new index entry to index block. The key for the new index entry
-  // should be >= last_key_in_current_block and < first_key_in_next_block.
-  // The previous index entry key and the new index entry key cover
-  // all the keys in the data block associated with the new index entry.
-  //
-  // Called before the OnKeyAdded() call for first_key_in_next_block.
-  // @last_key_in_current_block: The last key in the current data block
-  // @first_key_in_next_block: it will be nullptr if the entry being added is
-  //                           the last one in the table
-  // @block_handle: offset/size of the data block referenced by this index
-  //                entry. This should be stored along with the index entry
-  //                key
-  // @separator_scratch: a scratch buffer to back a computed separator between
-  //                     those, as needed. May be modified on each call.
-  // @return: the key or separator stored in the index, which could be
-  //          last_key_in_current_block or a computed separator backed by
-  //          separator_scratch.
-  virtual Slice AddIndexEntry(const Slice& last_key_in_current_block,
-                              const Slice* first_key_in_next_block,
-                              const BlockHandle& block_handle,
-                              std::string* separator_scratch) = 0;
-
-  // This method will be called whenever a key is added. The subclasses may
-  // override OnKeyAdded() if they need to collect additional information.
-  // The type argument indicates whether the value is a full value or partial.
-  // At the moment, only full values are supported.
-  virtual void OnKeyAdded(const Slice& /*key*/, ValueType /*type*/,
-                          const Slice& /*value*/) {}
-
-  // Finish building the index.
-  // Returns a Status and the serialized index contents.
-  // The memory backing the contents should not be freed until this builder
-  // object is destructed.
-  virtual Status Finish(Slice* index_contents) = 0;
-};
-
-// The interface for iterating the user defined index. This will be
-// instantiated and used by a scan to iterate through the index entries
-// covered by the scan.
-class UserDefinedIndexIterator {
- public:
-  virtual ~UserDefinedIndexIterator() = default;
-
-  // Prepare the iterator for a series of scans. The iterator should use
-  // this as an opportunity to do any prefetching and buffering of results.
-  virtual void Prepare(const ScanOptions scan_opts[], size_t num_opts) = 0;
-
-  // Given the target key, position the index iterator at the index entry
-  // with the smallest key >= target. The result must be updated with the
-  // index key, and the bound_check_result. The bound_check_result should
-  // be set to kOutOfBound if no block satisfies the target key and
-  // termination criteria, kInbound if the data block is definitely fully
-  // within bounds, or kUnknown if the data block could be partially
-  // within bounds.
-  virtual Status SeekAndGetResult(const Slice& target,
-                                  IterateResult* result) = 0;
-
-  // Advance to the next index entry. The result must be populated similar
-  // to SeekAndGetResult.
-  virtual Status NextAndGetResult(IterateResult* result) = 0;
-
-  // Return the BlockHandle in the current index entry
-  virtual UserDefinedIndexBuilder::BlockHandle value() = 0;
-};
-
-// A reader interface for the user defined index
-class UserDefinedIndexReader {
- public:
-  virtual ~UserDefinedIndexReader() = default;
-
-  // Allocate an iterator that will be used by RocksDB to perform scans
-  virtual std::unique_ptr<UserDefinedIndexIterator> NewIterator(
-      const ReadOptions& read_options) = 0;
-
-  // The memory usage of the index, including the size of the raw contents and
-  // any other heap data structures allocated by the reader
-  virtual size_t ApproximateMemoryUsage() const = 0;
-};
-
-// Factory for creating user-defined index builders.
-class UserDefinedIndexFactory : public Customizable {
- public:
-  virtual ~UserDefinedIndexFactory() = default;
-
-  // Create a new builder for user-defined index.
-  virtual UserDefinedIndexBuilder* NewBuilder() const = 0;
-
-  // Create a new user defined index reader given the contents of the index
-  // block
-  virtual std::unique_ptr<UserDefinedIndexReader> NewReader(
-      Slice& index_block) const = 0;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

include/rocksdb/utilities/db_ttl.h

@@ -63,8 +63,6 @@ class DBWithTTL : public StackableDB {
 
   virtual void SetTtl(ColumnFamilyHandle* h, int32_t ttl) = 0;
 
-  virtual Status GetTtl(ColumnFamilyHandle* h, int32_t* ttl) = 0;
-
  protected:
   explicit DBWithTTL(DB* db) : StackableDB(db) {}
 };

include/rocksdb/utilities/table_properties_collectors.h

@@ -23,20 +23,15 @@ class CompactOnDeletionCollectorFactory
   // A factory of a table property collector that marks a SST
   // file as need-compaction when it observe at least "D" deletion
   // entries in any "N" consecutive entries, or the ratio of tombstone
-  // entries >= deletion_ratio for the entire file.
+  // entries >= deletion_ratio.
   //
   // @param sliding_window_size "N"
   // @param deletion_trigger "D"
   // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction
   //     based on deletion ratio.
-  // @param min_file_size, a file needs to be at least this size to be marked
-  //     for compaction. See comments above
-  //     TablePropertiesCollector::AddUserKey() for limitations/inaccuracies on
-  //     the file size.
   CompactOnDeletionCollectorFactory(size_t sliding_window_size,
                                     size_t deletion_trigger,
-                                    double deletion_ratio,
-                                    uint64_t min_file_size = 0);
+                                    double deletion_ratio);
 
   ~CompactOnDeletionCollectorFactory() override {}
 
@@ -64,12 +59,6 @@ class CompactOnDeletionCollectorFactory
   }
 
   double GetDeletionRatio() const { return deletion_ratio_.load(); }
-
-  uint64_t GetMinFileSize() const { return min_file_size_.load(); }
-  void SetMinFileSize(uint64_t min_file_size) {
-    min_file_size_.store(min_file_size);
-  }
-
   static const char* kClassName() { return "CompactOnDeletionCollector"; }
   const char* Name() const override { return kClassName(); }
 
@@ -79,7 +68,6 @@ class CompactOnDeletionCollectorFactory
   std::atomic<size_t> sliding_window_size_;
   std::atomic<size_t> deletion_trigger_;
   std::atomic<double> deletion_ratio_;
-  std::atomic<uint64_t> min_file_size_;
 };
 
 // Creates a factory of a table property collector that marks a SST
@@ -97,8 +85,7 @@ class CompactOnDeletionCollectorFactory
 std::shared_ptr<CompactOnDeletionCollectorFactory>
 NewCompactOnDeletionCollectorFactory(size_t sliding_window_size,
                                      size_t deletion_trigger,
-                                     double deletion_ratio = 0,
-                                     uint64_t min_file_size = 0);
+                                     double deletion_ratio = 0);
 
 // A factory of a table property collector that marks a SST file as
 // need-compaction when for the tiering use case, it observes, among all the

include/rocksdb/utilities/transaction_db.h

@@ -396,9 +396,9 @@ struct TransactionOptions {
   // due to too many memtables.
   // Note that the ingestion relies on the transaction's underlying index,
   // (WriteBatchWithIndex), so updates that are added to the transaction
-  // without indexing (i.e. added directly to the transaction underlying
+  // without indexing (e.g. added directly to the transaction underlying
   // write batch through Transaction::GetWriteBatch()->GetWriteBatch())
-  // are not supported, and the optimization will not apply in that case.
+  // are not supported. They will not be applied to the DB.
   //
   // NOTE: since WBWI keep track of the most recent update per key, a Put
   // followed by a SingleDelete will be written to DB as a SingleDelete. This

include/rocksdb/utilities/write_batch_with_index.h

@@ -379,8 +379,6 @@ class WriteBatchWithIndex : public WriteBatchBase {
   };
   const std::unordered_map<uint32_t, CFStat>& GetCFStats() const;
 
-  // The total number of operations issued into this WBWI.
-  size_t GetWBWIOpCount() const;
   bool GetOverwriteKey() const;
 
  private:

include/rocksdb/version.h

@@ -12,7 +12,7 @@
 // NOTE: in 'main' development branch, this should be the *next*
 // minor or major version number planned for release.
 #define ROCKSDB_MAJOR 10
-#define ROCKSDB_MINOR 6
+#define ROCKSDB_MINOR 4
 #define ROCKSDB_PATCH 0
 
 // Do not use these. We made the mistake of declaring macros starting with

java/src/main/java/org/rocksdb/NativeLibraryLoader.java

@@ -30,14 +30,6 @@ public class NativeLibraryLoader {
   private static final String tempFilePrefix = "librocksdbjni";
   private static final String tempFileSuffix = Environment.getJniLibraryExtension();
 
-  /**
-   * If you set the System Property ROCKS_JAVA_DEBUG_NLL can be to true
-   * messages about attempts to load the native library will be printed
-   * to std out.
-   */
-  private static boolean DEBUG_LOADING =
-      "true".equals(System.getProperty("ROCKS_JAVA_DEBUG_NLL", "false"));
-
   /**
    * Get a reference to the NativeLibraryLoader
    *
@@ -63,7 +55,7 @@ public class NativeLibraryLoader {
    *
    * @throws java.io.IOException if a filesystem operation fails.
    */
-  @SuppressWarnings({"PMD.EmptyCatchBlock", "PMD.SystemPrintln"})
+  @SuppressWarnings("PMD.EmptyCatchBlock")
   public synchronized void loadLibrary(final String tmpDir) throws IOException {
     try {
       // try dynamic library
@@ -71,9 +63,6 @@ public class NativeLibraryLoader {
       return;
     } catch (final UnsatisfiedLinkError ule) {
       // ignore - try from static library
-      if (DEBUG_LOADING) {
-        System.out.println("Unable to load shared dynamic library: " + sharedLibraryName);
-      }
     }
 
     try {
@@ -82,9 +71,6 @@ public class NativeLibraryLoader {
       return;
     } catch (final UnsatisfiedLinkError ule) {
       // ignore - then try static library fallback or from jar
-      if (DEBUG_LOADING) {
-        System.out.println("Unable to load shared static library: " + jniLibraryName);
-      }
     }
 
     if (fallbackJniLibraryName != null) {
@@ -94,10 +80,6 @@ public class NativeLibraryLoader {
         return;
       } catch (final UnsatisfiedLinkError ule) {
         // ignore - then try from jar
-        if (DEBUG_LOADING) {
-          System.out.println(
-              "Unable to load shared static fallback library: " + fallbackJniLibraryName);
-        }
       }
     }
 
@@ -155,23 +137,18 @@ public class NativeLibraryLoader {
     }
   }
 
-  @SuppressWarnings({"PMD.UseProperClassLoader", "PMD.UseTryWithResources", "PMD.SystemPrintln"})
+  @SuppressWarnings({"PMD.UseProperClassLoader", "PMD.UseTryWithResources"})
   File loadLibraryFromJarToTemp(final String tmpDir) throws IOException {
     try (InputStream is = getClass().getClassLoader().getResourceAsStream(jniLibraryFileName)) {
       if (is != null) {
         final File temp = createTemp(tmpDir, jniLibraryFileName);
         Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
         return temp;
-      } else {
-        if (DEBUG_LOADING) {
-          System.out.println("Unable to find: " + jniLibraryFileName + " on the classpath");
-        }
       }
     }
 
     if (fallbackJniLibraryFileName == null) {
-      throw new RuntimeException(
-          jniLibraryFileName + " was not found inside JAR, and there is no fallback.");
+      throw new RuntimeException(fallbackJniLibraryFileName + " was not found inside JAR.");
     }
 
     try (InputStream is =
@@ -180,16 +157,10 @@ public class NativeLibraryLoader {
         final File temp = createTemp(tmpDir, fallbackJniLibraryFileName);
         Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
         return temp;
-      } else {
-        if (DEBUG_LOADING) {
-          System.out.println(
-              "Unable to find fallback: " + fallbackJniLibraryFileName + " on the classpath");
-        }
       }
     }
 
-    throw new RuntimeException("Neither " + jniLibraryFileName + " or " + fallbackJniLibraryFileName
-        + " were found inside the JAR, and there is no fallback.");
+    throw new RuntimeException(jniLibraryFileName + " was not found inside JAR.");
   }
 
   /**

java/src/main/java/org/rocksdb/RocksDB.java

@@ -84,7 +84,13 @@ public class RocksDB extends RocksObject {
       return;
     }
 
-    waitForLibraryToBeLoaded();
+    while (libraryLoaded.get() == LibraryState.LOADING) {
+      try {
+        Thread.sleep(10);
+      } catch(final InterruptedException e) {
+        //ignore
+      }
+    }
   }
 
   /**
@@ -140,28 +146,12 @@ public class RocksDB extends RocksObject {
       return;
     }
 
-    waitForLibraryToBeLoaded();
-  }
-
-  private static void waitForLibraryToBeLoaded() {
-    final long wait = 10; // Time to wait before re-checking if another thread loaded the library
-    final long timeout =
-        10 * 1000; // Maximum time to wait for another thread to load the library (10 seconds)
-    long waited = 0;
-    try {
-      while (libraryLoaded.get() == LibraryState.LOADING) {
-        Thread.sleep(wait);
-        waited += wait;
-
-        if (waited >= timeout) {
-          throw new RuntimeException(
-              "Exceeded timeout whilst trying to load the RocksDB shared library");
-        }
+    while (libraryLoaded.get() == LibraryState.LOADING) {
+      try {
+        Thread.sleep(10);
+      } catch(final InterruptedException e) {
+        //ignore
       }
-    } catch (final InterruptedException e) {
-      // restore interrupted status
-      Thread.currentThread().interrupt();
-      throw new RuntimeException("Interrupted whilst trying to load the RocksDB shared library", e);
     }
   }
 

memtable/vectorrep.cc

@@ -30,8 +30,6 @@ class VectorRep : public MemTableRep {
   // collection.
   void Insert(KeyHandle handle) override;
 
-  void InsertConcurrently(KeyHandle handle) override;
-
   // Returns true iff an entry that compares equal to key is in the collection.
   bool Contains(const char* key) const override;
 
@@ -42,8 +40,6 @@ class VectorRep : public MemTableRep {
   void Get(const LookupKey& k, void* callback_args,
            bool (*callback_func)(void* arg, const char* entry)) override;
 
-  void BatchPostProcess() override;
-
   ~VectorRep() override = default;
 
   class Iterator : public MemTableRep::Iterator {
@@ -104,40 +100,19 @@ class VectorRep : public MemTableRep {
 
  private:
   friend class Iterator;
-  ALIGN_AS(CACHE_LINE_SIZE) RelaxedAtomic<size_t> bucket_size_;
   using Bucket = std::vector<const char*>;
   std::shared_ptr<Bucket> bucket_;
   mutable port::RWMutex rwlock_;
   bool immutable_;
   bool sorted_;
   const KeyComparator& compare_;
-  // Thread-local vector to buffer concurrent writes.
-  using TlBucket = std::vector<const char*>;
-  ThreadLocalPtr tl_writes_;
-
-  static void DeleteTlBucket(void* ptr) {
-    auto* v = static_cast<TlBucket*>(ptr);
-    delete v;
-  }
 };
 
 void VectorRep::Insert(KeyHandle handle) {
   auto* key = static_cast<char*>(handle);
-  {
-    WriteLock l(&rwlock_);
-    assert(!immutable_);
-    bucket_->push_back(key);
-  }
-  bucket_size_.FetchAddRelaxed(1);
-}
-
-void VectorRep::InsertConcurrently(KeyHandle handle) {
-  auto* v = static_cast<TlBucket*>(tl_writes_.Get());
-  if (!v) {
-    v = new TlBucket();
-    tl_writes_.Reset(v);
-  }
-  v->push_back(static_cast<char*>(handle));
+  WriteLock l(&rwlock_);
+  assert(!immutable_);
+  bucket_->push_back(key);
 }
 
 // Returns true iff an entry that compares equal to key is in the collection.
@@ -152,35 +127,19 @@ void VectorRep::MarkReadOnly() {
 }
 
 size_t VectorRep::ApproximateMemoryUsage() {
-  return bucket_size_.LoadRelaxed() *
-         sizeof(std::remove_reference<decltype(*bucket_)>::type::value_type);
-}
-
-void VectorRep::BatchPostProcess() {
-  auto* v = static_cast<TlBucket*>(tl_writes_.Get());
-  if (v) {
-    {
-      WriteLock l(&rwlock_);
-      assert(!immutable_);
-      for (auto& key : *v) {
-        bucket_->push_back(key);
-      }
-    }
-    bucket_size_.FetchAddRelaxed(v->size());
-    delete v;
-    tl_writes_.Reset(nullptr);
-  }
+  return sizeof(bucket_) + sizeof(*bucket_) +
+         bucket_->size() *
+             sizeof(
+                 std::remove_reference<decltype(*bucket_)>::type::value_type);
 }
 
 VectorRep::VectorRep(const KeyComparator& compare, Allocator* allocator,
                      size_t count)
     : MemTableRep(allocator),
-      bucket_size_(0),
       bucket_(new Bucket()),
       immutable_(false),
       sorted_(false),
-      compare_(compare),
-      tl_writes_(DeleteTlBucket) {
+      compare_(compare) {
   bucket_.get()->reserve(count);
 }
 

options/configurable.cc

@@ -272,8 +272,7 @@ Status ConfigurableHelper::ConfigureOptions(
   if (config_options.ignore_unknown_options) {
     s = Status::OK();
   } else if (s.ok() && unused == nullptr && !remaining.empty()) {
-    s = Status::NotFound("Extra option not recognized",
-                         remaining.begin()->first);
+    s = Status::NotFound("Could not find option: ", remaining.begin()->first);
   }
   return s;
 }
@@ -370,7 +369,7 @@ Status ConfigurableHelper::ConfigureSingleOption(
   const auto opt_info =
       FindOption(configurable, opt_name, &elem_name, &opt_ptr);
   if (opt_info == nullptr) {
-    return Status::NotFound("Could not find option", name);
+    return Status::NotFound("Could not find option: ", name);
   } else {
     return ConfigureOption(config_options, configurable, *opt_info, opt_name,
                            elem_name, value, opt_ptr);
@@ -466,7 +465,7 @@ Status ConfigurableHelper::ConfigureOption(
     return configurable.ParseOption(config_options, opt_info, name, value,
                                     opt_ptr);
   } else {
-    return Status::NotFound("Unknown how to configure option", name);
+    return Status::NotFound("Could not find option: ", name);
   }
 }
 

options/options_helper.cc

@@ -386,133 +386,6 @@ std::unordered_map<std::string, CompressionType>
         {"kLZ4HCCompression", kLZ4HCCompression},
         {"kXpressCompression", kXpressCompression},
         {"kZSTD", kZSTD},
-        {"kCustomCompression80", kCustomCompression80},
-        {"kCustomCompression81", kCustomCompression81},
-        {"kCustomCompression82", kCustomCompression82},
-        {"kCustomCompression83", kCustomCompression83},
-        {"kCustomCompression84", kCustomCompression84},
-        {"kCustomCompression85", kCustomCompression85},
-        {"kCustomCompression86", kCustomCompression86},
-        {"kCustomCompression87", kCustomCompression87},
-        {"kCustomCompression88", kCustomCompression88},
-        {"kCustomCompression89", kCustomCompression89},
-        {"kCustomCompression8A", kCustomCompression8A},
-        {"kCustomCompression8B", kCustomCompression8B},
-        {"kCustomCompression8C", kCustomCompression8C},
-        {"kCustomCompression8D", kCustomCompression8D},
-        {"kCustomCompression8E", kCustomCompression8E},
-        {"kCustomCompression8F", kCustomCompression8F},
-        {"kCustomCompression90", kCustomCompression90},
-        {"kCustomCompression91", kCustomCompression91},
-        {"kCustomCompression92", kCustomCompression92},
-        {"kCustomCompression93", kCustomCompression93},
-        {"kCustomCompression94", kCustomCompression94},
-        {"kCustomCompression95", kCustomCompression95},
-        {"kCustomCompression96", kCustomCompression96},
-        {"kCustomCompression97", kCustomCompression97},
-        {"kCustomCompression98", kCustomCompression98},
-        {"kCustomCompression99", kCustomCompression99},
-        {"kCustomCompression9A", kCustomCompression9A},
-        {"kCustomCompression9B", kCustomCompression9B},
-        {"kCustomCompression9C", kCustomCompression9C},
-        {"kCustomCompression9D", kCustomCompression9D},
-        {"kCustomCompression9E", kCustomCompression9E},
-        {"kCustomCompression9F", kCustomCompression9F},
-        {"kCustomCompressionA0", kCustomCompressionA0},
-        {"kCustomCompressionA1", kCustomCompressionA1},
-        {"kCustomCompressionA2", kCustomCompressionA2},
-        {"kCustomCompressionA3", kCustomCompressionA3},
-        {"kCustomCompressionA4", kCustomCompressionA4},
-        {"kCustomCompressionA5", kCustomCompressionA5},
-        {"kCustomCompressionA6", kCustomCompressionA6},
-        {"kCustomCompressionA7", kCustomCompressionA7},
-        {"kCustomCompressionA8", kCustomCompressionA8},
-        {"kCustomCompressionA9", kCustomCompressionA9},
-        {"kCustomCompressionAA", kCustomCompressionAA},
-        {"kCustomCompressionAB", kCustomCompressionAB},
-        {"kCustomCompressionAC", kCustomCompressionAC},
-        {"kCustomCompressionAD", kCustomCompressionAD},
-        {"kCustomCompressionAE", kCustomCompressionAE},
-        {"kCustomCompressionAF", kCustomCompressionAF},
-        {"kCustomCompressionB0", kCustomCompressionB0},
-        {"kCustomCompressionB1", kCustomCompressionB1},
-        {"kCustomCompressionB2", kCustomCompressionB2},
-        {"kCustomCompressionB3", kCustomCompressionB3},
-        {"kCustomCompressionB4", kCustomCompressionB4},
-        {"kCustomCompressionB5", kCustomCompressionB5},
-        {"kCustomCompressionB6", kCustomCompressionB6},
-        {"kCustomCompressionB7", kCustomCompressionB7},
-        {"kCustomCompressionB8", kCustomCompressionB8},
-        {"kCustomCompressionB9", kCustomCompressionB9},
-        {"kCustomCompressionBA", kCustomCompressionBA},
-        {"kCustomCompressionBB", kCustomCompressionBB},
-        {"kCustomCompressionBC", kCustomCompressionBC},
-        {"kCustomCompressionBD", kCustomCompressionBD},
-        {"kCustomCompressionBE", kCustomCompressionBE},
-        {"kCustomCompressionBF", kCustomCompressionBF},
-        {"kCustomCompressionC0", kCustomCompressionC0},
-        {"kCustomCompressionC1", kCustomCompressionC1},
-        {"kCustomCompressionC2", kCustomCompressionC2},
-        {"kCustomCompressionC3", kCustomCompressionC3},
-        {"kCustomCompressionC4", kCustomCompressionC4},
-        {"kCustomCompressionC5", kCustomCompressionC5},
-        {"kCustomCompressionC6", kCustomCompressionC6},
-        {"kCustomCompressionC7", kCustomCompressionC7},
-        {"kCustomCompressionC8", kCustomCompressionC8},
-        {"kCustomCompressionC9", kCustomCompressionC9},
-        {"kCustomCompressionCA", kCustomCompressionCA},
-        {"kCustomCompressionCB", kCustomCompressionCB},
-        {"kCustomCompressionCC", kCustomCompressionCC},
-        {"kCustomCompressionCD", kCustomCompressionCD},
-        {"kCustomCompressionCE", kCustomCompressionCE},
-        {"kCustomCompressionCF", kCustomCompressionCF},
-        {"kCustomCompressionD0", kCustomCompressionD0},
-        {"kCustomCompressionD1", kCustomCompressionD1},
-        {"kCustomCompressionD2", kCustomCompressionD2},
-        {"kCustomCompressionD3", kCustomCompressionD3},
-        {"kCustomCompressionD4", kCustomCompressionD4},
-        {"kCustomCompressionD5", kCustomCompressionD5},
-        {"kCustomCompressionD6", kCustomCompressionD6},
-        {"kCustomCompressionD7", kCustomCompressionD7},
-        {"kCustomCompressionD8", kCustomCompressionD8},
-        {"kCustomCompressionD9", kCustomCompressionD9},
-        {"kCustomCompressionDA", kCustomCompressionDA},
-        {"kCustomCompressionDB", kCustomCompressionDB},
-        {"kCustomCompressionDC", kCustomCompressionDC},
-        {"kCustomCompressionDD", kCustomCompressionDD},
-        {"kCustomCompressionDE", kCustomCompressionDE},
-        {"kCustomCompressionDF", kCustomCompressionDF},
-        {"kCustomCompressionE0", kCustomCompressionE0},
-        {"kCustomCompressionE1", kCustomCompressionE1},
-        {"kCustomCompressionE2", kCustomCompressionE2},
-        {"kCustomCompressionE3", kCustomCompressionE3},
-        {"kCustomCompressionE4", kCustomCompressionE4},
-        {"kCustomCompressionE5", kCustomCompressionE5},
-        {"kCustomCompressionE6", kCustomCompressionE6},
-        {"kCustomCompressionE7", kCustomCompressionE7},
-        {"kCustomCompressionE8", kCustomCompressionE8},
-        {"kCustomCompressionE9", kCustomCompressionE9},
-        {"kCustomCompressionEA", kCustomCompressionEA},
-        {"kCustomCompressionEB", kCustomCompressionEB},
-        {"kCustomCompressionEC", kCustomCompressionEC},
-        {"kCustomCompressionED", kCustomCompressionED},
-        {"kCustomCompressionEE", kCustomCompressionEE},
-        {"kCustomCompressionEF", kCustomCompressionEF},
-        {"kCustomCompressionF0", kCustomCompressionF0},
-        {"kCustomCompressionF1", kCustomCompressionF1},
-        {"kCustomCompressionF2", kCustomCompressionF2},
-        {"kCustomCompressionF3", kCustomCompressionF3},
-        {"kCustomCompressionF4", kCustomCompressionF4},
-        {"kCustomCompressionF5", kCustomCompressionF5},
-        {"kCustomCompressionF6", kCustomCompressionF6},
-        {"kCustomCompressionF7", kCustomCompressionF7},
-        {"kCustomCompressionF8", kCustomCompressionF8},
-        {"kCustomCompressionF9", kCustomCompressionF9},
-        {"kCustomCompressionFA", kCustomCompressionFA},
-        {"kCustomCompressionFB", kCustomCompressionFB},
-        {"kCustomCompressionFC", kCustomCompressionFC},
-        {"kCustomCompressionFD", kCustomCompressionFD},
-        {"kCustomCompressionFE", kCustomCompressionFE},
         {"kDisableCompressionOption", kDisableCompressionOption}};
 
 const std::vector<CompressionType>& GetSupportedCompressions() {
@@ -1037,7 +910,7 @@ Status OptionTypeInfo::Parse(const ConfigOptions& config_options,
                                        : value;
 
     if (opt_ptr == nullptr) {
-      return Status::NotFound("Nullptr option", opt_name);
+      return Status::NotFound("Could not find option", opt_name);
     } else if (parse_func_ != nullptr) {
       ConfigOptions copy = config_options;
       copy.invoke_prepare_options = false;

options/options_helper.h

@@ -72,9 +72,6 @@ std::unique_ptr<Configurable> CFOptionsAsConfigurable(
 Status StringToMap(const std::string& opts_str,
                    std::unordered_map<std::string, std::string>* opts_map);
 
-Status GetStringFromCompressionType(std::string* compression_str,
-                                    CompressionType compression_type);
-
 struct OptionsHelper {
   static const std::string kCFOptionsName /*= "ColumnFamilyOptions"*/;
   static const std::string kDBOptionsName /*= "DBOptions" */;

options/options_settable_test.cc

@@ -129,8 +129,6 @@ TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) {
        sizeof(CacheUsageOptions)},
       {offsetof(struct BlockBasedTableOptions, filter_policy),
        sizeof(std::shared_ptr<const FilterPolicy>)},
-      {offsetof(struct BlockBasedTableOptions, user_defined_index_factory),
-       sizeof(std::shared_ptr<UserDefinedIndexFactory>)},
   };
 
   // In this test, we catch a new option of BlockBasedTableOptions that is not
@@ -705,11 +703,8 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
       new_options->compaction_options_fifo.file_temperature_age_thresholds[0]
           .age,
       12345);
-  // TODO: try to enhance ObjectLibrary to support singletons
-  // ASSERT_EQ(new_options->compression_manager,
-  //           GetBuiltinV2CompressionManager());
-  ASSERT_STREQ(new_options->compression_manager->Name(),
-               GetBuiltinV2CompressionManager()->Name());
+  ASSERT_EQ(new_options->compression_manager,
+            GetBuiltinCompressionManager(/*compression_format_version*/ 2));
 
   ColumnFamilyOptions rnd_filled_options = *new_options;