Finalize patch release 11.0.4

(This essentially reverts commit 2e5f177eeb2532714e5b99f81163a657921a6d5c.)

.github/workflows/clang-tidy-comment.yml

@@ -1,105 +1,58 @@
-name: clang-tidy
+name: Post clang-tidy PR comment
 on:
-  push:
-  pull_request_target:
-    types: [opened, synchronize, reopened]
+  workflow_run:
+    workflows: ["clang-tidy"]
+    types: [completed]
 
 permissions:
   pull-requests: write
 
 jobs:
-  clang-tidy:
-    if: github.repository_owner == 'facebook'
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:24.0
+  comment:
+    if: github.event.workflow_run.event == 'pull_request'
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4.1.0
+    - name: Download clang-tidy results
+      id: download
+      uses: actions/download-artifact@v4.0.0
       with:
-        ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
-    - name: Mark workspace as safe for git
-      run: git config --global --add safe.directory $GITHUB_WORKSPACE
-    - name: Determine diff base
-      id: diff-base
-      run: |
-        if [ "${{ github.event_name }}" = "pull_request_target" ]; then
-          BASE="${{ github.event.pull_request.base.sha }}"
-        else
-          BASE="${{ github.event.before }}"
-        fi
-        if [ -z "$BASE" ] || echo "$BASE" | grep -q '^0\{40\}$'; then
-          echo "skip=true" >> "$GITHUB_OUTPUT"
-          echo "No valid diff base; skipping clang-tidy."
-        else
-          git fetch --depth=1 origin "$BASE"
-          echo "ref=$BASE" >> "$GITHUB_OUTPUT"
-          echo "skip=false" >> "$GITHUB_OUTPUT"
-        fi
-    - name: Install clang-tidy
-      if: steps.diff-base.outputs.skip != 'true'
-      run: apt-get update && apt-get install -y clang-tidy
-    - name: Generate compile_commands.json
-      if: steps.diff-base.outputs.skip != 'true'
-      run: |
-        mkdir build && cd build
-        cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-              -DCMAKE_C_COMPILER=clang-18 \
-              -DCMAKE_CXX_COMPILER=clang++-18 ..
-        cd ..
-        ln -sf build/compile_commands.json compile_commands.json
-    - name: Run clang-tidy on changed files
-      id: clang-tidy
-      if: steps.diff-base.outputs.skip != 'true'
-      run: |
-        python3 tools/run_clang_tidy.py \
-          -j 4 \
-          --diff-base ${{ steps.diff-base.outputs.ref }} \
-          --github-annotations \
-          --github-step-summary \
-          --comment-output clang-tidy-comment.md
+        name: clang-tidy-result
+        run-id: ${{ github.event.workflow_run.id }}
+        github-token: ${{ secrets.GITHUB_TOKEN }}
       continue-on-error: true
-    - name: Post clang-tidy results to PR
-      if: github.event_name == 'pull_request_target' && always() && steps.diff-base.outputs.skip != 'true'
+    - name: Post or update PR comment
+      if: steps.download.outcome == 'success'
       uses: actions/github-script@v7
       with:
         script: |
           const fs = require('fs');
-          const commentPath = 'clang-tidy-comment.md';
-          if (!fs.existsSync(commentPath)) {
-            core.info('No comment file generated; skipping PR comment.');
+          if (!fs.existsSync('clang-tidy-comment.md') || !fs.existsSync('pr_number.txt')) {
+            core.info('No clang-tidy results found; skipping.');
             return;
           }
-          const body = fs.readFileSync(commentPath, 'utf8');
+          const body = fs.readFileSync('clang-tidy-comment.md', 'utf8');
+          const prNumber = parseInt(fs.readFileSync('pr_number.txt', 'utf8').trim());
           const marker = '<!-- clang-tidy-bot -->';
-          const prNumber = context.payload.pull_request.number;
-          try {
-            const { data: comments } = await github.rest.issues.listComments({
+          const { data: comments } = await github.rest.issues.listComments({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: prNumber,
+          });
+          const existing = comments.find(c => c.body.includes(marker));
+          if (existing) {
+            await github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: existing.id,
+              body,
+            });
+            core.info(`Updated existing comment ${existing.id}`);
+          } else {
+            await github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
               issue_number: prNumber,
+              body,
             });
-            const existing = comments.find(c => c.body.includes(marker));
-            if (existing) {
-              await github.rest.issues.updateComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                comment_id: existing.id,
-                body,
-              });
-              core.info(`Updated existing comment ${existing.id}`);
-            } else {
-              await github.rest.issues.createComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: prNumber,
-                body,
-              });
-              core.info('Created new PR comment');
-            }
-          } catch (err) {
-            core.warning(`Could not post PR comment: ${err.message}`);
+            core.info('Created new PR comment');
           }
-    - name: Fail if clang-tidy found issues
-      if: steps.clang-tidy.outcome == 'failure'
-      run: exit 1

.github/workflows/clang-tidy.yml

@@ -0,0 +1,74 @@
+name: clang-tidy
+on:
+  push:
+  pull_request:
+
+permissions: {}
+
+jobs:
+  clang-tidy:
+    if: github.repository_owner == 'facebook'
+    runs-on:
+      labels: 4-core-ubuntu
+    container:
+      image: ghcr.io/facebook/rocksdb_ubuntu:24.0
+    steps:
+    - uses: actions/checkout@v4.1.0
+      with:
+        fetch-depth: 2
+    - name: Mark workspace as safe for git
+      run: git config --global --add safe.directory $GITHUB_WORKSPACE
+    - name: Determine diff base
+      id: diff-base
+      run: |
+        if [ "${{ github.event_name }}" = "pull_request" ]; then
+          BASE="${{ github.event.pull_request.base.sha }}"
+          git fetch --depth=1 origin "$BASE"
+        else
+          BASE="${{ github.event.before }}"
+          if echo "$BASE" | grep -q '^0\{40\}$'; then
+            echo "skip=true" >> "$GITHUB_OUTPUT"
+            echo "New branch push; skipping clang-tidy."
+            exit 0
+          fi
+        fi
+        echo "ref=$BASE" >> "$GITHUB_OUTPUT"
+        echo "skip=false" >> "$GITHUB_OUTPUT"
+    - name: Install clang-tidy
+      if: steps.diff-base.outputs.skip != 'true'
+      run: apt-get update && apt-get install -y clang-tidy
+    - name: Generate compile_commands.json
+      if: steps.diff-base.outputs.skip != 'true'
+      run: |
+        mkdir build && cd build
+        cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+              -DCMAKE_C_COMPILER=clang-18 \
+              -DCMAKE_CXX_COMPILER=clang++-18 ..
+        cd ..
+        ln -sf build/compile_commands.json compile_commands.json
+    - name: Run clang-tidy on changed files
+      id: clang-tidy
+      if: steps.diff-base.outputs.skip != 'true'
+      run: |
+        python3 tools/run_clang_tidy.py \
+          -j 4 \
+          --diff-base ${{ steps.diff-base.outputs.ref }} \
+          --github-annotations \
+          --github-step-summary \
+          --comment-output clang-tidy-comment.md
+      continue-on-error: true
+    - name: Save PR number
+      if: github.event_name == 'pull_request' && always()
+      run: echo "${{ github.event.pull_request.number }}" > pr_number.txt
+    - name: Upload clang-tidy results
+      if: always()
+      uses: actions/upload-artifact@v4.0.0
+      with:
+        name: clang-tidy-result
+        path: |
+          clang-tidy-comment.md
+          pr_number.txt
+        if-no-files-found: ignore
+    - name: Fail if clang-tidy found issues
+      if: steps.clang-tidy.outcome == 'failure'
+      run: exit 1

HISTORY.md

@@ -1,6 +1,45 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
 
+## 11.0.4 (03/11/2026)
+### Public API Changes
+* Re-remove deprecated DB::Open raw pointer variants in db.h
+
+## 11.0.3 (03/03/2026)
+### Public API Changes
+* Relax option sanitization for kv ratio compaction
+
+## 11.0.2 (02/26/2026)
+### Public API Changes
+* Revert removing deprecated DB::Open raw pointer variants in db.h
+
+## 11.0.0 (02/23/2026)
+### New Features
+* Added support for storing wide-column entity column values in blob files. When `min_blob_size` is configured, large column values in wide-column entities will be stored in blob files, reducing SST file size and improving read performance.
+* Added `CompactionOptionsFIFO::max_data_files_size` to support FIFO compaction trimming based on combined SST and blob file sizes. Added `CompactionOptionsFIFO::use_kv_ratio_compaction` to enable a capacity-derived intra-L0 compaction strategy optimized for BlobDB workloads, producing uniform-sized compacted files for predictable FIFO trimming.
+* Include interpolation search as an alternative to binary search, which typically performs better when keys are uniformly distributed. This is exposed as a new table option `index_block_search_type`. The default is `binary_search`.
+
+### Public API Changes
+* Added new virtual methods `AbortAllCompactions()` and `ResumeAllCompactions()` to the `DB` class. Added new `Status::SubCode::kCompactionAborted` to indicate a compaction was aborted. Added `Status::IsCompactionAborted()` helper method to check if a status represents an aborted compaction.
+* Drop support for reading (and writing) SST files using `BlockBasedTableOptions.format_version` < 2, which hasn't been the default format for about 10 years. An upgrade path is still possible with full compaction using a RocksDB version >= 4.6.0 and < 11.0.0 and then using the newer version.
+* Remove deprecated raw `DB*` variants of `DB::Open` and related functions. Some other minor public APIs were updated as a result
+* Remove deprecated `DB::MaxMemCompactionLevel()`
+* Remove useless option `CompressedSecondaryCacheOptions::compress_format_version`
+* Remove deprecated DB option `skip_checking_sst_file_sizes_on_db_open`. The option was deprecated in 10.5.0 and has been a no-op since then. File size validation is now always performed in parallel during DB open.
+* Remove deprecated `SliceTransform::InRange()` virtual method and the `in_range` callback parameter from `rocksdb_slicetransform_create()` in the C API. `InRange()` was never called by RocksDB and existed only for backward compatibility.
+* Remove deprecated, unused APIs and options: `ReadOptions::managed` and `ColumnFamilyOptions::snap_refresh_nanos`. Corresponding C and Java APIs are also removed.
+* Remove deprecated `SstFileWriter::Add()` method (use `Put()` instead) and the deprecated `skip_filters` parameter from `SstFileWriter` constructors (use `BlockBasedTableOptions::filter_policy` set to `nullptr` to skip filter generation instead).
+
+### Behavior Changes
+* Change the default value of `CompactionOptionsUniversal::reduce_file_locking` from `false` to `true` to improve write stall and reduce read regression
+
+### Bug Fixes
+* Fix longstanding failures that can arise from reading and/or compacting old DB dirs with range deletions (likely from version < 5.19.0) in many newer versions.
+* Fix a bug where WritePrepared/WriteUnprepared TransactionDB with two_write_queues=true could experience "sequence number going backwards" corruption during recovery from a background error, due to allocated-but-not-published sequence numbers not being synced before creating new WAL files.
+
+### Performance Improvements
+* Add a new table option `separate_key_value_in_data_block`. When set to true keys and values will be stored separately in the data block, which can result in higher cpu cache hit rate and better compression. Works best with data blocks with sufficient restart intervals and large values. Previous versions of RocksDB will reject files written using this option.
+
 ## 10.11.0 (01/23/2026)
 ### Public API Changes
 * New SetOptions API that allows setting options for multiple CFs, avoiding the need to reserialize OPTIONS file for each CF

db/column_family.cc

@@ -1569,34 +1569,6 @@ Status ColumnFamilyData::ValidateOptions(
         "FIFO compaction only supported with max_open_files = -1.");
   }
 
-  if (cf_options.compaction_options_fifo.use_kv_ratio_compaction) {
-    if (cf_options.compaction_style != kCompactionStyleFIFO) {
-      return Status::InvalidArgument(
-          "use_kv_ratio_compaction is only supported with FIFO compaction "
-          "style.");
-    }
-    if (!cf_options.compaction_options_fifo.allow_compaction) {
-      return Status::InvalidArgument(
-          "use_kv_ratio_compaction requires allow_compaction = true. "
-          "allow_compaction enables intra-L0 compaction, and "
-          "use_kv_ratio_compaction selects the picking strategy.");
-    }
-    if (cf_options.compaction_options_fifo.max_data_files_size == 0) {
-      return Status::InvalidArgument(
-          "use_kv_ratio_compaction requires max_data_files_size > 0 to "
-          "compute the target compacted file size from data capacity.");
-    }
-  }
-
-  if (cf_options.compaction_options_fifo.max_data_files_size > 0 &&
-      cf_options.compaction_options_fifo.max_data_files_size <
-          cf_options.compaction_options_fifo.max_table_files_size) {
-    return Status::InvalidArgument(
-        "max_data_files_size (total data = SST + blob) must be >= "
-        "max_table_files_size (SST only) when non-zero, since total data "
-        "always includes SST data.");
-  }
-
   std::vector<uint32_t> supported{0, 1, 2, 4, 8};
   if (std::find(supported.begin(), supported.end(),
                 cf_options.memtable_protection_bytes_per_key) ==

db/compaction/compaction_picker_fifo.cc

@@ -473,8 +473,28 @@ Compaction* FIFOCompactionPicker::PickIntraL0Compaction(
   }
 
   if (fifo_opts.use_kv_ratio_compaction) {
-    return PickRatioBasedIntraL0Compaction(
-        cf_name, mutable_cf_options, mutable_db_options, vstorage, log_buffer);
+    if (fifo_opts.max_data_files_size == 0) {
+      ROCKS_LOG_BUFFER(
+          log_buffer,
+          "[%s] FIFO kv-ratio compaction: skipping — "
+          "max_data_files_size is 0, cannot compute target file size. ",
+          cf_name.c_str());
+    } else if (fifo_opts.max_data_files_size < fifo_opts.max_table_files_size) {
+      ROCKS_LOG_BUFFER(log_buffer,
+                       "[%s] FIFO kv-ratio compaction: skipping — "
+                       "max_data_files_size (%" PRIu64
+                       ") < max_table_files_size "
+                       "(%" PRIu64 ").",
+                       cf_name.c_str(), fifo_opts.max_data_files_size,
+                       fifo_opts.max_table_files_size);
+    } else {
+      return PickRatioBasedIntraL0Compaction(cf_name, mutable_cf_options,
+                                             mutable_db_options, vstorage,
+                                             log_buffer);
+    }
+    ROCKS_LOG_BUFFER(
+        log_buffer, "[%s] FIFO: falling back to cost-based intra-L0 compaction",
+        cf_name.c_str());
   }
 
   // Old intra-L0 path: merge small files using PickCostBasedIntraL0Compaction.

db/compaction/compaction_picker_test.cc

@@ -5205,8 +5205,8 @@ TEST_F(CompactionPickerTest, FIFORatioBasedCompactionFallbackToOldPath) {
 
 // ============================================================================
 // FIFO Option Validation Tests
-// Tests that ColumnFamilyData::ValidateOptions rejects invalid configurations
-// for use_kv_ratio_compaction.
+// Tests that ColumnFamilyData::ValidateOptions accepts kv_ratio configs
+// (prerequisites are checked at runtime in the picker, not at validation time).
 // ============================================================================
 
 TEST_F(CompactionPickerTest, FIFOOptionValidation) {
@@ -5222,20 +5222,30 @@ TEST_F(CompactionPickerTest, FIFOOptionValidation) {
     return ColumnFamilyData::ValidateOptions(DBOptions(), cf_opts);
   };
 
-  // use_kv_ratio_compaction requires FIFO compaction style
-  ASSERT_TRUE(validate([](auto& o) {
-                o.compaction_style = kCompactionStyleLevel;
-              }).IsInvalidArgument());
+  // These used to be InvalidArgument, now accepted (handled at runtime
+  // in the picker with fallback to cost-based intra-L0 compaction):
 
-  // use_kv_ratio_compaction requires allow_compaction
-  ASSERT_TRUE(validate([](auto& o) {
-                o.compaction_options_fifo.allow_compaction = false;
-              }).IsInvalidArgument());
+  // use_kv_ratio_compaction with non-FIFO compaction style
+  ASSERT_OK(
+      validate([](auto& o) { o.compaction_style = kCompactionStyleLevel; }));
 
-  // use_kv_ratio_compaction requires max_data_files_size > 0
-  ASSERT_TRUE(validate([](auto& o) {
-                o.compaction_options_fifo.max_data_files_size = 0;
-              }).IsInvalidArgument());
+  // use_kv_ratio_compaction without allow_compaction
+  ASSERT_OK(validate(
+      [](auto& o) { o.compaction_options_fifo.allow_compaction = false; }));
+
+  // use_kv_ratio_compaction without max_data_files_size
+  ASSERT_OK(validate(
+      [](auto& o) { o.compaction_options_fifo.max_data_files_size = 0; }));
+
+  // max_data_files_size < max_table_files_size
+  ASSERT_OK(validate([](auto& o) {
+    o.compaction_options_fifo.use_kv_ratio_compaction = false;
+    o.compaction_options_fifo.max_data_files_size = 0;
+    o.compaction_options_fifo.max_table_files_size = 1ULL * 1024 * 1024 * 1024;
+    o.compaction_options_fifo.max_data_files_size = 500ULL * 1024 * 1024;
+  }));
+
+  // These should still pass:
 
   // Accepts multi-level (for migration from level/universal to FIFO)
   ASSERT_OK(validate([](auto& o) { o.num_levels = 4; }));
@@ -5243,16 +5253,6 @@ TEST_F(CompactionPickerTest, FIFOOptionValidation) {
   // Accepts valid single-level config
   ASSERT_OK(validate([](auto& /*o*/) {}));
 
-  // max_data_files_size < max_table_files_size is invalid when non-zero
-  ASSERT_TRUE(validate([](auto& o) {
-                o.compaction_options_fifo.use_kv_ratio_compaction = false;
-                o.compaction_options_fifo.max_data_files_size = 0;
-                o.compaction_options_fifo.max_table_files_size =
-                    1ULL * 1024 * 1024 * 1024;
-                o.compaction_options_fifo.max_data_files_size =
-                    500ULL * 1024 * 1024;
-              }).IsInvalidArgument());
-
   // max_data_files_size == max_table_files_size is valid
   ASSERT_OK(validate([](auto& o) {
     o.compaction_options_fifo.use_kv_ratio_compaction = false;
@@ -5262,6 +5262,50 @@ TEST_F(CompactionPickerTest, FIFOOptionValidation) {
   }));
 }
 
+TEST_F(CompactionPickerTest, FIFORatioBasedFallbackOnInvalidConfig) {
+  // When use_kv_ratio_compaction is true but prerequisites are not met,
+  // the picker should fall back to cost-based intra-L0 compaction.
+
+  // Sub-test 1: max_data_files_size == 0 -> falls back to cost-based
+  {
+    SetupFIFORatioBased(/*max_table_files_size=*/10 * 1024 * 1024,
+                        /*max_data_files_size=*/0,
+                        /*trigger=*/4,
+                        /*allow_compaction=*/true,
+                        /*use_kv_ratio=*/true);
+    mutable_cf_options_.max_compaction_bytes = 10ULL * 1024 * 1024;
+    FIFOCompactionPicker picker(ioptions_, &icmp_);
+    Add(0, 1U, "100", "200", 64 * 1024);
+    Add(0, 2U, "200", "300", 64 * 1024);
+    Add(0, 3U, "300", "400", 64 * 1024);
+    Add(0, 4U, "400", "500", 64 * 1024);
+    auto compaction = PickFIFOCompaction(picker);
+    // kv_ratio skipped, cost-based should pick these files
+    ASSERT_NE(nullptr, compaction.get());
+    ASSERT_EQ(CompactionReason::kFIFOReduceNumFiles,
+              compaction->compaction_reason());
+  }
+
+  // Sub-test 2: max_data_files_size < max_table_files_size -> falls back
+  {
+    SetupFIFORatioBased(/*max_table_files_size=*/1ULL * 1024 * 1024 * 1024,
+                        /*max_data_files_size=*/500ULL * 1024 * 1024,
+                        /*trigger=*/4,
+                        /*allow_compaction=*/true,
+                        /*use_kv_ratio=*/true);
+    mutable_cf_options_.max_compaction_bytes = 10ULL * 1024 * 1024;
+    FIFOCompactionPicker picker(ioptions_, &icmp_);
+    Add(0, 1U, "100", "200", 64 * 1024);
+    Add(0, 2U, "200", "300", 64 * 1024);
+    Add(0, 3U, "300", "400", 64 * 1024);
+    Add(0, 4U, "400", "500", 64 * 1024);
+    auto compaction = PickFIFOCompaction(picker);
+    ASSERT_NE(nullptr, compaction.get());
+    ASSERT_EQ(CompactionReason::kFIFOReduceNumFiles,
+              compaction->compaction_reason());
+  }
+}
+
 // ============================================================================
 // FIFO Ratio-Based Compaction: Multi-Level Migration Graceful Skip
 // Tests that PickRatioBasedIntraL0Compaction gracefully skips when non-L0

db_stress_tool/db_stress_gflags.cc

@@ -428,8 +428,9 @@ DEFINE_uint64(fifo_compaction_max_data_files_size_mb, 0,
 DEFINE_bool(fifo_compaction_use_kv_ratio_compaction, false,
             "If true, set "
             "`Options::compaction_options_fifo.use_kv_ratio_compaction = "
-            "true`. Requires fifo_allow_compaction and "
-            "fifo_compaction_max_data_files_size_mb > 0.");
+            "true`. Recommends fifo_allow_compaction and "
+            "fifo_compaction_max_data_files_size_mb > 0 (falls back to "
+            "cost-based intra-L0 compaction if not set).");
 
 DEFINE_bool(allow_concurrent_memtable_write, false,
             "Allow multi-writers to update mem tables in parallel.");

db_stress_tool/db_stress_test_base.cc

@@ -4384,11 +4384,6 @@ void InitializeOptionsFromFlags(
     if (FLAGS_fifo_compaction_max_data_files_size_mb > 0) {
       options.compaction_options_fifo.max_data_files_size =
           FLAGS_fifo_compaction_max_data_files_size_mb * 1024 * 1024;
-      // max_table_files_size is ignored when max_data_files_size is non-zero,
-      // but validation requires max_data_files_size >= max_table_files_size.
-      options.compaction_options_fifo.max_table_files_size =
-          std::min(options.compaction_options_fifo.max_table_files_size,
-                   options.compaction_options_fifo.max_data_files_size);
     }
     options.compaction_options_fifo.use_kv_ratio_compaction =
         FLAGS_fifo_compaction_use_kv_ratio_compaction;

include/rocksdb/advanced_options.h

@@ -160,10 +160,11 @@ struct CompactionOptionsFIFO {
   // explicitly set to a non-zero value, it overrides the auto-calculated
   // target.
   //
-  // Requires:
+  // Recommends:
   //   - allow_compaction = true (master switch for intra-L0 compaction)
   //   - max_data_files_size > 0 (needed to compute the target file size)
-  // Setting this to true without these will fail option validation.
+  // If these are not met, kv_ratio compaction is skipped and the old
+  // cost-based intra-L0 compaction algorithm is used as a fallback.
   //
   // When false, the old intra-L0 strategy is used if allow_compaction is
   // true (PickCostBasedIntraL0Compaction with 1.1 * write_buffer_size guard).

include/rocksdb/utilities/ldb_cmd.h

@@ -75,6 +75,7 @@ class LDBCommand {
   static const std::string ARG_BLOB_FILE_STARTING_LEVEL;
   static const std::string ARG_PREPOPULATE_BLOB_CACHE;
   static const std::string ARG_DECODE_BLOB_INDEX;
+  static const std::string ARG_DUMP_UNCOMPRESSED_BLOBS;
   static const std::string ARG_READ_TIMESTAMP;
   static const std::string ARG_GET_WRITE_UNIX_TIME;
 

include/rocksdb/version.h

@@ -13,7 +13,7 @@
 // minor or major version number planned for release.
 #define ROCKSDB_MAJOR 11
 #define ROCKSDB_MINOR 0
-#define ROCKSDB_PATCH 0
+#define ROCKSDB_PATCH 4
 
 // Make it easy to do conditional compilation based on version checks, i.e.
 // #if ROCKSDB_VERSION_GE(4, 5, 6)

table/block_based/block.cc

@@ -687,6 +687,15 @@ bool BlockIter<TValue>::ParseNextKey(bool* is_shared) {
     CorruptionError();
     return false;
   } else {
+    if constexpr (StrictCheck) {
+      auto entry_length =
+          non_shared + (values_section_ == nullptr ? value_length : 0);
+      if (static_cast<uint32_t>(key_limit - p) < entry_length) {
+        CorruptionError();
+        return false;
+      }
+    }
+
     assert(values_section_ == nullptr ||
            cur_entry_idx_ % block_restart_interval_ != 0 || shared == 0);
     entry_ = Slice(p_old, p - p_old + non_shared);
@@ -726,18 +735,18 @@ bool BlockIter<TValue>::ParseNextKey(bool* is_shared) {
         assert(value_.data() >= values_section_);
         value_ = Slice(value_.data() + value_.size(), value_length);
       }
+
+      if constexpr (StrictCheck) {
+        if ((value_.data() + value_.size()) > data_ + restarts_) {
+          CorruptionError();
+          return false;
+        }
+      }
     } else {
       value_ = Slice(entry_.data() + entry_.size(), value_length);
       // extend entry slice to contain value as well
       entry_ = Slice(entry_.data(), entry_.size() + value_.size());
     }
-
-    if constexpr (StrictCheck) {
-      if ((value_.data() + value_.size()) > data_ + restarts_) {
-        CorruptionError();
-        return false;
-      }
-    }
     assert((value_.data() + value_.size()) <= data_ + restarts_);
     return true;
   }

table/block_based/block_test.cc

@@ -25,6 +25,7 @@
 #include "rocksdb/table.h"
 #include "table/block_based/block_based_table_reader.h"
 #include "table/block_based/block_builder.h"
+#include "table/block_based/data_block_footer.h"
 #include "table/format.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
@@ -1942,6 +1943,105 @@ TEST_P(MetaIndexBlockKVChecksumCorruptionTest, CorruptEntry) {
     }
   }
 }
+
+class MetaBlockEntryCorruptionTest : public testing::TestWithParam<bool> {
+ public:
+  bool useSeparatedKVStorage() const { return GetParam(); }
+
+  std::string BuildBlock() {
+    BlockBuilder builder(1 /* restart_interval */,
+                         true /* use_delta_encoding */,
+                         false /* use_value_delta_encoding */,
+                         BlockBasedTableOptions::kDataBlockBinarySearch,
+                         0 /* data_block_hash_table_util_ratio */,
+                         0 /* ts_sz */, false /* persist_udt */,
+                         true /* is_user_key */, useSeparatedKVStorage());
+    builder.Add("key001", "val01");
+    builder.Add("key002", "val02");
+    builder.Add("key003", "val03");
+    builder.Add("key004", "val04");
+    Slice raw = builder.Finish();
+    return std::string(raw.data(), raw.size());
+  }
+
+  // Get the restart offset for a given restart index from the raw block data.
+  uint32_t GetRestartOffset(const std::string& block_data, int restart_idx) {
+    size_t footer_size = useSeparatedKVStorage() ? 8 : 4;
+    uint32_t packed = DecodeFixed32(block_data.data() + block_data.size() - 4);
+    uint32_t num_restarts = packed & DataBlockFooter::kMaxNumRestarts;
+    size_t restarts_start =
+        block_data.size() - footer_size - num_restarts * sizeof(uint32_t);
+    return DecodeFixed32(block_data.data() + restarts_start +
+                         restart_idx * sizeof(uint32_t));
+  }
+
+  uint32_t GetKeyEnd(const std::string& block_data) {
+    size_t footer_size = useSeparatedKVStorage() ? 8 : 4;
+    uint32_t packed = DecodeFixed32(block_data.data() + block_data.size() - 4);
+    uint32_t num_restarts = packed & DataBlockFooter::kMaxNumRestarts;
+    uint32_t restarts_start = static_cast<uint32_t>(
+        block_data.size() - footer_size - num_restarts * sizeof(uint32_t));
+    if (useSeparatedKVStorage()) {
+      // values_section_offset is stored as the 4 bytes before the packed word.
+      return DecodeFixed32(block_data.data() + block_data.size() - 8);
+    }
+    return restarts_start;
+  }
+
+  uint32_t GetValueEnd(const std::string& block_data) {
+    size_t footer_size = useSeparatedKVStorage() ? 8 : 4;
+    uint32_t packed = DecodeFixed32(block_data.data() + block_data.size() - 4);
+    uint32_t num_restarts = packed & DataBlockFooter::kMaxNumRestarts;
+    return static_cast<uint32_t>(block_data.size() - footer_size -
+                                 num_restarts * sizeof(uint32_t));
+  }
+};
+
+INSTANTIATE_TEST_CASE_P(P, MetaBlockEntryCorruptionTest, ::testing::Bool(),
+                        [](const testing::TestParamInfo<bool>& args) {
+                          return args.param ? "SeparatedKV" : "InlineKV";
+                        });
+
+TEST_P(MetaBlockEntryCorruptionTest, CorruptedKeyLengthPastKeyEnd) {
+  std::string block_data = BuildBlock();
+  uint32_t key_end = GetKeyEnd(block_data);
+
+  // Corrupt the key length of the first entry so that
+  // the key data would extend past the keys-end boundary.
+  uint32_t first_entry_offset = GetRestartOffset(block_data, 0);
+  block_data[first_entry_offset + 1] = static_cast<char>(key_end);
+
+  BlockContents contents;
+  contents.data = Slice(block_data);
+  Block block(std::move(contents), 0, nullptr, 1 /* restart_interval */);
+  std::unique_ptr<MetaBlockIter> iter(block.NewMetaIterator());
+
+  // SeekToFirst should hit the corrupted first entry immediately.
+  iter->SeekToFirst();
+  ASSERT_FALSE(iter->Valid());
+  ASSERT_TRUE(iter->status().IsCorruption());
+}
+
+TEST_P(MetaBlockEntryCorruptionTest, CorruptedValueLengthPastValueEnd) {
+  std::string block_data = BuildBlock();
+  uint32_t value_end = GetValueEnd(block_data);
+
+  // Corrupt the first entry so that its value would extend past the
+  // values-end boundary (start of the restart array).
+  uint32_t first_entry_offset = GetRestartOffset(block_data, 0);
+  block_data[first_entry_offset + 2] = static_cast<char>(value_end);
+
+  BlockContents contents;
+  contents.data = Slice(block_data);
+  Block block(std::move(contents), 0, nullptr, 1 /* restart_interval */);
+  std::unique_ptr<MetaBlockIter> iter(block.NewMetaIterator());
+
+  // SeekToFirst should hit the corrupted first entry immediately.
+  iter->SeekToFirst();
+  ASSERT_FALSE(iter->Valid());
+  ASSERT_TRUE(iter->status().IsCorruption());
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

table/sst_file_reader.cc

@@ -93,6 +93,7 @@ std::vector<Status> SstFileReader::MultiGet(const ReadOptions& roptions,
   auto r = rep_.get();
   const Comparator* user_comparator =
       r->ioptions.internal_comparator.user_comparator();
+  Statistics* statistics = r->ioptions.stats;
 
   autovector<KeyContext, MultiGetContext::MAX_BATCH_SIZE> key_context;
   autovector<KeyContext*, MultiGetContext::MAX_BATCH_SIZE> sorted_keys;
@@ -105,11 +106,12 @@ std::vector<Status> SstFileReader::MultiGet(const ReadOptions& roptions,
     merge_ctx.emplace_back();
     key_context.emplace_back(nullptr, keys[i], val, nullptr,
                              nullptr /* timestamp */, &statuses[i]);
-    get_ctx.emplace_back(user_comparator, r->ioptions.merge_operator.get(),
-                         nullptr, nullptr, GetContext::kNotFound,
-                         *key_context[i].key, val, nullptr, nullptr, nullptr,
-                         &merge_ctx[i], true,
-                         &key_context[i].max_covering_tombstone_seq, nullptr);
+    get_ctx.emplace_back(
+        user_comparator, r->ioptions.merge_operator.get(), nullptr /* logger */,
+        statistics, GetContext::kNotFound, *key_context[i].key, val,
+        nullptr /* columns */, nullptr /* timestamp */,
+        nullptr /* value_found */, &merge_ctx[i], true,
+        &key_context[i].max_covering_tombstone_seq, r->ioptions.clock);
     key_context[i].get_context = &get_ctx[i];
   }
   for (size_t i = 0; i < num_keys; ++i) {
@@ -139,6 +141,8 @@ std::vector<Status> SstFileReader::MultiGet(const ReadOptions& roptions,
 
   values->resize(num_keys);
   for (size_t i = 0; i < num_keys; ++i) {
+    get_ctx[i].ReportCounters();
+
     if (statuses[i].ok()) {
       switch (get_ctx[i].State()) {
         case GetContext::kFound:

table/sst_file_reader_test.cc

@@ -801,6 +801,7 @@ TEST_F(SstFileReaderTableMultiGetTest, Basic) {
   options.comparator = ucmp;
   options.disable_auto_compactions = true;
   options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  options.statistics = CreateDBStatistics();
   BlockBasedTableOptions bbto;
   bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
   options.table_factory.reset(NewBlockBasedTableFactory(bbto));
@@ -830,6 +831,8 @@ TEST_F(SstFileReaderTableMultiGetTest, Basic) {
   ASSERT_OK(reader.Open(file_name));
   ASSERT_OK(reader.VerifyChecksum());
 
+  ASSERT_OK(options.statistics->Reset());
+
   std::vector<Slice> keys;
   std::vector<std::string> values;
 
@@ -850,6 +853,10 @@ TEST_F(SstFileReaderTableMultiGetTest, Basic) {
   ASSERT_TRUE(statuses[4].ok());
   ASSERT_EQ("val4,val5", values[4]);
 
+  uint64_t cache_hits = options.statistics->getTickerCount(BLOCK_CACHE_HIT);
+  uint64_t cache_misses = options.statistics->getTickerCount(BLOCK_CACHE_MISS);
+  ASSERT_GT(cache_hits + cache_misses, 0);
+
   dbfull()->ReleaseSnapshot(snapshot1);
   dbfull()->ReleaseSnapshot(snapshot2);
   dbfull()->ReleaseSnapshot(snapshot3);

tools/blob_dump.cc

@@ -27,10 +27,12 @@ int main(int argc, char** argv) {
       {"file", required_argument, nullptr, 'f'},
       {"show_key", optional_argument, nullptr, 'k'},
       {"show_blob", optional_argument, nullptr, 'b'},
+      {"show_uncompressed_blob", optional_argument, nullptr, 'r'},
       {"show_summary", optional_argument, nullptr, 's'},
   };
   DisplayType show_key = DisplayType::kRaw;
   DisplayType show_blob = DisplayType::kNone;
+  DisplayType show_uncompressed_blob = DisplayType::kNone;
   bool show_summary = false;
   std::string file;
   while (true) {
@@ -45,6 +47,7 @@ int main(int argc, char** argv) {
                 "Usage: blob_dump --file=filename "
                 "[--show_key[=none|raw|hex|detail]] "
                 "[--show_blob[=none|raw|hex|detail]] "
+                "[--show_uncompressed_blob[=none|raw|hex|detail]] "
                 "[--show_summary]\n");
         return 0;
       case 'f':
@@ -70,6 +73,17 @@ int main(int argc, char** argv) {
           show_blob = DisplayType::kHex;
         }
         break;
+      case 'r':
+        if (optarg) {
+          if (display_types.count(arg_str) == 0) {
+            fprintf(stderr, "Unrecognized blob display type.\n");
+            return -1;
+          }
+          show_uncompressed_blob = display_types.at(arg_str);
+        } else {
+          show_uncompressed_blob = DisplayType::kHex;
+        }
+        break;
       case 's':
         show_summary = true;
         break;
@@ -79,7 +93,8 @@ int main(int argc, char** argv) {
     }
   }
   BlobDumpTool tool;
-  Status s = tool.Run(file, show_key, show_blob, show_summary);
+  Status s =
+      tool.Run(file, show_key, show_blob, show_uncompressed_blob, show_summary);
   if (!s.ok()) {
     fprintf(stderr, "Failed: %s\n", s.ToString().c_str());
     return -1;

tools/db_crashtest.py

@@ -972,14 +972,6 @@ def finalize_and_sanitize(src_params):
         # Disable irrelevant tiering options
         dest_params["preclude_last_level_data_seconds"] = 0
         dest_params["last_level_temperature"] = "kUnknown"
-        # use_kv_ratio_compaction requires allow_compaction and
-        # max_data_files_size > 0
-        if dest_params.get("fifo_compaction_use_kv_ratio_compaction", 0) == 1:
-            if (
-                dest_params.get("fifo_allow_compaction", 0) != 1
-                or dest_params.get("fifo_compaction_max_data_files_size_mb", 0) == 0
-            ):
-                dest_params["fifo_compaction_use_kv_ratio_compaction"] = 0
     else:
         # Disable irrelevant tiering options
         dest_params["file_temperature_age_thresholds"] = ""

tools/ldb_cmd.cc

@@ -112,6 +112,8 @@ const std::string LDBCommand::ARG_BLOB_FILE_STARTING_LEVEL =
 const std::string LDBCommand::ARG_PREPOPULATE_BLOB_CACHE =
     "prepopulate_blob_cache";
 const std::string LDBCommand::ARG_DECODE_BLOB_INDEX = "decode_blob_index";
+const std::string LDBCommand::ARG_DUMP_UNCOMPRESSED_BLOBS =
+    "dump_uncompressed_blobs";
 const std::string LDBCommand::ARG_READ_TIMESTAMP = "read_timestamp";
 const std::string LDBCommand::ARG_GET_WRITE_UNIX_TIME = "get_write_unix_time";
 
@@ -201,7 +203,7 @@ void DumpSstFile(Options options, std::string filename, bool output_hex,
                  std::string from_key = "", std::string to_key = "");
 
 void DumpBlobFile(const std::string& filename, bool is_key_hex,
-                  bool is_value_hex);
+                  bool is_value_hex, bool dump_uncompressed_blobs);
 
 Status EncodeUserProvidedTimestamp(const std::string& user_timestamp,
                                    std::string* ts_buf);
@@ -2332,19 +2334,21 @@ DBDumperCommand::DBDumperCommand(
     const std::vector<std::string>& /*params*/,
     const std::map<std::string, std::string>& options,
     const std::vector<std::string>& flags)
-    : LDBCommand(options, flags, true /* is_read_only */,
-                 BuildCmdLineOptions(
-                     {ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
-                      ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_COUNT_DELIM,
-                      ARG_STATS, ARG_TTL_START, ARG_TTL_END, ARG_TTL_BUCKET,
-                      ARG_TIMESTAMP, ARG_PATH, ARG_DECODE_BLOB_INDEX})),
+    : LDBCommand(
+          options, flags, true /* is_read_only */,
+          BuildCmdLineOptions(
+              {ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO,
+               ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_COUNT_DELIM, ARG_STATS,
+               ARG_TTL_START, ARG_TTL_END, ARG_TTL_BUCKET, ARG_TIMESTAMP,
+               ARG_PATH, ARG_DECODE_BLOB_INDEX, ARG_DUMP_UNCOMPRESSED_BLOBS})),
       null_from_(true),
       null_to_(true),
       max_keys_(-1),
       count_only_(false),
       count_delim_(false),
       print_stats_(false),
-      decode_blob_index_(false) {
+      decode_blob_index_(false),
+      dump_uncompressed_blobs_(false) {
   auto itr = options.find(ARG_FROM);
   if (itr != options.end()) {
     null_from_ = false;
@@ -2385,6 +2389,7 @@ DBDumperCommand::DBDumperCommand(
   print_stats_ = IsFlagPresent(flags, ARG_STATS);
   count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
   decode_blob_index_ = IsFlagPresent(flags, ARG_DECODE_BLOB_INDEX);
+  dump_uncompressed_blobs_ = IsFlagPresent(flags, ARG_DUMP_UNCOMPRESSED_BLOBS);
 
   if (is_key_hex_) {
     if (!null_from_) {
@@ -2419,6 +2424,7 @@ void DBDumperCommand::Help(std::string& ret) {
   ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
   ret.append(" [--" + ARG_PATH + "=<path_to_a_file>]");
   ret.append(" [--" + ARG_DECODE_BLOB_INDEX + "]");
+  ret.append(" [--" + ARG_DUMP_UNCOMPRESSED_BLOBS + "]");
   ret.append("\n");
 }
 
@@ -2465,7 +2471,8 @@ void DBDumperCommand::DoCommand() {
                          /*  json_ */ false, column_families_);
         break;
       case kBlobFile:
-        DumpBlobFile(path_, is_key_hex_, is_value_hex_);
+        DumpBlobFile(path_, is_key_hex_, is_value_hex_,
+                     dump_uncompressed_blobs_);
         break;
       default:
         exec_state_ = LDBCommandExecuteResult::Failed(
@@ -4758,16 +4765,21 @@ void DumpSstFile(Options options, std::string filename, bool output_hex,
 }
 
 void DumpBlobFile(const std::string& filename, bool is_key_hex,
-                  bool is_value_hex) {
+                  bool is_value_hex, bool dump_uncompressed_blobs) {
   using ROCKSDB_NAMESPACE::blob_db::BlobDumpTool;
   BlobDumpTool tool;
-  BlobDumpTool::DisplayType show_blob = is_value_hex
+  BlobDumpTool::DisplayType blob_type = is_value_hex
                                             ? BlobDumpTool::DisplayType::kHex
                                             : BlobDumpTool::DisplayType::kRaw;
+  BlobDumpTool::DisplayType show_uncompressed_blob =
+      dump_uncompressed_blobs ? blob_type : BlobDumpTool::DisplayType::kNone;
+  BlobDumpTool::DisplayType show_blob =
+      dump_uncompressed_blobs ? BlobDumpTool::DisplayType::kNone : blob_type;
   BlobDumpTool::DisplayType show_key = is_key_hex
                                            ? BlobDumpTool::DisplayType::kHex
                                            : BlobDumpTool::DisplayType::kRaw;
-  Status s = tool.Run(filename, show_key, show_blob, /* show_summary */ true);
+  Status s = tool.Run(filename, show_key, show_blob, show_uncompressed_blob,
+                      /* show_summary */ true);
   if (!s.ok()) {
     fprintf(stderr, "Failed: %s\n", s.ToString().c_str());
   }
@@ -4791,13 +4803,17 @@ DBFileDumperCommand::DBFileDumperCommand(
     const std::map<std::string, std::string>& options,
     const std::vector<std::string>& flags)
     : LDBCommand(options, flags, true /* is_read_only */,
-                 BuildCmdLineOptions({ARG_DECODE_BLOB_INDEX})),
-      decode_blob_index_(IsFlagPresent(flags, ARG_DECODE_BLOB_INDEX)) {}
+                 BuildCmdLineOptions(
+                     {ARG_DECODE_BLOB_INDEX, ARG_DUMP_UNCOMPRESSED_BLOBS})),
+      decode_blob_index_(IsFlagPresent(flags, ARG_DECODE_BLOB_INDEX)),
+      dump_uncompressed_blobs_(
+          IsFlagPresent(flags, ARG_DUMP_UNCOMPRESSED_BLOBS)) {}
 
 void DBFileDumperCommand::Help(std::string& ret) {
   ret.append("  ");
   ret.append(DBFileDumperCommand::Name());
   ret.append(" [--" + ARG_DECODE_BLOB_INDEX + "]");
+  ret.append(" [--" + ARG_DUMP_UNCOMPRESSED_BLOBS + "]");
   ret.append("\n");
 }
 
@@ -4865,7 +4881,8 @@ void DBFileDumperCommand::DoCommand() {
       filename = NormalizePath(filename);
       std::cout << filename << std::endl;
       std::cout << "------------------------------" << std::endl;
-      DumpBlobFile(filename, /* is_key_hex */ false, /* is_value_hex */ false);
+      DumpBlobFile(filename, /* is_key_hex */ false, /* is_value_hex */ false,
+                   dump_uncompressed_blobs_);
       std::cout << std::endl;
     }
   }

tools/ldb_cmd_impl.h

@@ -47,6 +47,7 @@ class DBFileDumperCommand : public LDBCommand {
 
  private:
   bool decode_blob_index_;
+  bool dump_uncompressed_blobs_;
 };
 
 class DBLiveFilesMetadataDumperCommand : public LDBCommand {
@@ -108,6 +109,7 @@ class DBDumperCommand : public LDBCommand {
   bool print_stats_;
   std::string path_;
   bool decode_blob_index_;
+  bool dump_uncompressed_blobs_;
 
   static const std::string ARG_COUNT_ONLY;
   static const std::string ARG_COUNT_DELIM;

tools/ldb_test.py

@@ -2,7 +2,6 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 
 import glob
-
 import os
 import os.path
 import re
@@ -99,7 +98,9 @@ class LDBTestCase(unittest.TestCase):
         Uses the default test db.
         """
         self.assertRunOKFull(
-            "{} {}".format(self.dbParam(self.DB_NAME), params), expectedOutput, unexpected
+            "{} {}".format(self.dbParam(self.DB_NAME), params),
+            expectedOutput,
+            unexpected,
         )
 
     def assertRunFAIL(self, params):
@@ -186,11 +187,15 @@ class LDBTestCase(unittest.TestCase):
 
     def writeExternSst(self, params, inputDumpFile, outputSst):
         return 0 == run_err_null(
-            "cat {} | ./ldb write_extern_sst {} {}".format(inputDumpFile, outputSst, params)
+            "cat {} | ./ldb write_extern_sst {} {}".format(
+                inputDumpFile, outputSst, params
+            )
         )
 
     def ingestExternSst(self, params, inputSst):
-        return 0 == run_err_null("./ldb ingest_extern_sst {} {}".format(inputSst, params))
+        return 0 == run_err_null(
+            "./ldb ingest_extern_sst {} {}".format(inputSst, params)
+        )
 
     def testStringBatchPut(self):
         print("Running testStringBatchPut...")
@@ -312,8 +317,12 @@ class LDBTestCase(unittest.TestCase):
         self.assertRunOK("get --hex 0x6131", "0x6231")
         self.assertRunOK("get a2", "b2")
         self.assertRunOK("get --hex 0x6132", "0x6232")
-        self.assertRunOK("multi_get --hex 0x6131 0x6132", "0x6131 ==> 0x6231\n0x6132 ==> 0x6232")
-        self.assertRunOK("multi_get --hex 0x6131 0xBEEF", "0x6131 ==> 0x6231\nKey not found: 0xBEEF")
+        self.assertRunOK(
+            "multi_get --hex 0x6131 0x6132", "0x6131 ==> 0x6231\n0x6132 ==> 0x6232"
+        )
+        self.assertRunOK(
+            "multi_get --hex 0x6131 0xBEEF", "0x6131 ==> 0x6231\nKey not found: 0xBEEF"
+        )
         self.assertRunOK("get --key_hex 0x6132", "b2")
         self.assertRunOK("get --key_hex --value_hex 0x6132", "0x6232")
         self.assertRunOK("get --value_hex a2", "0x6232")
@@ -540,7 +549,9 @@ class LDBTestCase(unittest.TestCase):
             "'b' seq:2, type:1 => val\nInternal keys in range: 2",
         )
         self.assertRunOK(
-            "idump --input_key_hex --from={} --to={}".format(hex(ord("a")), hex(ord("b"))),
+            "idump --input_key_hex --from={} --to={}".format(
+                hex(ord("a")), hex(ord("b"))
+            ),
             "'a' seq:1, type:1 => val\nInternal keys in range: 1",
         )
 
@@ -626,7 +637,9 @@ class LDBTestCase(unittest.TestCase):
         self.assertRunFAIL("checkconsistency")
 
     def dumpLiveFiles(self, params, dumpFile):
-        return 0 == run_err_null("./ldb dump_live_files {} > {}".format(params, dumpFile))
+        return 0 == run_err_null(
+            "./ldb dump_live_files {} > {}".format(params, dumpFile)
+        )
 
     def testDumpLiveFiles(self):
         print("Running testDumpLiveFiles...")
@@ -651,7 +664,7 @@ class LDBTestCase(unittest.TestCase):
         # Call the dump_live_files function with the edited dbPath name.
         self.assertTrue(
             self.dumpLiveFiles(
-                "--db=%s --decode_blob_index" % dbPath,
+                "--db=%s --decode_blob_index --dump_uncompressed_blobs" % dbPath,
                 dumpFilePath,
             )
         )
@@ -919,7 +932,7 @@ class LDBTestCase(unittest.TestCase):
         expected_pattern = re.compile(regex)
         blob_files = self.getBlobFiles(dbPath)
         self.assertTrue(len(blob_files) >= 1)
-        cmd = "dump --path=%s"
+        cmd = "dump --path=%s --dump_uncompressed_blobs"
         self.assertRunOKFull(
             (cmd) % (blob_files[0]), expected_pattern, unexpected=False, isPattern=True
         )

unreleased_history/behavior_changes/reduce_file_locking_default_true.md

@@ -1 +0,0 @@
-Change the default value of `CompactionOptionsUniversal::reduce_file_locking` from `false` to `true` to improve write stall and reduce read regression

unreleased_history/bug_fixes/deleterange_format_compatible.md

@@ -1 +0,0 @@
-* Fix longstanding failures that can arise from reading and/or compacting old DB dirs with range deletions (likely from version < 5.19.0) in many newer versions.

unreleased_history/bug_fixes/txn_two_write_queues_seqno_recovery.md

@@ -1 +0,0 @@
-Fix a bug where WritePrepared/WriteUnprepared TransactionDB with two_write_queues=true could experience "sequence number going backwards" corruption during recovery from a background error, due to allocated-but-not-published sequence numbers not being synced before creating new WAL files.

unreleased_history/new_features/fifo_kv_ratio_compaction.md

@@ -1 +0,0 @@
-Added `CompactionOptionsFIFO::max_data_files_size` to support FIFO compaction trimming based on combined SST and blob file sizes. Added `CompactionOptionsFIFO::use_kv_ratio_compaction` to enable a capacity-derived intra-L0 compaction strategy optimized for BlobDB workloads, producing uniform-sized compacted files for predictable FIFO trimming.

unreleased_history/new_features/interpolation_search

@@ -1 +0,0 @@
-Include interpolation search as an alternative to binary search, which typically performs better when keys are uniformly distributed. This is exposed as a new table option `index_block_search_type`. The default is `binary_search`.

unreleased_history/new_features/wide_column_blob_support.md

@@ -1 +0,0 @@
-Added support for storing wide-column entity column values in blob files. When `min_blob_size` is configured, large column values in wide-column entities will be stored in blob files, reducing SST file size and improving read performance.

unreleased_history/performance_improvements/separate_kv_in_data_blocks

@@ -1 +0,0 @@
-Add a new table option `separate_key_value_in_data_block`. When set to true keys and values will be stored separately in the data block, which can result in higher cpu cache hit rate and better compression. Works best with data blocks with sufficient restart intervals and large values. Previous versions of RocksDB will reject files written using this option.

unreleased_history/public_api_changes/abort_compaction_apis.md

@@ -1 +0,0 @@
-Added new virtual methods `AbortAllCompactions()` and `ResumeAllCompactions()` to the `DB` class. Added new `Status::SubCode::kCompactionAborted` to indicate a compaction was aborted. Added `Status::IsCompactionAborted()` helper method to check if a status represents an aborted compaction.

unreleased_history/public_api_changes/remove_deprecated_apis_batch1.md

@@ -1 +0,0 @@
-Remove deprecated, unused APIs and options: `ReadOptions::managed` and `ColumnFamilyOptions::snap_refresh_nanos`. Corresponding C and Java APIs are also removed.

unreleased_history/public_api_changes/remove_fv_1.md

@@ -1 +0,0 @@
-* Drop support for reading (and writing) SST files using `BlockBasedTableOptions.format_version` < 2, which hasn't been the default format for about 10 years. An upgrade path is still possible with full compaction using a RocksDB version >= 4.6.0 and < 11.0.0 and then using the newer version.

unreleased_history/public_api_changes/remove_raw_ptr_db_open.md

@@ -1,2 +0,0 @@
-* Remove deprecated raw `DB*` variants of `DB::Open` and related functions. Some other minor public APIs were updated as a result
-* Remove deprecated `DB::MaxMemCompactionLevel()`

unreleased_history/public_api_changes/remove_secondary_compress_format_version.md

@@ -1 +0,0 @@
-* Remove useless option `CompressedSecondaryCacheOptions::compress_format_version`

unreleased_history/public_api_changes/remove_skip_checking_sst_file_sizes_on_db_open.md

@@ -1 +0,0 @@
-Remove deprecated DB option `skip_checking_sst_file_sizes_on_db_open`. The option was deprecated in 10.5.0 and has been a no-op since then. File size validation is now always performed in parallel during DB open.

unreleased_history/public_api_changes/remove_slice_transform_inrange.md

@@ -1 +0,0 @@
-Remove deprecated `SliceTransform::InRange()` virtual method and the `in_range` callback parameter from `rocksdb_slicetransform_create()` in the C API. `InRange()` was never called by RocksDB and existed only for backward compatibility.

unreleased_history/public_api_changes/remove_sst_file_writer_deprecated.md

@@ -1 +0,0 @@
-Remove deprecated `SstFileWriter::Add()` method (use `Put()` instead) and the deprecated `skip_filters` parameter from `SstFileWriter` constructors (use `BlockBasedTableOptions::filter_policy` set to `nullptr` to skip filter generation instead).

util/stop_watch.h

@@ -112,6 +112,7 @@ class StopWatchNano {
     }
   }
   void Start() {
+    assert(clock_);
     if constexpr (use_cpu_time) {
       start_ = clock_->CPUNanos();
     } else {
@@ -119,6 +120,7 @@ class StopWatchNano {
     }
   }
   uint64_t ElapsedNanos(bool reset = false) {
+    assert(clock_);
     uint64_t now = 0;
     if constexpr (use_cpu_time) {
       now = clock_->CPUNanos();

utilities/blob_db/blob_dump_tool.cc

@@ -14,8 +14,10 @@
 #include "file/random_access_file_reader.h"
 #include "file/readahead_raf.h"
 #include "port/port.h"
+#include "rocksdb/advanced_compression.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/file_system.h"
+#include "table/format.h"
 #include "util/coding.h"
 #include "util/string_util.h"
 
@@ -25,7 +27,9 @@ BlobDumpTool::BlobDumpTool()
     : reader_(nullptr), buffer_(nullptr), buffer_size_(0) {}
 
 Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
-                         DisplayType show_blob, bool show_summary) {
+                         DisplayType show_blob,
+                         DisplayType show_uncompressed_blob,
+                         bool show_summary) {
   constexpr size_t kReadaheadSize = 2 * 1024 * 1024;
   Status s;
   const auto fs = FileSystem::Default();
@@ -51,7 +55,8 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
   reader_.reset(new RandomAccessFileReader(std::move(file), filename));
   uint64_t offset = 0;
   uint64_t footer_offset = 0;
-  s = DumpBlobLogHeader(&offset);
+  CompressionType compression = kNoCompression;
+  s = DumpBlobLogHeader(&offset, &compression);
   if (!s.ok()) {
     return s;
   }
@@ -62,10 +67,12 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
   uint64_t total_records = 0;
   uint64_t total_key_size = 0;
   uint64_t total_blob_size = 0;
-  if (show_key != DisplayType::kNone) {
+  uint64_t total_uncompressed_blob_size = 0;
+  if (show_key != DisplayType::kNone || show_summary) {
     while (offset < footer_offset) {
-      s = DumpRecord(show_key, show_blob, &offset, &total_records,
-                     &total_key_size, &total_blob_size);
+      s = DumpRecord(show_key, show_blob, show_uncompressed_blob, show_summary,
+                     compression, &offset, &total_records, &total_key_size,
+                     &total_blob_size, &total_uncompressed_blob_size);
       if (!s.ok()) {
         break;
       }
@@ -76,6 +83,10 @@ Status BlobDumpTool::Run(const std::string& filename, DisplayType show_key,
     fprintf(stdout, "  total records: %" PRIu64 "\n", total_records);
     fprintf(stdout, "  total key size: %" PRIu64 "\n", total_key_size);
     fprintf(stdout, "  total blob size: %" PRIu64 "\n", total_blob_size);
+    if (compression != kNoCompression) {
+      fprintf(stdout, "  total raw blob size: %" PRIu64 "\n",
+              total_uncompressed_blob_size);
+    }
   }
   return s;
 }
@@ -101,7 +112,8 @@ Status BlobDumpTool::Read(uint64_t offset, size_t size, Slice* result) {
   return s;
 }
 
-Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset) {
+Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset,
+                                       CompressionType* compression) {
   Slice slice;
   Status s = Read(0, BlobLogHeader::kSize, &slice);
   if (!s.ok()) {
@@ -116,10 +128,17 @@ Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset) {
   fprintf(stdout, "  Version          : %" PRIu32 "\n", header.version);
   fprintf(stdout, "  Column Family ID : %" PRIu32 "\n",
           header.column_family_id);
-  fprintf(stdout, "  Compression      : kNoCompression\n");
+  std::string compression_str;
+  if (!GetStringFromCompressionType(&compression_str, header.compression)
+           .ok()) {
+    compression_str = "Unrecongnized compression type (" +
+                      std::to_string((int)header.compression) + ")";
+  }
+  fprintf(stdout, "  Compression      : %s\n", compression_str.c_str());
   fprintf(stdout, "  Expiration range : %s\n",
           GetString(header.expiration_range).c_str());
   *offset = BlobLogHeader::kSize;
+  *compression = header.compression;
   return s;
 }
 
@@ -152,9 +171,12 @@ Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size,
 }
 
 Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
+                                DisplayType show_uncompressed_blob,
+                                bool show_summary, CompressionType compression,
                                 uint64_t* offset, uint64_t* total_records,
                                 uint64_t* total_key_size,
-                                uint64_t* total_blob_size) {
+                                uint64_t* total_blob_size,
+                                uint64_t* total_uncompressed_blob_size) {
   if (show_key != DisplayType::kNone) {
     fprintf(stdout, "Read record with offset 0x%" PRIx64 " (%" PRIu64 "):\n",
             *offset, *offset);
@@ -181,6 +203,31 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
   if (!s.ok()) {
     return s;
   }
+  // Decompress value
+  std::string uncompressed_value;
+  if (compression != kNoCompression &&
+      (show_uncompressed_blob != DisplayType::kNone || show_summary)) {
+    auto decompressor =
+        GetBuiltinV2CompressionManager()->GetDecompressorOptimizeFor(
+            compression);
+    if (!decompressor) {
+      return Status::NotSupported("Unsupported compression type");
+    }
+    Slice compressed_data(slice.data() + key_size,
+                          static_cast<size_t>(value_size));
+    Decompressor::Args args;
+    args.compression_type = compression;
+    args.compressed_data = compressed_data;
+    s = decompressor->ExtractUncompressedSize(args);
+    if (!s.ok()) {
+      return s;
+    }
+    uncompressed_value.resize(args.uncompressed_size);
+    s = decompressor->DecompressBlock(args, uncompressed_value.data());
+    if (!s.ok()) {
+      return s;
+    }
+  }
   if (show_key != DisplayType::kNone) {
     fprintf(stdout, "  key        : ");
     DumpSlice(Slice(slice.data(), static_cast<size_t>(key_size)), show_key);
@@ -190,11 +237,16 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
                       static_cast<size_t>(value_size)),
                 show_blob);
     }
+    if (show_uncompressed_blob != DisplayType::kNone) {
+      fprintf(stdout, "  raw blob   : ");
+      DumpSlice(Slice(uncompressed_value), show_uncompressed_blob);
+    }
   }
   *offset += key_size + value_size;
   *total_records += 1;
   *total_key_size += key_size;
   *total_blob_size += value_size;
+  *total_uncompressed_blob_size += uncompressed_value.size();
   return s;
 }
 

utilities/blob_db/blob_dump_tool.h

@@ -28,7 +28,8 @@ class BlobDumpTool {
   BlobDumpTool();
 
   Status Run(const std::string& filename, DisplayType show_key,
-             DisplayType show_blob, bool show_summary);
+             DisplayType show_blob, DisplayType show_uncompressed_blob,
+             bool show_summary);
 
  private:
   std::unique_ptr<RandomAccessFileReader> reader_;
@@ -36,11 +37,14 @@ class BlobDumpTool {
   size_t buffer_size_;
 
   Status Read(uint64_t offset, size_t size, Slice* result);
-  Status DumpBlobLogHeader(uint64_t* offset);
+  Status DumpBlobLogHeader(uint64_t* offset, CompressionType* compression);
   Status DumpBlobLogFooter(uint64_t file_size, uint64_t* footer_offset);
   Status DumpRecord(DisplayType show_key, DisplayType show_blob,
-                    uint64_t* offset, uint64_t* total_records,
-                    uint64_t* total_key_size, uint64_t* total_blob_size);
+                    DisplayType show_uncompressed_blob, bool show_summary,
+                    CompressionType compression, uint64_t* offset,
+                    uint64_t* total_records, uint64_t* total_key_size,
+                    uint64_t* total_blob_size,
+                    uint64_t* total_uncompressed_blob_size);
   void DumpSlice(const Slice s, DisplayType type);
 
   template <class T>