Remove yield in point lock manager

The yield is actually of not much use because waitFor should already be doing that internally.

HISTORY.md

@@ -1,26 +1,9 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
 
-## 10.5.4 (08/21/2025)
-### Bug Fixes
-* Fix a crash in iterator Prepare() when fill_cache=false
-
-## 10.5.3 (08/14/2025)
-### Bug Fixes
-* Allow a user defined index to be configured from a string.
-
-## 10.5.2 (08/12/2025)
-### Bug Fixes
-* Make the User Defined Index interface consistently use the user key format, fixing the previous mixed usage of internal and user key.
-
-## 10.5.1 (08/04/2025)
-### Bug Fixes
-* Fixed a bug in remote compaction that may mistakenly delete live SST file(s) during the cleanup phase when no keys survive the compaction (all expired)
-
-## 10.5.0 (07/21/2025)
+## 10.5.0 (07/18/2025)
 ### Public API Changes
 * DB option skip_checking_sst_file_sizes_on_db_open is deprecated, in favor of validating file size in parallel in a thread pool, when db is opened. When DB is opened, with paranoid check enabled, a file with the wrong size would fail the DB open. With paranoid check disabled, the DB open would succeed, the column family with the corrupted file would not be read or write, while the other healthy column families could be read and write normally. When max_open_files option is not set to -1, only a subset of the files will be opened and checked. The rest of the files will be opened and checked when they are accessed.
-* GetTtl() API is now available in TTL DB
 
 ### Behavior Changes
 * PessimisticTransaction::GetWaitingTxns now returns waiting transaction information even if the current transaction has timed out. This allows the information to be surfaced to users for debugging purposes once it is known that the timeout has occured.

Makefile

@@ -2492,7 +2492,7 @@ checkout_folly:
 	fi
 	@# Pin to a particular version for public CI, so that PR authors don't
 	@# need to worry about folly breaking our integration. Update periodically
-	cd third-party/folly && git reset --hard 5c626dd6a028a02e461edb5396694d48305e9284
+	cd third-party/folly && git reset --hard e95383b7c8b5b1e46cf47acf2f317d54f93c8268
 	@# Apparently missing include
 	perl -pi -e 's/(#include <atomic>)/$$1\n#include <cstring>/' third-party/folly/folly/lang/Exception.h
 	@# Warning-as-error on memcpy

db/compaction/compaction_job.cc

@@ -1736,11 +1736,14 @@ Status CompactionJob::FinishCompactionOutputFile(
   if (s.ok()) {
     tp = outputs.GetTableProperties();
   }
+
   if (s.ok() && current_entries == 0 && tp.num_range_deletions == 0) {
     // If there is nothing to output, no necessary to generate a sst file.
     // This happens when the output level is bottom level, at the same time
     // the sub_compact output nothing.
-    std::string fname = GetTableFileName(meta->fd.GetNumber());
+    std::string fname =
+        TableFileName(sub_compact->compaction->immutable_options().cf_paths,
+                      meta->fd.GetNumber(), meta->fd.GetPathId());
 
     // TODO(AR) it is not clear if there are any larger implications if
     // DeleteFile fails here
@@ -1939,10 +1942,6 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
 
   // no need to lock because VersionSet::next_file_number_ is atomic
   uint64_t file_number = versions_->NewFileNumber();
-#ifndef NDEBUG
-  TEST_SYNC_POINT_CALLBACK(
-      "CompactionJob::OpenCompactionOutputFile::NewFileNumber", &file_number);
-#endif
   std::string fname = GetTableFileName(file_number);
   // Fire events.
   ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();

db/compaction/compaction_picker.cc

@@ -337,9 +337,7 @@ Compaction* CompactionPicker::CompactFiles(
     const CompactionOptions& compact_options,
     const std::vector<CompactionInputFiles>& input_files, int output_level,
     VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options,
-    const MutableDBOptions& mutable_db_options, uint32_t output_path_id,
-    std::optional<SequenceNumber> earliest_snapshot,
-    const SnapshotChecker* snapshot_checker) {
+    const MutableDBOptions& mutable_db_options, uint32_t output_path_id) {
 #ifndef NDEBUG
   assert(input_files.size());
   // This compaction output should not overlap with a running compaction as
@@ -382,8 +380,8 @@ Compaction* CompactionPicker::CompactFiles(
       GetCompressionOptions(mutable_cf_options, vstorage, output_level),
       mutable_cf_options.default_write_temperature,
       compact_options.max_subcompactions,
-      /* grandparents */ {}, earliest_snapshot, snapshot_checker,
-      CompactionReason::kManualCompaction);
+      /* grandparents */ {}, /* earliest_snapshot */ std::nullopt,
+      /* snapshot_checker */ nullptr, CompactionReason::kManualCompaction);
   RegisterCompaction(c);
   return c;
 }

db/compaction/compaction_picker.h

@@ -117,17 +117,12 @@ class CompactionPicker {
   // Caller must provide a set of input files that has been passed through
   // `SanitizeAndConvertCompactionInputFiles` earlier. The lock should not be
   // released between that call and this one.
-  //
-  //  TODO - Remove default values for earliest_snapshot and snapshot_checker
-  //  and require all callers to pass them in so that DB::CompactFiles() can
-  //  also benefit from Standalone Range Tombstone Optimization
-  Compaction* CompactFiles(
-      const CompactionOptions& compact_options,
-      const std::vector<CompactionInputFiles>& input_files, int output_level,
-      VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options,
-      const MutableDBOptions& mutable_db_options, uint32_t output_path_id,
-      std::optional<SequenceNumber> earliest_snapshot = std::nullopt,
-      const SnapshotChecker* snapshot_checker = nullptr);
+  Compaction* CompactFiles(const CompactionOptions& compact_options,
+                           const std::vector<CompactionInputFiles>& input_files,
+                           int output_level, VersionStorageInfo* vstorage,
+                           const MutableCFOptions& mutable_cf_options,
+                           const MutableDBOptions& mutable_db_options,
+                           uint32_t output_path_id);
 
   // Converts a set of compaction input file numbers into
   // a list of CompactionInputFiles.

db/compaction/compaction_service_test.cc

@@ -461,97 +461,6 @@ TEST_F(CompactionServiceTest, ManualCompaction) {
   ASSERT_EQ(handles_[1]->GetName(), info.cf_name);
 }
 
-TEST_F(CompactionServiceTest, StandaloneDeleteRangeTombstoneOptimization) {
-  Options options = CurrentOptions();
-  options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
-  ReopenWithCompactionService(&options);
-
-  size_t num_files_after_filtered = 0;
-  SyncPoint::GetInstance()->SetCallBack(
-      "VersionSet::MakeInputIterator:NewCompactionMergingIterator",
-      [&](void* arg) {
-        num_files_after_filtered = *static_cast<size_t*>(arg);
-      });
-
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  std::vector<std::string> files;
-  {
-    // Writes first version of data in range partitioned files.
-    SstFileWriter sst_file_writer(EnvOptions(), options);
-    std::string file1 = dbname_ + "file1.sst";
-    ASSERT_OK(sst_file_writer.Open(file1));
-    ASSERT_OK(sst_file_writer.Put("a", "a1"));
-    ASSERT_OK(sst_file_writer.Put("b", "b1"));
-    ExternalSstFileInfo file1_info;
-    ASSERT_OK(sst_file_writer.Finish(&file1_info));
-    files.push_back(std::move(file1));
-
-    std::string file2 = dbname_ + "file2.sst";
-    ASSERT_OK(sst_file_writer.Open(file2));
-    ASSERT_OK(sst_file_writer.Put("x", "x1"));
-    ASSERT_OK(sst_file_writer.Put("y", "y1"));
-    ExternalSstFileInfo file2_info;
-    ASSERT_OK(sst_file_writer.Finish(&file2_info));
-    files.push_back(std::move(file2));
-  }
-
-  IngestExternalFileOptions ifo;
-  ASSERT_OK(db_->IngestExternalFile(files, ifo));
-  ASSERT_EQ(Get("a"), "a1");
-  ASSERT_EQ(Get("b"), "b1");
-  ASSERT_EQ(Get("x"), "x1");
-  ASSERT_EQ(Get("y"), "y1");
-  ASSERT_EQ(2, NumTableFilesAtLevel(6));
-
-  auto my_cs = GetCompactionService();
-  uint64_t comp_num = my_cs->GetCompactionNum();
-
-  {
-    // Atomically delete old version of data with one range delete file.
-    // And a new batch of range partitioned files with new version of data.
-    files.clear();
-    SstFileWriter sst_file_writer(EnvOptions(), options);
-    std::string file2 = dbname_ + "file2.sst";
-    ASSERT_OK(sst_file_writer.Open(file2));
-    ASSERT_OK(sst_file_writer.DeleteRange("a", "z"));
-    ExternalSstFileInfo file2_info;
-    ASSERT_OK(sst_file_writer.Finish(&file2_info));
-    files.push_back(std::move(file2));
-
-    std::string file3 = dbname_ + "file3.sst";
-    ASSERT_OK(sst_file_writer.Open(file3));
-    ASSERT_OK(sst_file_writer.Put("a", "a2"));
-    ASSERT_OK(sst_file_writer.Put("b", "b2"));
-    ExternalSstFileInfo file3_info;
-    ASSERT_OK(sst_file_writer.Finish(&file3_info));
-    files.push_back(std::move(file3));
-
-    std::string file4 = dbname_ + "file4.sst";
-    ASSERT_OK(sst_file_writer.Open(file4));
-    ASSERT_OK(sst_file_writer.Put("x", "x2"));
-    ASSERT_OK(sst_file_writer.Put("y", "y2"));
-    ExternalSstFileInfo file4_info;
-    ASSERT_OK(sst_file_writer.Finish(&file4_info));
-    files.push_back(std::move(file4));
-  }
-
-  ASSERT_OK(db_->IngestExternalFile(files, ifo));
-  ASSERT_OK(db_->WaitForCompact(WaitForCompactOptions()));
-  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
-
-  CompactionServiceResult result;
-  my_cs->GetResult(&result);
-  ASSERT_OK(result.status);
-  ASSERT_TRUE(result.stats.is_manual_compaction);
-  ASSERT_TRUE(result.stats.is_remote_compaction);
-
-  ASSERT_EQ(num_files_after_filtered, 1);
-
-  Close();
-  SyncPoint::GetInstance()->DisableProcessing();
-}
-
 TEST_F(CompactionServiceTest, CompactionOutputFileIOError) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
@@ -884,79 +793,6 @@ TEST_F(CompactionServiceTest, VerifyInputRecordCount) {
   SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
-TEST_F(CompactionServiceTest, EmptyResult) {
-  Options options = CurrentOptions();
-  options.disable_auto_compactions = true;
-  ReopenWithCompactionService(&options);
-  GenerateTestData();
-
-  auto my_cs = GetCompactionService();
-
-  uint64_t comp_num = my_cs->GetCompactionNum();
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
-
-  // Delete range to cover entire range
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), "key", "keyz"));
-  ASSERT_OK(Flush());
-
-  // In this unit test, both remote compaction and primary db instance are
-  // running in the same process, so NewFileNumber will never have a collision.
-  // In the real-world remote compactions, when the compaction is indeed running
-  // in another process, this is not going to be the case.
-  // To simulate the SST file with the same name created in the tmp directory,
-  // override the file number in remote compaction to re-use old SST file
-  // number.
-  bool need_to_override_file_number = false;
-  SyncPoint::GetInstance()->SetCallBack(
-      "DBImplSecondary::OpenAndCompact::BeforeLoadingOptions:0",
-      [&](void*) { need_to_override_file_number = true; });
-
-  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionJob::OpenCompactionOutputFile::NewFileNumber",
-      [&](void* file_number) {
-        if (need_to_override_file_number) {
-          auto n = static_cast<uint64_t*>(file_number);
-          ColumnFamilyMetaData cf_meta;
-          db_->GetColumnFamilyMetaData(&cf_meta);
-          for (const auto& level : cf_meta.levels) {
-            for (const auto& file : level.files) {
-              // Use one of the existing file name
-              *n = test::GetFileNumber(file.name);
-              need_to_override_file_number = false;
-              return;
-            }
-          }
-        }
-      });
-
-  // Inject failure, so that the remote compaction fails after
-  // ProcessKeyValueCompaction()
-  SyncPoint::GetInstance()->SetCallBack(
-      "DBImplSecondary::CompactWithoutInstallation::End", [&](void* status) {
-        // override job status
-        auto s = static_cast<Status*>(status);
-        *s = Status::Aborted("MyTestCompactionService failed to compact!");
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  // Compaction should fail and SST files in the primary db should exist
-  {
-    ASSERT_NOK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-    ColumnFamilyMetaData meta;
-    db_->GetColumnFamilyMetaData(&meta);
-    for (const auto& level : meta.levels) {
-      for (const auto& file : level.files) {
-        std::string fname = file.db_path + "/" + file.name;
-        ASSERT_OK(db_->GetEnv()->FileExists(fname));
-      }
-    }
-  }
-  Close();
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
 TEST_F(CompactionServiceTest, CorruptedOutput) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;

db/db_impl/db_impl_secondary.cc

@@ -848,6 +848,7 @@ Status DBImplSecondary::CompactWithoutInstallation(
 
   VersionStorageInfo* vstorage = version->storage_info();
 
+  // Use comp_options to reuse some CompactFiles functions
   CompactionOptions comp_options;
   comp_options.compression = kDisableCompressionOption;
   comp_options.output_file_size_limit = MaxFileSizeForLevel(
@@ -866,27 +867,13 @@ Status DBImplSecondary::CompactWithoutInstallation(
     return s;
   }
 
-  const int job_id = next_job_id_.fetch_add(1);
-  JobContext job_context(job_id, true /*create_superversion*/);
-  std::vector<SequenceNumber> snapshots = input.snapshots;
-
-  // TODO - snapshot_checker support in Remote Compaction
-  job_context.InitSnapshotContext(/*checker=*/nullptr,
-                                  /*managed_snapshot=*/nullptr,
-                                  kMaxSequenceNumber, std::move(snapshots));
-
-  // TODO - consider serializing the entire Compaction object and using it as
-  // input instead of recreating it in the remote worker
   std::unique_ptr<Compaction> c;
   assert(cfd->compaction_picker());
   c.reset(cfd->compaction_picker()->CompactFiles(
       comp_options, input_files, input.output_level, vstorage,
-      cfd->GetLatestMutableCFOptions(), mutable_db_options_, 0,
-      /*earliest_snapshot=*/job_context.snapshot_seqs.empty()
-          ? kMaxSequenceNumber
-          : job_context.snapshot_seqs.front(),
-      job_context.snapshot_checker));
+      cfd->GetLatestMutableCFOptions(), mutable_db_options_, 0));
   assert(c != nullptr);
+
   c->FinalizeInputInfo(version);
 
   // Create output directory if it's not existed yet
@@ -899,6 +886,11 @@ Status DBImplSecondary::CompactWithoutInstallation(
   LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL,
                        immutable_db_options_.info_log.get());
 
+  const int job_id = next_job_id_.fetch_add(1);
+  JobContext job_context(0, true /*create_superversion*/);
+  std::vector<SequenceNumber> snapshots = input.snapshots;
+  job_context.InitSnapshotContext(nullptr, nullptr, kMaxSequenceNumber,
+                                  std::move(snapshots));
   // use primary host's db_id for running the compaction, but db_session_id is
   // using the local one, which is to make sure the unique id is unique from
   // the remote compactors. Because the id is generated from db_id,

db/db_iterator_test.cc

@@ -4142,17 +4142,13 @@ TEST_P(DBIteratorTest, AverageMemtableOpsScanFlushTriggerByOverwrites) {
   ASSERT_EQ(1, NumTableFilesAtLevel(0));
 }
 
-class DBMultiScanIteratorTest : public DBTestBase,
-                                public ::testing::WithParamInterface<bool> {
+class DBMultiScanIteratorTest : public DBTestBase {
  public:
   DBMultiScanIteratorTest()
       : DBTestBase("db_multi_scan_iterator_test", /*env_do_fsync=*/true) {}
 };
 
-INSTANTIATE_TEST_CASE_P(DBMultiScanIteratorTest, DBMultiScanIteratorTest,
-                        ::testing::Bool());
-
-TEST_P(DBMultiScanIteratorTest, BasicTest) {
+TEST_F(DBMultiScanIteratorTest, BasicTest) {
   // Create a file
   for (int i = 0; i < 100; ++i) {
     std::stringstream ss;
@@ -4163,7 +4159,6 @@ TEST_P(DBMultiScanIteratorTest, BasicTest) {
 
   std::vector<std::string> key_ranges({"k03", "k10", "k25", "k50"});
   ReadOptions ro;
-  ro.fill_cache = GetParam();
   std::vector<ScanOptions> scan_options(
       {ScanOptions(key_ranges[0], key_ranges[1]),
        ScanOptions(key_ranges[2], key_ranges[3])});
@@ -4250,7 +4245,7 @@ TEST_P(DBMultiScanIteratorTest, BasicTest) {
   iter.reset();
 }
 
-TEST_P(DBMultiScanIteratorTest, MixedBoundsTest) {
+TEST_F(DBMultiScanIteratorTest, MixedBoundsTest) {
   // Create a file
   for (int i = 0; i < 100; ++i) {
     std::stringstream ss;
@@ -4262,7 +4257,6 @@ TEST_P(DBMultiScanIteratorTest, MixedBoundsTest) {
   std::vector<std::string> key_ranges(
       {"k03", "k10", "k25", "k50", "k75", "k90"});
   ReadOptions ro;
-  ro.fill_cache = GetParam();
   std::vector<ScanOptions> scan_options(
       {ScanOptions(key_ranges[0], key_ranges[1]), ScanOptions(key_ranges[2]),
        ScanOptions(key_ranges[4], key_ranges[5])});

db/version_set.cc

@@ -95,6 +95,8 @@ namespace ROCKSDB_NAMESPACE {
 
 namespace {
 
+using ScanOptionsMap = std::unordered_map<size_t, std::vector<ScanOptions>>;
+
 // Find File in LevelFilesBrief data structure
 // Within an index range defined by left and right
 int FindFileInRange(const InternalKeyComparator& icmp,
@@ -1100,9 +1102,47 @@ class LevelIterator final : public InternalIterator {
   }
 
   void Prepare(const std::vector<ScanOptions>* scan_opts) override {
+    // We assume here that scan_opts is sorted such that
+    // scan_opts[0].range.start < scan_opts[1].range.start, and non overlapping
     scan_opts_ = scan_opts;
-    if (file_iter_.iter()) {
-      file_iter_.Prepare(scan_opts_);
+    if (scan_opts_ == nullptr) {
+      return;
+    }
+
+    file_to_scan_opts_ = std::make_unique<ScanOptionsMap>();
+    for (size_t k = 0; k < scan_opts_->size(); k++) {
+      const ScanOptions& opt = scan_opts_->at(k);
+      auto start = opt.range.start;
+      auto end = opt.range.limit;
+
+      if (!start.has_value()) {
+        continue;
+      }
+
+      // We can capture this case in the future, but for now lets skip this.
+      if (!end.has_value()) {
+        continue;
+      }
+
+      InternalKey istart(start.value(), kMaxSequenceNumber, kValueTypeForSeek);
+      InternalKey iend(end.value(), 0, kValueTypeForSeekForPrev);
+
+      // TODO: This needs to be optimized, right now we iterate twice, which
+      // we dont need to. We can do this in N rather than 2N.
+      size_t fstart = FindFile(icomparator_, *flevel_, istart.Encode());
+      size_t fend = FindFile(icomparator_, *flevel_, iend.Encode());
+
+      // We need to check the relevant cases
+      // Cases:
+      // 1. [  S        E  ]
+      // 2. [  S  ]  [  E  ]
+      // 3. [  S  ] ...... [  E  ]
+      for (auto i = fstart; i <= fend; i++) {
+        if (i < flevel_->num_files) {
+          (*file_to_scan_opts_)[i].emplace_back(start.value(), end.value());
+          (*file_to_scan_opts_)[i].back().property_bag = opt.property_bag;
+        }
+      }
     }
   }
 
@@ -1233,6 +1273,9 @@ class LevelIterator final : public InternalIterator {
   bool to_return_sentinel_ = false;
   const std::vector<ScanOptions>* scan_opts_;
 
+  // Our stored scan_opts for each prefix
+  std::unique_ptr<ScanOptionsMap> file_to_scan_opts_ = nullptr;
+
   // Sets flags for if we should return the sentinel key next.
   // The condition for returning sentinel is reaching the end of current
   // file_iter_: !Valid() && status.().ok().
@@ -1494,7 +1537,18 @@ bool LevelIterator::SkipEmptyFileForward() {
     // LevelIterator::Seek*, it should also call Seek* into the corresponding
     // range tombstone iterator.
     if (file_iter_.iter() != nullptr) {
-      file_iter_.SeekToFirst();
+      // If we are doing prepared scan opts then we should seek to the values
+      // specified by the scan opts
+      if (scan_opts_ && (*file_to_scan_opts_)[file_index_].size()) {
+        const ScanOptions& opts = file_to_scan_opts_->at(file_index_).front();
+        if (opts.range.start.has_value()) {
+          InternalKey target(*opts.range.start.AsPtr(), kMaxSequenceNumber,
+                             kValueTypeForSeek);
+          file_iter_.Seek(target.Encode());
+        }
+      } else {
+        file_iter_.SeekToFirst();
+      }
       if (range_tombstone_iter_) {
         if (*range_tombstone_iter_) {
           (*range_tombstone_iter_)->SeekToFirst();
@@ -1542,10 +1596,15 @@ void LevelIterator::SetFileIterator(InternalIterator* iter) {
   }
 
   InternalIterator* old_iter = file_iter_.Set(iter);
-  // Since this is a new table iterator, no need to call Prepare() if
-  // scan_opts_ is null
   if (iter && scan_opts_) {
-    file_iter_.Prepare(scan_opts_);
+    if (file_to_scan_opts_.get() &&
+        file_to_scan_opts_->find(file_index_) != file_to_scan_opts_->end()) {
+      const std::vector<ScanOptions>& opts =
+          file_to_scan_opts_->at(file_index_);
+      file_iter_.Prepare(&opts);
+    } else {
+      file_iter_.Prepare(scan_opts_);
+    }
   }
 
   // Update the read pattern for PrefetchBuffer.
@@ -1582,6 +1641,7 @@ void LevelIterator::InitFileIterator(size_t new_file_index) {
     }
   }
 }
+
 }  // anonymous namespace
 
 Status Version::GetTableProperties(const ReadOptions& read_options,

include/rocksdb/user_defined_index.h

@@ -27,11 +27,6 @@ inline const std::string kUserDefinedIndexPrefix =
 // It allows users to define their own index format and build custom
 // indexes during table building. Currently, only a monolithic index
 // block is supported (no partitioned index).
-//
-// This is currently supported only for a restricted set of use cases. The
-// CF must be ingest only, and only files containing Puts generated by
-// SstFileWriter are supported. The user_comparator used for the CF must
-// be BytewiseComparator.
 
 // The interface for building user-defined index.
 class UserDefinedIndexBuilder {
@@ -56,10 +51,6 @@ class UserDefinedIndexBuilder {
   // The previous index entry key and the new index entry key cover
   // all the keys in the data block associated with the new index entry.
   //
-  // The last_key_in_current_block and first_key_in_next_block will be user
-  // keys, i.e the user key string, and optionally the user timestamp if one
-  // is configured, without a sequence number suffix.
-  //
   // Called before the OnKeyAdded() call for first_key_in_next_block.
   // @last_key_in_current_block: The last key in the current data block
   // @first_key_in_next_block: it will be nullptr if the entry being added is
@@ -81,9 +72,6 @@ class UserDefinedIndexBuilder {
   // override OnKeyAdded() if they need to collect additional information.
   // The type argument indicates whether the value is a full value or partial.
   // At the moment, only full values are supported.
-  //
-  // The key will be a user key. RocksDB guarantees that there will only be
-  // one entry for each key in the file/index.
   virtual void OnKeyAdded(const Slice& /*key*/, ValueType /*type*/,
                           const Slice& /*value*/) {}
 
@@ -112,14 +100,6 @@ class UserDefinedIndexIterator {
   // termination criteria, kInbound if the data block is definitely fully
   // within bounds, or kUnknown if the data block could be partially
   // within bounds.
-  // The UDI implementation needs to be careful about returning kOutOfBound.
-  // If a limit key is specified in ScanOptions, an implementation that
-  // does not store the first key in the block for the corresponding index
-  // entry cannot reliably determine if the block is out of bounds. It must
-  // compare against the previous index key to determine if the current block
-  // is out of bounds w.r.t the limit. Other termination criteria (specified
-  // in property_bag) may cause the scan to terminate earlier, in which case
-  // kOutOfBound can be returned earlier.
   virtual Status SeekAndGetResult(const Slice& target,
                                   IterateResult* result) = 0;
 
@@ -150,12 +130,6 @@ class UserDefinedIndexFactory : public Customizable {
  public:
   virtual ~UserDefinedIndexFactory() = default;
 
-  static const char* Type() { return "UserDefinedIndexFactory"; }
-
-  static Status CreateFromString(
-      const ConfigOptions& config_options, const std::string& value,
-      std::shared_ptr<UserDefinedIndexFactory>* factory);
-
   // Create a new builder for user-defined index.
   virtual UserDefinedIndexBuilder* NewBuilder() const = 0;
 

include/rocksdb/version.h

@@ -12,8 +12,8 @@
 // NOTE: in 'main' development branch, this should be the *next*
 // minor or major version number planned for release.
 #define ROCKSDB_MAJOR 10
-#define ROCKSDB_MINOR 5
-#define ROCKSDB_PATCH 5
+#define ROCKSDB_MINOR 6
+#define ROCKSDB_PATCH 0
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these

table/block_based/block_based_table_builder.cc

@@ -907,12 +907,6 @@ struct BlockBasedTableBuilder::Rep {
         SetStatus(
             Status::InvalidArgument("user_defined_index_factory not supported "
                                     "with parallel compression"));
-      } else if (ioptions.user_comparator != BytewiseComparator()) {
-        // TODO: Pass the user_comparator to the UDI and let it validate. Do
-        // it in a major release.
-        SetStatus(
-            Status::InvalidArgument("user_defined_index_factory only supported "
-                                    "with bytewise comparator"));
       } else {
         std::unique_ptr<UserDefinedIndexBuilder> user_defined_index_builder(
             table_options.user_defined_index_factory->NewBuilder());
@@ -1064,7 +1058,7 @@ struct BlockBasedTableBuilder::Rep {
       char* ptr = compression_name.data() + pos;
       // Populate the field contents
       for (CompressionType t : compression_types_used) {
-        PutBaseChars<16>(&ptr, /*n=*/2, static_cast<unsigned char>(t),
+        PutBaseChars<16>(&ptr, /*digits=*/2, static_cast<unsigned char>(t),
                          /*uppercase=*/true);
       }
       assert(ptr == compression_name.data() + pos + ctype_count * 2);
@@ -1833,15 +1827,14 @@ void BlockBasedTableBuilder::WriteIndexBlock(
   }
   IndexBuilder::IndexBlocks index_blocks;
   auto index_builder_status = rep_->index_builder->Finish(&index_blocks);
-  if (ok() && !index_builder_status.ok() &&
-      !index_builder_status.IsIncomplete()) {
-    // If the index builder failed for non-Incomplete errors, we should
-    // mark the entire builder as having failed wit that status. However,
-    // If the index builder failed with an incomplete error, we should
-    // continue writing out any meta blocks that may have been generated.
+  if (index_builder_status.IsIncomplete()) {
+    // We we have more than one index partition then meta_blocks are not
+    // supported for the index. Currently meta_blocks are used only by
+    // HashIndexBuilder which is not multi-partition.
+    assert(index_blocks.meta_blocks.empty());
+  } else if (ok() && !index_builder_status.ok()) {
     rep_->SetStatus(index_builder_status);
   }
-
   if (ok()) {
     for (const auto& item : index_blocks.meta_blocks) {
       BlockHandle block_handle;

table/block_based/block_based_table_factory.cc

@@ -25,8 +25,6 @@
 #include "rocksdb/flush_block_policy.h"
 #include "rocksdb/rocksdb_namespace.h"
 #include "rocksdb/table.h"
-#include "rocksdb/user_defined_index.h"
-#include "rocksdb/utilities/customizable_util.h"
 #include "rocksdb/utilities/options_type.h"
 #include "table/block_based/block_based_table_builder.h"
 #include "table/block_based/block_based_table_reader.h"
@@ -314,11 +312,6 @@ static struct BlockBasedTableTypeInfo {
          OptionTypeInfo::AsCustomSharedPtr<const FilterPolicy>(
              offsetof(struct BlockBasedTableOptions, filter_policy),
              OptionVerificationType::kByNameAllowFromNull)},
-        {"user_defined_index_factory",
-         OptionTypeInfo::AsCustomSharedPtr<UserDefinedIndexFactory>(
-             offsetof(struct BlockBasedTableOptions,
-                      user_defined_index_factory),
-             OptionVerificationType::kByNameAllowFromNull)},
         {"whole_key_filtering",
          {offsetof(struct BlockBasedTableOptions, whole_key_filtering),
           OptionType::kBoolean, OptionVerificationType::kNormal}},
@@ -1018,13 +1011,6 @@ TableFactory* NewBlockBasedTableFactory(
   return new BlockBasedTableFactory(_table_options);
 }
 
-Status UserDefinedIndexFactory::CreateFromString(
-    const ConfigOptions& config_options, const std::string& value,
-    std::shared_ptr<UserDefinedIndexFactory>* factory) {
-  return LoadSharedObject<UserDefinedIndexFactory>(config_options, value,
-                                                   factory);
-}
-
 const std::string BlockBasedTablePropertyNames::kIndexType =
     "rocksdb.block.based.table.index.type";
 const std::string BlockBasedTablePropertyNames::kWholeKeyFiltering =

table/block_based/block_based_table_iterator.cc

@@ -1162,8 +1162,7 @@ void BlockBasedTableIterator::Prepare(
 #endif
         assert(pinned_data_blocks_guard[block_idx].IsEmpty());
         s = table_->CreateAndPinBlockInCache<Block_kData>(
-            read_options_, block, table_->get_rep()->decompressor.get(),
-            &tmp_contents,
+            read_options_, block, &tmp_contents,
             &(pinned_data_blocks_guard[block_idx].As<Block_kData>()));
         if (!s.ok()) {
           // Abort: failed to create and pin block in cache

table/block_based/block_based_table_reader.cc

@@ -109,8 +109,8 @@ CacheAllocationPtr CopyBufferToHeap(MemoryAllocator* allocator, Slice& buf) {
       CachableEntry<T>* out_parsed_block) const;                               \
   template Status BlockBasedTable::CreateAndPinBlockInCache<T>(                \
       const ReadOptions& ro, const BlockHandle& handle,                        \
-      UnownedPtr<Decompressor> decomp, BlockContents* block_contents,          \
-      CachableEntry<T>* out_parsed_block) const;
+      BlockContents* block_contents, CachableEntry<T>* out_parsed_block)       \
+      const;
 
 INSTANTIATE_BLOCKLIKE_TEMPLATES(ParsedFullFilterBlock);
 INSTANTIATE_BLOCKLIKE_TEMPLATES(DecompressorDict);
@@ -1741,55 +1741,13 @@ Status BlockBasedTable::LookupAndPinBlocksInCache(
 
 template <typename TBlocklike>
 Status BlockBasedTable::CreateAndPinBlockInCache(
-    const ReadOptions& ro, const BlockHandle& handle,
-    UnownedPtr<Decompressor> decomp, BlockContents* contents,
+    const ReadOptions& ro, const BlockHandle& handle, BlockContents* contents,
     CachableEntry<TBlocklike>* out_parsed_block) const {
-  CompressionType compression_type = GetBlockCompressionType(*contents);
-  // If we don't own the contents and we don't need to decompress, copy
-  // the block to heap in order to have ownership. If decompression is
-  // needed, then the decompressor will allocate a buffer.
-  if (!contents->own_bytes() && compression_type == kNoCompression) {
-    Slice src = Slice(contents->data.data(), BlockSizeWithTrailer(handle));
-    *contents = BlockContents(
-        CopyBufferToHeap(GetMemoryAllocator(rep_->table_options), src),
-        handle.size());
-#ifndef NDEBUG
-    contents->has_trailer = true;
-#endif
-  }
-
-  Status s;
-  if (ro.fill_cache) {
-    s = MaybeReadBlockAndLoadToCache(nullptr, ro, handle, decomp,
-                                     /*for_compaction=*/false, out_parsed_block,
-                                     nullptr, nullptr, contents,
-                                     /*async_read=*/false,
-                                     /*use_block_cache_for_lookup=*/true);
-  }
-
-  if (!s.ok()) {
-    return s;
-  }
-
-  // fill_cache could be false, or no block cache is configured. In that
-  // case, decompress if necessary and take ownership of the block
-  if (out_parsed_block->GetValue() == nullptr && contents != nullptr) {
-    BlockContents tmp_contents;
-    if (compression_type != kNoCompression) {
-      s = DecompressSerializedBlock(contents->data.data(), handle.size(),
-                                    compression_type, *decomp, &tmp_contents,
-                                    rep_->ioptions,
-                                    GetMemoryAllocator(rep_->table_options));
-    } else {
-      tmp_contents = std::move(*contents);
-    }
-    if (s.ok()) {
-      std::unique_ptr<TBlocklike> block_holder;
-      rep_->create_context.Create(&block_holder, std::move(tmp_contents));
-      out_parsed_block->SetOwnedValue(std::move(block_holder));
-    }
-  }
-  return s;
+  return MaybeReadBlockAndLoadToCache(
+      nullptr, ro, handle, rep_->decompressor.get(),
+      /*for_compaction=*/false, out_parsed_block, nullptr, nullptr, contents,
+      /*async_read=*/false,
+      /*use_block_cache_for_lookup=*/true);
 }
 
 // If contents is nullptr, this function looks up the block caches for the

table/block_based/block_based_table_reader.h

@@ -311,7 +311,7 @@ class BlockBasedTable : public TableReader {
   template <typename TBlocklike>
   Status CreateAndPinBlockInCache(
       const ReadOptions& ro, const BlockHandle& handle,
-      UnownedPtr<Decompressor> decomp, BlockContents* block_contents,
+      BlockContents* block_contents,
       CachableEntry<TBlocklike>* out_parsed_block) const;
 
   struct Rep;

table/block_based/block_based_table_reader_sync_and_async.h

@@ -37,6 +37,8 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
   RandomAccessFileReader* file = rep_->file.get();
   const Footer& footer = rep_->footer;
   const ImmutableOptions& ioptions = rep_->ioptions;
+  size_t read_amp_bytes_per_bit = rep_->table_options.read_amp_bytes_per_bit;
+  MemoryAllocator* memory_allocator = GetMemoryAllocator(rep_->table_options);
 
   if (ioptions.allow_mmap_reads) {
     size_t idx_in_batch = 0;
@@ -264,8 +266,79 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
     }
 
     if (s.ok()) {
-      s = CreateAndPinBlockInCache(options, handle, decomp, &serialized_block,
-                                   &results[idx_in_batch]);
+      // When the blocks share the same underlying buffer (scratch or direct io
+      // buffer), we may need to manually copy the block into heap if the
+      // serialized block has to be inserted into a cache. That falls into the
+      // following cases -
+      // 1. serialized block is not compressed, it needs to be inserted into
+      //    the uncompressed block cache if there is one
+      // 2. If the serialized block is compressed, it needs to be inserted
+      //    into the compressed block cache if there is one
+      //
+      // In all other cases, the serialized block is either uncompressed into a
+      // heap buffer or there is no cache at all.
+      CompressionType compression_type =
+          GetBlockCompressionType(serialized_block);
+      if ((use_fs_scratch || use_shared_buffer) &&
+          compression_type == kNoCompression) {
+        Slice serialized =
+            Slice(req.result.data() + req_offset, BlockSizeWithTrailer(handle));
+        serialized_block = BlockContents(
+            CopyBufferToHeap(GetMemoryAllocator(rep_->table_options),
+                             serialized),
+            handle.size());
+#ifndef NDEBUG
+        serialized_block.has_trailer = true;
+#endif
+      }
+    }
+
+    if (s.ok()) {
+      if (options.fill_cache) {
+        CachableEntry<Block_kData>* block_entry = &results[idx_in_batch];
+        // MaybeReadBlockAndLoadToCache will insert into the block caches if
+        // necessary. Since we're passing the serialized block contents, it
+        // will avoid looking up the block cache
+        s = MaybeReadBlockAndLoadToCache(
+            nullptr, options, handle, decomp,
+            /*for_compaction=*/false, block_entry, mget_iter->get_context,
+            /*lookup_context=*/nullptr, &serialized_block,
+            /*async_read=*/false, /*use_block_cache_for_lookup=*/true);
+
+        if (!s.ok()) {
+          statuses[idx_in_batch] = s;
+          continue;
+        }
+        // block_entry value could be null if no block cache is present, i.e
+        // BlockBasedTableOptions::no_block_cache is true and no compressed
+        // block cache is configured. In that case, fall
+        // through and set up the block explicitly
+        if (block_entry->GetValue() != nullptr) {
+          continue;
+        }
+      }
+
+      CompressionType compression_type =
+          GetBlockCompressionType(serialized_block);
+      BlockContents contents;
+      if (compression_type != kNoCompression) {
+        s = DecompressSerializedBlock(
+            req.result.data() + req_offset, handle.size(), compression_type,
+            *decomp, &contents, rep_->ioptions, memory_allocator);
+      } else {
+        // There are two cases here:
+        // 1) caller uses the shared buffer (scratch or direct io buffer);
+        // 2) we use the requst buffer.
+        // If scratch buffer or direct io buffer is used, we ensure that
+        // all serialized blocks are copyed to the heap as single blocks. If
+        // scratch buffer is not used, we also have no combined read, so the
+        // serialized block can be used directly.
+        contents = std::move(serialized_block);
+      }
+      if (s.ok()) {
+        results[idx_in_batch].SetOwnedValue(std::make_unique<Block_kData>(
+            std::move(contents), read_amp_bytes_per_bit, ioptions.stats));
+      }
     }
     statuses[idx_in_batch] = s;
   }

table/block_based/user_defined_index_wrapper.h

@@ -46,24 +46,9 @@ class UserDefinedIndexBuilderWrapper : public IndexBuilder {
     handle.offset = block_handle.offset();
     handle.size = block_handle.size();
     // Forward the call to both index builders
-    ParsedInternalKey pkey_last;
-    ParsedInternalKey pkey_first;
-    // There's no way to return an error here, so we remember the statsu and
-    // return it in Finish()
-    if (status_.ok()) {
-      status_ = ParseInternalKey(last_key_in_current_block, &pkey_last,
-                                 /*lof_err_key*/ false);
-    }
-    if (status_.ok() && first_key_in_next_block) {
-      status_ = ParseInternalKey(*first_key_in_next_block, &pkey_first,
-                                 /*lof_err_key*/ false);
-    }
-    if (status_.ok()) {
-      user_defined_index_builder_->AddIndexEntry(
-          pkey_last.user_key,
-          first_key_in_next_block ? &pkey_first.user_key : nullptr, handle,
-          separator_scratch);
-    }
+    user_defined_index_builder_->AddIndexEntry(last_key_in_current_block,
+                                               first_key_in_next_block, handle,
+                                               separator_scratch);
     return internal_index_builder_->AddIndexEntry(
         last_key_in_current_block, first_key_in_next_block, block_handle,
         separator_scratch);
@@ -71,13 +56,13 @@ class UserDefinedIndexBuilderWrapper : public IndexBuilder {
 
   void OnKeyAdded(const Slice& key,
                   const std::optional<Slice>& value) override {
-    ParsedInternalKey pkey;
     if (status_.ok()) {
       if (!value.has_value()) {
         status_ = Status::InvalidArgument(
             "user_defined_index_factory not supported with parallel "
             "compression");
       } else {
+        ParsedInternalKey pkey;
         status_ = ParseInternalKey(key, &pkey, /*lof_err_key*/ false);
         if (status_.ok() && pkey.type != ValueType::kTypeValue) {
           status_ = Status::InvalidArgument(
@@ -91,38 +76,16 @@ class UserDefinedIndexBuilderWrapper : public IndexBuilder {
 
     // Forward the call to both index builders
     internal_index_builder_->OnKeyAdded(key, value);
-
-    // Pass the user key to the UDI. We don't expect multiple entries with
-    // different sequence numbers for the same key in the file. RocksDB may
-    // enforce it in the future by allowing UDIs only for read only
-    // bulkloaded use cases, and only allow ingestion of files with
-    // sequence number 0.
     user_defined_index_builder_->OnKeyAdded(
-        pkey.user_key, UserDefinedIndexBuilder::ValueType::kValue,
-        value.value());
+        key, UserDefinedIndexBuilder::ValueType::kValue, value.value());
   }
 
   Status Finish(IndexBlocks* index_blocks,
                 const BlockHandle& last_partition_block_handle) override {
-    if (!status_.ok() && !status_.IsIncomplete()) {
+    if (!status_.ok()) {
       return status_;
     }
 
-    if (!udi_finished_) {
-      // Finish the user defined index builder
-      Slice user_index_contents;
-      status_ = user_defined_index_builder_->Finish(&user_index_contents);
-      if (!status_.ok()) {
-        return status_;
-      }
-
-      // Add the user defined index to the meta blocks
-      std::string block_name = kUserDefinedIndexPrefix + name_;
-      index_blocks->meta_blocks.insert(
-          {block_name, {BlockType::kUserDefinedIndex, user_index_contents}});
-      udi_finished_ = true;
-    }
-
     // Finish the internal index builder
     status_ = internal_index_builder_->Finish(index_blocks,
                                               last_partition_block_handle);
@@ -130,6 +93,18 @@ class UserDefinedIndexBuilderWrapper : public IndexBuilder {
       return status_;
     }
 
+    // Finish the user defined index builder
+    Slice user_index_contents;
+    status_ = user_defined_index_builder_->Finish(&user_index_contents);
+    if (!status_.ok()) {
+      return status_;
+    }
+
+    // Add the user defined index to the meta blocks
+    std::string block_name = kUserDefinedIndexPrefix + name_;
+    index_blocks->meta_blocks.insert(
+        {block_name, {BlockType::kUserDefinedIndex, user_index_contents}});
+
     index_size_ = internal_index_builder_->IndexSize();
     return status_;
   }
@@ -145,7 +120,6 @@ class UserDefinedIndexBuilderWrapper : public IndexBuilder {
   std::unique_ptr<IndexBuilder> internal_index_builder_;
   std::unique_ptr<UserDefinedIndexBuilder> user_defined_index_builder_;
   Status status_;
-  bool udi_finished_ = false;
 };
 
 class UserDefinedIndexIteratorWrapper
@@ -170,41 +144,23 @@ class UserDefinedIndexIteratorWrapper
     status_ = ParseInternalKey(target, &pkey, /*log_err_key=*/false);
     if (status_.ok()) {
       status_ = udi_iter_->SeekAndGetResult(pkey.user_key, &result_);
-      if (status_.ok()) {
-        valid_ = result_.bound_check_result == IterBoundCheck::kInbound;
-        if (valid_) {
-          ikey_.Set(result_.key, 0, ValueType::kTypeValue);
-        }
-      }
-    } else {
-      valid_ = false;
+      valid_ = status_.ok() &&
+               result_.bound_check_result == IterBoundCheck::kInbound;
     }
   }
 
   void Next() override {
     status_ = udi_iter_->NextAndGetResult(&result_);
-    if (status_.ok()) {
-      valid_ = result_.bound_check_result == IterBoundCheck::kInbound;
-      if (valid_) {
-        ikey_.Set(result_.key, 0, ValueType::kTypeValue);
-      }
-    } else {
-      valid_ = false;
-    }
+    valid_ =
+        status_.ok() && result_.bound_check_result == IterBoundCheck::kInbound;
   }
 
   bool NextAndGetResult(IterateResult* result) override {
     status_ = udi_iter_->NextAndGetResult(&result_);
+    valid_ =
+        status_.ok() && result_.bound_check_result == IterBoundCheck::kInbound;
     if (status_.ok()) {
-      valid_ = result_.bound_check_result == IterBoundCheck::kInbound;
-      if (valid_) {
-        ikey_.Set(result_.key, 0, ValueType::kTypeValue);
-      }
-      if (status_.ok()) {
-        *result = result_;
-      }
-    } else {
-      valid_ = false;
+      *result = result_;
     }
     return valid_;
   }
@@ -215,7 +171,7 @@ class UserDefinedIndexIteratorWrapper
 
   void Prev() override { status_ = Status::NotSupported("Prev not supported"); }
 
-  Slice key() const override { return Slice(*ikey_.const_rep()); }
+  Slice key() const override { return result_.key; }
 
   IndexValue value() const override {
     auto handle = udi_iter_->value();
@@ -232,7 +188,6 @@ class UserDefinedIndexIteratorWrapper
  private:
   std::unique_ptr<UserDefinedIndexIterator> udi_iter_;
   IterateResult result_;
-  InternalKey ikey_;
   Status status_;
   bool valid_;
 };

table/table_test.cc

@@ -53,7 +53,6 @@
 #include "rocksdb/trace_record.h"
 #include "rocksdb/unique_id.h"
 #include "rocksdb/user_defined_index.h"
-#include "rocksdb/utilities/object_registry.h"
 #include "rocksdb/write_buffer_manager.h"
 #include "table/block_based/block.h"
 #include "table/block_based/block_based_table_builder.h"
@@ -2060,7 +2059,7 @@ TEST_P(BlockBasedTableTest, PrefetchTest) {
 
   // Simple
   PrefetchRange(&c, &opt, &table_options,
-                /*key_begin=*/"k01", /*key_end=*/"k05",
+                /*key_range=*/"k01", "k05",
                 /*keys_in_cache=*/{"k01", "k02", "k03", "k04", "k05"},
                 /*keys_not_in_cache=*/{"k06", "k07"});
   PrefetchRange(&c, &opt, &table_options, "k01", "k01", {"k01", "k02", "k03"},
@@ -7458,11 +7457,8 @@ class UserDefinedIndexTest : public BlockBasedTableTestBase {
                           const Slice* first_key_in_next_block,
                           const BlockHandle& block_handle,
                           std::string* separator_scratch) override {
-        EXPECT_EQ(last_key_in_current_block.size(), 5);
-        if (first_key_in_next_block) {
-          EXPECT_EQ(first_key_in_next_block->size(), 5);
-        }
         // Unused parameters
+        (void)first_key_in_next_block;
         (void)separator_scratch;
         entries_added_++;
         // Store the block handle for each key
@@ -7476,9 +7472,8 @@ class UserDefinedIndexTest : public BlockBasedTableTestBase {
         return last_key_in_current_block;
       }
 
-      void OnKeyAdded(const Slice& key, ValueType /*value*/,
+      void OnKeyAdded(const Slice& /*key*/, ValueType /*value*/,
                       const Slice& /*value*/) override {
-        EXPECT_EQ(key.size(), 5);
         // Track keys added to the index
         keys_added_++;
       }
@@ -7662,12 +7657,9 @@ class UserDefinedIndexTest : public BlockBasedTableTestBase {
           index_data_;
     };
   };
-
- protected:
-  void BasicTest(bool use_partitioned_index);
 };
 
-void UserDefinedIndexTest::BasicTest(bool use_partitioned_index) {
+TEST_F(UserDefinedIndexTest, BasicTest) {
   Options options;
   BlockBasedTableOptions table_options;
   std::string dbname = test::PerThreadDBPath("user_defined_index_test");
@@ -7677,11 +7669,7 @@ void UserDefinedIndexTest::BasicTest(bool use_partitioned_index) {
   auto user_defined_index_factory =
       std::make_shared<TestUserDefinedIndexFactory>();
   table_options.user_defined_index_factory = user_defined_index_factory;
-  if (use_partitioned_index) {
-    table_options.partition_filters = true;
-    table_options.decouple_partitioned_filters = true;
-    table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch;
-  }
+
   // Set up custom flush block policy that flushes every 3 keys
   table_options.flush_block_policy_factory =
       std::make_shared<CustomFlushBlockPolicyFactory>();
@@ -7821,14 +7809,6 @@ void UserDefinedIndexTest::BasicTest(bool use_partitioned_index) {
   ASSERT_OK(iter->status());
 }
 
-TEST_F(UserDefinedIndexTest, BasicTestWithPartitionedIndex) {
-  BasicTest(/*use_partitioned_index=*/true);
-}
-
-TEST_F(UserDefinedIndexTest, BasicTestWithoutPartitionedIndex) {
-  BasicTest(/*use_partitioned_index=*/false);
-}
-
 TEST_F(UserDefinedIndexTest, InvalidArgumentTest1) {
   Options options;
   BlockBasedTableOptions table_options;
@@ -8044,91 +8024,6 @@ TEST_F(UserDefinedIndexTest, IngestFailTest) {
   ASSERT_OK(db->Close());
   ASSERT_OK(DestroyDB(dbname, options));
 }
-
-TEST_F(UserDefinedIndexTest, ConfigTest) {
-  Options options;
-  BlockBasedTableOptions table_options;
-  std::string dbname = test::PerThreadDBPath("user_defined_index_test");
-  std::string ingest_file = dbname + "test.sst";
-
-  // Set up the user-defined index factory
-  auto user_defined_index_factory =
-      std::make_shared<TestUserDefinedIndexFactory>();
-  table_options.user_defined_index_factory = user_defined_index_factory;
-
-  // Set up custom flush block policy that flushes every 3 keys
-  table_options.flush_block_policy_factory =
-      std::make_shared<CustomFlushBlockPolicyFactory>();
-
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-
-  std::unique_ptr<SstFileWriter> writer;
-  writer.reset(new SstFileWriter(EnvOptions(), options));
-  ASSERT_OK(writer->Open(ingest_file));
-
-  // Add 100 keys instead of just 5
-  for (int i = 0; i < 100; i++) {
-    std::stringstream ss;
-    ss << std::setw(2) << std::setfill('0') << i;
-    std::string key = "key" + ss.str();
-    std::string value = "value" + ss.str();
-    ASSERT_OK(writer->Put(key, value));
-  }
-  ASSERT_OK(writer->Finish());
-  writer.reset();
-
-  table_options.user_defined_index_factory.reset();
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  // Set up the user-defined index factory
-  ObjectLibrary::Default().get()->AddFactory<UserDefinedIndexFactory>(
-      "test_index", [](const std::string& /* uri */,
-                       std::unique_ptr<UserDefinedIndexFactory>* guard,
-                       std::string* /* errmsg */) {
-        auto factory = new TestUserDefinedIndexFactory();
-        guard->reset(factory);
-        return guard->get();
-      });
-  ASSERT_OK(GetColumnFamilyOptionsFromString(
-      ConfigOptions(), options,
-      "block_based_table_factory={user_defined_index_factory=test_index;}",
-      &options));
-
-  std::unique_ptr<DB> db;
-  options.create_if_missing = true;
-  Status s = DB::Open(options, dbname, &db);
-  ASSERT_OK(s);
-  ASSERT_TRUE(db != nullptr);
-  ColumnFamilyHandle* cfh = nullptr;
-  ASSERT_OK(db->CreateColumnFamily(options, "new_cf", &cfh));
-
-  IngestExternalFileOptions ifo;
-  s = db->IngestExternalFile(cfh, {ingest_file}, ifo);
-  ASSERT_OK(s);
-
-  ReadOptions ro;
-  ro.table_index_factory = user_defined_index_factory.get();
-  std::unique_ptr<Iterator> iter(db->NewIterator(ro, cfh));
-  ASSERT_NE(iter, nullptr);
-
-  std::vector<ScanOptions> scan_opts({ScanOptions("key20")});
-  scan_opts[0].property_bag.emplace().emplace("count", std::to_string(25));
-  iter->Prepare(scan_opts);
-  // Test that we can read all the keys
-  int key_count = 0;
-  for (iter->Seek(scan_opts[0].range.start.value()); iter->Valid();
-       iter->Next()) {
-    key_count++;
-  }
-  ASSERT_GE(key_count, 25);
-  // The index may undercount by 2 blocks
-  ASSERT_LE(key_count, 30);
-  ASSERT_OK(iter->status());
-  iter.reset();
-
-  ASSERT_OK(db->DestroyColumnFamilyHandle(cfh));
-  ASSERT_OK(db->Close());
-  ASSERT_OK(DestroyDB(dbname, options));
-}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

tools/check_format_compatible.sh

@@ -137,7 +137,7 @@ EOF
 
 # To check for DB forward compatibility with loading options (old version
 # reading data from new), as well as backward compatibility
-declare -a db_forward_with_options_refs=("8.6.fb" "8.7.fb" "8.8.fb" "8.9.fb" "8.10.fb" "8.11.fb" "9.0.fb" "9.1.fb" "9.2.fb" "9.3.fb" "9.4.fb" "9.5.fb" "9.6.fb" "9.7.fb" "9.8.fb" "9.9.fb" "9.10.fb" "9.11.fb" "10.0.fb" "10.1.fb" "10.2.fb" "10.3.fb" "10.4.fb")
+declare -a db_forward_with_options_refs=("8.6.fb" "8.7.fb" "8.8.fb" "8.9.fb" "8.10.fb" "8.11.fb" "9.0.fb" "9.1.fb" "9.2.fb" "9.3.fb" "9.4.fb" "9.5.fb" "9.6.fb" "9.7.fb" "9.8.fb" "9.9.fb" "9.10.fb" "9.11.fb" "10.0.fb" "10.1.fb" "10.2.fb" "10.3.fb" "10.4.fb" "10.5.fb")
 # To check for DB forward compatibility without loading options (in addition
 # to the "with loading options" set), as well as backward compatibility
 declare -a db_forward_no_options_refs=() # N/A at the moment

unreleased_history/public_api_changes/get_ttl_in_ttl_db.md

@@ -0,0 +1 @@
+GetTtl() API is now available in TTL DB

utilities/transactions/lock/point/point_lock_manager.cc

@@ -322,9 +322,6 @@ Status PointLockManager::AcquireWithTimeout(
         // instead of exiting this while loop below.
         uint64_t now = env->NowMicros();
         if (static_cast<uint64_t>(cv_end_time) > now) {
-          // This may be invoked multiple times since we divide
-          // the time into smaller intervals.
-          (void)ROCKSDB_THREAD_YIELD_CHECK_ABORT();
           result = stripe->stripe_cv->WaitFor(stripe->stripe_mutex,
                                               cv_end_time - now);
           cv_wait_fail = !result.ok() && !result.IsTimedOut();