Update HISTORY and version

Summary:
**Context/Summary:**
Fix a race condition (illustrated below) in FIFO size-based compaction where concurrent threads could select the same non-L0 file, causing assertion failures in debug builds or "Cannot delete table file from LSM tree" errors in release builds.
```
Thread 1                           Thread 2
--------                           --------
FIFO size-based compaction
   ↓
Pick L2 file
   ↓
Mark: file.being_compacted = true (file.being_compacted was false)
   ↓
WriteManifestStart (unlock mutex) ─→ FIFO size-based compaction starts
   ↓                                   ↓
Continue manifest write...          Pick SAME L2 file
   ↓                                Mark: file.being_compacted = true  (file.being_compacted was true) ❌
   ↓                                   ↓
   ↓                                Unlock mutex, wait for manifest
   ↓                                   ↓
Lock mutex ←─────────────────────────────────┘
   ↓
Delete L2 file ✅
   ↓
Complete ─────────────────────────────→ Try delete same file ❌
                                        ↓
                                     ERROR: "file not in LSM tree"

🐛 BUG: Both threads pick the same file!
    Thread 2 doesn't properly check file.being_compacted flag
```

**Test**
New test that fails before the fix and passes after

Pull Request resolved: https://github.com/facebook/rocksdb/pull/13946

Reviewed By: xingbowang

Differential Revision: D82279731

Pulled By: hx235

fbshipit-source-id: b426517f2d1b23dd7d4951157822a2d322fe1435

.github/workflows/pr-jobs.yml

@@ -196,36 +196,21 @@ jobs:
     - uses: actions/checkout@v4.1.0
     - run: make V=1 -j32 LIB_MODE=shared release
     - run: ls librocksdb.so
-    - run: "./db_stress --version"
+    - run: "./trace_analyzer --version" # A tool dependent on gflags that can run in release build
     - run: make clean
-    - run: make V=1 -j32 release
+    - run: USE_RTTI=1 make V=1 -j32 release
     - run: ls librocksdb.a
-    - run: "./db_stress --version"
+    - run: "./trace_analyzer --version"
     - run: make clean
     - run: apt-get remove -y libgflags-dev
     - run: make V=1 -j32 LIB_MODE=shared release
     - run: ls librocksdb.so
-    - run: if ./db_stress --version; then false; else true; fi
+    - run: if ./trace_analyzer --version; then false; else true; fi
     - run: make clean
-    - run: make V=1 -j32 release
+    - run: USE_RTTI=1 make V=1 -j32 release
     - run: ls librocksdb.a
-    - run: if ./db_stress --version; then false; else true; fi
+    - run: if ./trace_analyzer --version; then false; else true; fi
     - uses: "./.github/actions/post-steps"
-  build-linux-release-rtti:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 8-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
-    - run: "./db_stress --version"
-    - run: make clean
-    - run: apt-get remove -y libgflags-dev
-    - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
-    - run: if ./db_stress --version; then false; else true; fi
   build-linux-clang-no_test_run:
     if: ${{ github.repository_owner == 'facebook' }}
     runs-on:
@@ -248,6 +233,8 @@ jobs:
     - uses: actions/checkout@v4.1.0
     - uses: "./.github/actions/pre-steps"
     - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench
+    - run: make clean
+    - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 DEBUG_LEVEL=0 make -j32 release
     - uses: "./.github/actions/post-steps"
   build-linux-gcc-8-no_test_run:
     if: ${{ github.repository_owner == 'facebook' }}
@@ -321,7 +308,7 @@ jobs:
     - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000' blackbox_crash_test_with_atomic_flush
     - uses: "./.github/actions/post-steps"
   # ======================= Linux with Sanitizers ===================== #
-  build-linux-clang10-asan:
+  build-linux-clang10-asan-ubsan:
     if: ${{ github.repository_owner == 'facebook' }}
     runs-on:
       labels: 32-core-ubuntu
@@ -331,19 +318,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4.1.0
     - uses: "./.github/actions/pre-steps"
-    - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check
-    - uses: "./.github/actions/post-steps"
-  build-linux-clang10-ubsan:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: zjay437/rocksdb:0.6
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - run: COMPILE_WITH_UBSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check
+    - run: COMPILE_WITH_ASAN=1 COMPILE_WITH_UBSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j40 check
     - uses: "./.github/actions/post-steps"
   build-linux-clang13-mini-tsan:
     if: ${{ github.repository_owner == 'facebook' }}

BUCK

@@ -114,6 +114,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "db/write_controller.cc",
         "db/write_stall_stats.cc",
         "db/write_thread.cc",
+        "db_stress_tool/db_stress_compression_manager.cc",
         "env/composite_env.cc",
         "env/env.cc",
         "env/env_chroot.cc",
@@ -422,6 +423,7 @@ rocks_cpp_library_wrapper(name="rocksdb_stress_lib", srcs=[
         "db_stress_tool/batched_ops_stress.cc",
         "db_stress_tool/cf_consistency_stress.cc",
         "db_stress_tool/db_stress_common.cc",
+        "db_stress_tool/db_stress_compression_manager.cc",
         "db_stress_tool/db_stress_driver.cc",
         "db_stress_tool/db_stress_filters.cc",
         "db_stress_tool/db_stress_gflags.cc",
@@ -5194,6 +5196,12 @@ cpp_unittest_wrapper(name="inlineskiplist_test",
             extra_compiler_flags=[])
 
 
+cpp_unittest_wrapper(name="interval_test",
+            srcs=["util/interval_test.cc"],
+            deps=[":rocksdb_test_lib"],
+            extra_compiler_flags=[])
+
+
 cpp_unittest_wrapper(name="io_posix_test",
             srcs=["env/io_posix_test.cc"],
             deps=[":rocksdb_test_lib"],

CMakeLists.txt

@@ -747,6 +747,7 @@ set(SOURCES
         db/write_controller.cc
         db/write_stall_stats.cc
         db/write_thread.cc
+        db_stress_tool/db_stress_compression_manager.cc
         env/composite_env.cc
         env/env.cc
         env/env_chroot.cc

HISTORY.md

@@ -1,6 +1,43 @@
 # Rocksdb Change Log
 > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
 
+## 10.6.2 (09/15/2025)
+### Bug Fixes
+* Fix a race condition in FIFO size-based compaction where concurrent threads could select the same non-L0 file, causing assertion failures in debug builds or "Cannot delete table file from LSM tree" errors in release builds.
+
+## 10.6.1 (09/05/2025)
+### New Features
+* Add the fail_if_no_udi_on_open flag in BlockBasedTableOption to control whether a missing user defined index block in a SST is a hard error or not.
+* Add new option `MultiScanArgs::max_prefetch_size` that limits the memory usage of per file pinning of prefetched blocks.
+
+## 10.6.0 (08/22/2025)
+### New Features
+* Introduce column family option `cf_allow_ingest_behind`. This option aims to replace `DBOptions::allow_ingest_behind` to enable ingest behind at the per-CF level. `DBOptions::allow_ingest_behind` is deprecated.
+* Introduce `MultiScanArgs::io_coalesce_threshold` to allow a configurable IO coalescing threshold.
+
+### Public API Changes
+* `IngestExternalFileOptions::allow_db_generated_files` now allows files ingestion of any DB generated SST file, instead of only the ones with all keys having sequence number 0.
+* `decouple_partitioned_filters = true` is now the default in BlockBasedTableOptions.
+* GetTtl() API is now available in TTL DB
+* Minimum supported version of LZ4 library is now 1.7.0 (r129 from 2015)
+* Some changes to experimental Compressor and CompressionManager APIs
+* A new Filesystem::SyncFile function is added for syncing a file that was already written, such as on file ingestion. The default implementation matches previous RocksDB behavior: re-open the file for read-write, sync it, and close it. We recommend overriding for FileSystems that do not require syncing for crash recovery or do not handle (well) re-opening for writes.
+
+### Behavior Changes
+* When `allow_ingest_behind` is enabled, compaction will no longer drop tombstones based on the absence of underlying data. Tombstones will be preserved to apply to ingested files.
+
+### Bug Fixes
+* Files in dropped column family won't be returned to the caller upon successful, offline MANIFEST iteration in `GetFileChecksumsFromCurrentManifest`.
+* Fix a bug in MultiScan that causes it to fall back to a normal scan when dictionary compression is enabled.
+* Fix a crash in iterator Prepare() when fill_cache=false
+* Fix a bug in MultiScan where incorrect results can be returned when a Scan's range is across multiple files.
+* Fixed a bug in remote compaction that may mistakenly delete live SST file(s) during the cleanup phase when no keys survive the compaction (all expired)
+* Allow a user defined index to be configured from a string.
+* Make the User Defined Index interface consistently use the user key format, fixing the previous mixed usage of internal and user key.
+
+### Performance Improvements
+* Small improvement to CPU efficiency of compression using built-in algorithms, and a dramatic efficiency improvement for LZ4HC, based on reusing data structures between invocations.
+
 ## 10.5.0 (07/18/2025)
 ### Public API Changes
 * DB option skip_checking_sst_file_sizes_on_db_open is deprecated, in favor of validating file size in parallel in a thread pool, when db is opened. When DB is opened, with paranoid check enabled, a file with the wrong size would fail the DB open. With paranoid check disabled, the DB open would succeed, the column family with the corrupted file would not be read or write, while the other healthy column families could be read and write normally. When max_open_files option is not set to -1, only a subset of the files will be opened and checked. The rest of the files will be opened and checked when they are accessed.

Makefile

@@ -1357,6 +1357,9 @@ filter_bench: $(OBJ_DIR)/util/filter_bench.o $(LIBRARY)
 db_stress: $(OBJ_DIR)/db_stress_tool/db_stress.o $(STRESS_LIBRARY) $(TOOLS_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+db_stress_compression_manager: $(OBJ_DIR)/db_stress_tool/db_stress_compression_manager.o $(LIBRARY)
+	$(AM_LINK)
+
 write_stress: $(OBJ_DIR)/tools/write_stress.o $(LIBRARY)
 	$(AM_LINK)
 
@@ -2037,6 +2040,9 @@ wide_column_serialization_test: $(OBJ_DIR)/db/wide/wide_column_serialization_tes
 wide_columns_helper_test: $(OBJ_DIR)/db/wide/wide_columns_helper_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
+interval_test: $(OBJ_DIR)/util/interval_test.o $(TEST_LIBRARY) $(LIBRARY)
+	$(AM_LINK)
+
 #-------------------------------------------------
 # make install related stuff
 PREFIX ?= /usr/local

build_tools/build_detect_platform

@@ -55,8 +55,11 @@ fi
 # we currently depend on POSIX platform
 COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX"
 
-# Default to fbcode gcc on internal fb machines
-if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
+# Default to fbcode gcc on Meta internal machines
+IS_META_HOST="$(hostname | grep -E '(facebook|meta).com|fbinfra.net')"
+if [ -z "$ROCKSDB_NO_FBCODE" -a "$IS_META_HOST" ]; then
+  if [ -d /mnt/gvfs/third-party ]; then
+    echo "NOTE: Using fbcode build" >&2
     FBCODE_BUILD="true"
     # If we're compiling with TSAN or shared lib, we need pic build
     PIC_BUILD=$COMPILE_WITH_TSAN
@@ -64,6 +67,11 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
       PIC_BUILD=1
     fi
     source "$PWD/build_tools/fbcode_config_platform010.sh"
+  else
+    echo "************************************************************************" >&2
+    echo "WARNING: -d /mnt/gvfs/third-party failed; no fbcode build" >&2
+    echo "************************************************************************" >&2
+  fi
 fi
 
 # Delete existing output, if it exists
@@ -71,7 +79,9 @@ rm -f "$OUTPUT"
 touch "$OUTPUT"
 
 if test -z "$CC"; then
-    if [ -x "$(command -v cc)" ]; then
+    if [ "$USE_CLANG" -a -x "$(command -v clang)" ]; then
+        CC=clang
+    elif [ -x "$(command -v cc)" ]; then
         CC=cc
     elif [ -x "$(command -v clang)" ]; then
         CC=clang
@@ -81,7 +91,9 @@ if test -z "$CC"; then
 fi
 
 if test -z "$CXX"; then
-    if [ -x "$(command -v g++)" ]; then
+    if [ "$USE_CLANG" -a -x "$(command -v clang++)" ]; then
+        CXX=clang++
+    elif [ -x "$(command -v g++)" ]; then
         CXX=g++
     elif [ -x "$(command -v clang++)" ]; then
         CXX=clang++
@@ -91,7 +103,9 @@ if test -z "$CXX"; then
 fi
 
 if test -z "$AR"; then
-    if [ -x "$(command -v gcc-ar)" ]; then
+    if [ "$USE_CLANG" -a -x "$(command -v llvm-ar)" ]; then
+        AR=llvm-ar
+    elif [ -x "$(command -v gcc-ar)" ]; then
         AR=gcc-ar
     elif [ -x "$(command -v llvm-ar)" ]; then
         AR=llvm-ar

cache/cache_bench_tool.cc

@@ -60,6 +60,8 @@ DEFINE_uint32(value_bytes, 8 * KiB, "Size of each value added.");
 DEFINE_uint32(value_bytes_estimate, 0,
               "If > 0, overrides estimated_entry_charge or "
               "min_avg_entry_charge depending on cache_type.");
+DEFINE_double(compressible_to_ratio, 0.5,
+              "Approximate size ratio that values can be compressed to.");
 
 DEFINE_int32(
     degenerate_hash_bits, 0,
@@ -291,10 +293,19 @@ struct KeyGen {
 
 Cache::ObjectPtr createValue(Random64& rnd, MemoryAllocator* alloc) {
   char* rv = AllocateBlock(FLAGS_value_bytes, alloc).release();
-  // Fill with some filler data, and take some CPU time
-  for (uint32_t i = 0; i < FLAGS_value_bytes; i += 8) {
+  // Fill with some filler data, and take some CPU time, but add redundancy
+  // as requested for compressibility.
+  uint32_t random_fill_size = std::max(
+      uint32_t{1}, std::min(FLAGS_value_bytes,
+                            static_cast<uint32_t>(FLAGS_compressible_to_ratio *
+                                                  FLAGS_value_bytes)));
+  uint32_t i = 0;
+  for (; i < random_fill_size; i += 8) {
     EncodeFixed64(rv + i, rnd.Next());
   }
+  for (; i < FLAGS_value_bytes; i++) {
+    rv[i] = rv[i % random_fill_size];
+  }
   return rv;
 }
 
@@ -309,16 +320,16 @@ Status SaveToFn(Cache::ObjectPtr from_obj, size_t /*from_offset*/,
 
 Status CreateFn(const Slice& data, CompressionType /*type*/,
                 CacheTier /*source*/, Cache::CreateContext* /*context*/,
-                MemoryAllocator* /*allocator*/, Cache::ObjectPtr* out_obj,
+                MemoryAllocator* alloc, Cache::ObjectPtr* out_obj,
                 size_t* out_charge) {
-  *out_obj = new char[data.size()];
+  *out_obj = AllocateBlock(data.size(), alloc).release();
   memcpy(*out_obj, data.data(), data.size());
   *out_charge = data.size();
   return Status::OK();
 };
 
 void DeleteFn(Cache::ObjectPtr value, MemoryAllocator* alloc) {
-  CustomDeleter{alloc}(static_cast<char*>(value));
+  CacheAllocationDeleter{alloc}(static_cast<char*>(value));
 }
 
 Cache::CacheItemHelper helper1_wos(CacheEntryRole::kDataBlock, DeleteFn);

cache/compressed_secondary_cache.cc

@@ -16,6 +16,31 @@
 #include "util/string_util.h"
 
 namespace ROCKSDB_NAMESPACE {
+namespace {
+// Format of values in CompressedSecondaryCache:
+// If enable_custom_split_merge:
+//  * A chain of CacheValueChunk representing the sequence of bytes for a tagged
+//    value. The overall length of the tagged value is determined by the chain
+//    of CacheValueChunks.
+// If !enable_custom_split_merge:
+//  * A LengthPrefixedSlice (starts with varint64 size) of a tagged value.
+//
+// A tagged value has a 2-byte header before the "saved" or compressed block
+// data:
+//  * 1 byte for "source" CacheTier indicating which tier is responsible for
+//    compression/decompression.
+//  * 1 byte for compression type which is generated/used by
+//    CompressedSecondaryCache iff source == CacheTier::kVolatileCompressedTier
+//    (original entry passed in was uncompressed). Otherwise, the compression
+//    type is preserved from the entry passed in.
+constexpr uint32_t kTagSize = 2;
+
+// Size of tag + varint size prefix when applicable
+uint32_t GetHeaderSize(size_t data_size, bool enable_split_merge) {
+  return (enable_split_merge ? 0 : VarintLength(kTagSize + data_size)) +
+         kTagSize;
+}
+}  // namespace
 
 CompressedSecondaryCache::CompressedSecondaryCache(
     const CompressedSecondaryCacheOptions& opts)
@@ -40,13 +65,9 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
     Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase,
     Statistics* stats, bool& kept_in_sec_cache) {
   assert(helper);
-  // This is a minor optimization. Its ok to skip it in TSAN in order to
-  // avoid a false positive.
-#ifndef __SANITIZE_THREAD__
-  if (disable_cache_) {
+  if (disable_cache_.LoadRelaxed()) {
     return nullptr;
   }
-#endif
 
   std::unique_ptr<SecondaryCacheResultHandle> handle;
   kept_in_sec_cache = false;
@@ -62,75 +83,58 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
     return nullptr;
   }
 
-  CacheAllocationPtr* ptr{nullptr};
-  CacheAllocationPtr merged_value;
-  size_t handle_value_charge{0};
-  const char* data_ptr = nullptr;
-  CacheTier source = CacheTier::kVolatileCompressedTier;
-  CompressionType type = cache_options_.compression_type;
+  std::string merged_value;
+  Slice tagged_data;
   if (cache_options_.enable_custom_split_merge) {
     CacheValueChunk* value_chunk_ptr =
-        reinterpret_cast<CacheValueChunk*>(handle_value);
-    merged_value = MergeChunksIntoValue(value_chunk_ptr, handle_value_charge);
-    ptr = &merged_value;
-    data_ptr = ptr->get();
+        static_cast<CacheValueChunk*>(handle_value);
+    merged_value = MergeChunksIntoValue(value_chunk_ptr);
+    tagged_data = Slice(merged_value);
   } else {
-    uint32_t type_32 = static_cast<uint32_t>(type);
-    uint32_t source_32 = static_cast<uint32_t>(source);
-    ptr = reinterpret_cast<CacheAllocationPtr*>(handle_value);
-    handle_value_charge = cache_->GetCharge(lru_handle);
-    data_ptr = ptr->get();
-    const char* limit = ptr->get() + handle_value_charge;
-    data_ptr =
-        GetVarint32Ptr(data_ptr, limit, static_cast<uint32_t*>(&type_32));
-    type = static_cast<CompressionType>(type_32);
-    data_ptr =
-        GetVarint32Ptr(data_ptr, limit, static_cast<uint32_t*>(&source_32));
-    source = static_cast<CacheTier>(source_32);
-    uint64_t data_size = 0;
-    data_ptr =
-        GetVarint64Ptr(data_ptr, limit, static_cast<uint64_t*>(&data_size));
-    assert(handle_value_charge > data_size);
-    handle_value_charge = data_size;
+    tagged_data = GetLengthPrefixedSlice(static_cast<char*>(handle_value));
   }
-  MemoryAllocator* allocator = cache_options_.memory_allocator.get();
 
-  Status s;
-  Cache::ObjectPtr value{nullptr};
-  size_t charge{0};
+  auto source = lossless_cast<CacheTier>(tagged_data[0]);
+  auto type = lossless_cast<CompressionType>(tagged_data[1]);
+
+  std::unique_ptr<char[]> uncompressed;
+  Slice saved(tagged_data.data() + kTagSize, tagged_data.size() - kTagSize);
   if (source == CacheTier::kVolatileCompressedTier) {
-    if (cache_options_.compression_type == kNoCompression ||
-        cache_options_.do_not_compress_roles.Contains(helper->role)) {
-      s = helper->create_cb(Slice(data_ptr, handle_value_charge),
-                            kNoCompression, CacheTier::kVolatileTier,
-                            create_context, allocator, &value, &charge);
-    } else {
-      // TODO: can we work some magic with create_cb, which might be based on
-      // custom compression, to decompress without an extra copy in create_cb?
+    if (type != kNoCompression) {
+      // TODO: can we do something to avoid yet another allocation?
       Decompressor::Args args;
-      args.compressed_data = Slice(data_ptr, handle_value_charge);
-      args.compression_type = cache_options_.compression_type;
-      s = decompressor_->ExtractUncompressedSize(args);
-      assert(s.ok());
+      args.compressed_data = saved;
+      args.compression_type = type;
+      Status s = decompressor_->ExtractUncompressedSize(args);
+      assert(s.ok());  // in-memory data
       if (s.ok()) {
-        auto uncompressed = std::make_unique<char[]>(args.uncompressed_size);
+        uncompressed = std::make_unique<char[]>(args.uncompressed_size);
         s = decompressor_->DecompressBlock(args, uncompressed.get());
-        assert(s.ok());
-        if (s.ok()) {
-          s = helper->create_cb(
-              Slice(uncompressed.get(), args.uncompressed_size), kNoCompression,
-              CacheTier::kVolatileTier, create_context, allocator, &value,
-              &charge);
-        }
+        assert(s.ok());  // in-memory data
       }
+      if (!s.ok()) {
+        cache_->Release(lru_handle, /*erase_if_last_ref=*/true);
+        return nullptr;
+      }
+      saved = Slice(uncompressed.get(), args.uncompressed_size);
+      type = kNoCompression;
+      // Free temporary compressed data as early as we can. This could matter
+      // for unusually large blocks because we also have
+      // * Another compressed copy above (from lru_cache).
+      // * The uncompressed copy in `uncompressed`.
+      // * Another uncompressed copy in `result_value` below.
+      // Let's try to max out at 3 copies instead of 4.
+      merged_value = std::string();
     }
-  } else {
-    // The item was not compressed by us. Let the helper create_cb
-    // uncompress it
-    s = helper->create_cb(Slice(data_ptr, handle_value_charge), type, source,
-                          create_context, allocator, &value, &charge);
+    // Reduced as if it came from primary cache
+    source = CacheTier::kVolatileTier;
   }
 
+  Cache::ObjectPtr result_value = nullptr;
+  size_t result_charge = 0;
+  Status s = helper->create_cb(saved, type, source, create_context,
+                               cache_options_.memory_allocator.get(),
+                               &result_value, &result_charge);
   if (!s.ok()) {
     cache_->Release(lru_handle, /*erase_if_last_ref=*/true);
     return nullptr;
@@ -148,7 +152,8 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
     kept_in_sec_cache = true;
     cache_->Release(lru_handle, /*erase_if_last_ref=*/false);
   }
-  handle.reset(new CompressedSecondaryCacheResultHandle(value, charge));
+  handle.reset(
+      new CompressedSecondaryCacheResultHandle(result_value, result_charge));
   RecordTick(stats, COMPRESSED_SECONDARY_CACHE_HITS);
   return handle;
 }
@@ -171,85 +176,111 @@ bool CompressedSecondaryCache::MaybeInsertDummy(const Slice& key) {
 
 Status CompressedSecondaryCache::InsertInternal(
     const Slice& key, Cache::ObjectPtr value,
-    const Cache::CacheItemHelper* helper, CompressionType type,
+    const Cache::CacheItemHelper* helper, CompressionType from_type,
     CacheTier source) {
-  if (source != CacheTier::kVolatileCompressedTier &&
-      cache_options_.enable_custom_split_merge) {
-    // We don't support custom split/merge for the tiered case
-    return Status::OK();
-  }
+  bool enable_split_merge = cache_options_.enable_custom_split_merge;
+  const Cache::CacheItemHelper* internal_helper = GetHelper(enable_split_merge);
 
-  auto internal_helper = GetHelper(cache_options_.enable_custom_split_merge);
-  char header[20];
-  char* payload = header;
-  payload = EncodeVarint32(payload, static_cast<uint32_t>(type));
-  payload = EncodeVarint32(payload, static_cast<uint32_t>(source));
-  size_t data_size = (*helper->size_cb)(value);
-  char* data_size_ptr = payload;
-  payload = EncodeVarint64(payload, data_size);
+  // TODO: variant of size_cb that also returns a pointer to the data if
+  // already available. Saves an allocation if we keep the compressed version.
+  const size_t data_size_original = (*helper->size_cb)(value);
 
-  size_t header_size = payload - header;
-  size_t total_size = data_size + header_size;
-  CacheAllocationPtr ptr =
-      AllocateBlock(total_size, cache_options_.memory_allocator.get());
-  char* data_ptr = ptr.get() + header_size;
+  // Allocate enough memory for header/tag + original data because (a) we might
+  // not be attempting compression at all, and (b) we might keep the original if
+  // compression is insufficient. But we don't need the length prefix with
+  // enable_split_merge. TODO: be smarter with CacheValueChunk to save an
+  // allocation in the enable_split_merge case.
+  size_t header_size = GetHeaderSize(data_size_original, enable_split_merge);
+  CacheAllocationPtr allocation = AllocateBlock(
+      header_size + data_size_original, cache_options_.memory_allocator.get());
+  char* data_ptr = allocation.get() + header_size;
+  Slice tagged_data(data_ptr - kTagSize, data_size_original + kTagSize);
+  assert(tagged_data.data() >= allocation.get());
 
-  Status s = (*helper->saveto_cb)(value, 0, data_size, data_ptr);
+  Status s = (*helper->saveto_cb)(value, 0, data_size_original, data_ptr);
   if (!s.ok()) {
     return s;
   }
-  Slice val(data_ptr, data_size);
 
-  std::string compressed_val;
-  if (cache_options_.compression_type != kNoCompression &&
-      type == kNoCompression &&
+  std::unique_ptr<char[]> tagged_compressed_data;
+  CompressionType to_type = kNoCompression;
+  if (compressor_ && from_type == kNoCompression &&
       !cache_options_.do_not_compress_roles.Contains(helper->role)) {
-    PERF_COUNTER_ADD(compressed_sec_cache_uncompressed_bytes, data_size);
+    assert(source == CacheTier::kVolatileCompressedTier);
 
-    CompressionType to_type = kNoCompression;
-    s = compressor_->CompressBlock(val, &compressed_val, &to_type,
+    // TODO: consider malloc sizes for max acceptable compressed size
+    // Or maybe max_compressed_bytes_per_kb
+    size_t data_size_compressed = data_size_original - 1;
+    tagged_compressed_data =
+        std::make_unique<char[]>(data_size_compressed + kTagSize);
+    s = compressor_->CompressBlock(Slice(data_ptr, data_size_original),
+                                   tagged_compressed_data.get() + kTagSize,
+                                   &data_size_compressed, &to_type,
                                    nullptr /*working_area*/);
     if (!s.ok()) {
       return s;
     }
-    // TODO: allow values not compressed when there's no size savings?
-    assert(to_type == cache_options_.compression_type);
-    if (to_type != cache_options_.compression_type) {
-      return Status::Corruption("Failed to compress value.");
-    }
-
-    val = Slice(compressed_val);
-    data_size = compressed_val.size();
-    payload = EncodeVarint64(data_size_ptr, data_size);
-    header_size = payload - header;
-    total_size = header_size + data_size;
-    PERF_COUNTER_ADD(compressed_sec_cache_compressed_bytes, data_size);
-
-    if (!cache_options_.enable_custom_split_merge) {
-      ptr = AllocateBlock(total_size, cache_options_.memory_allocator.get());
-      data_ptr = ptr.get() + header_size;
-      memcpy(data_ptr, compressed_val.data(), data_size);
+    PERF_COUNTER_ADD(compressed_sec_cache_uncompressed_bytes,
+                     data_size_original);
+    if (to_type == kNoCompression) {
+      // Compression rejected or otherwise aborted/failed
+      to_type = kNoCompression;
+      tagged_compressed_data.reset();
+      // TODO: consider separate counters for rejected compressions
+      PERF_COUNTER_ADD(compressed_sec_cache_compressed_bytes,
+                       data_size_original);
+    } else {
+      PERF_COUNTER_ADD(compressed_sec_cache_compressed_bytes,
+                       data_size_compressed);
+      if (enable_split_merge) {
+        // Only need tagged_data for copying into CacheValueChunks.
+        tagged_data = Slice(tagged_compressed_data.get(),
+                            data_size_compressed + kTagSize);
+        allocation.reset();
+      } else {
+        // Replace allocation with compressed version, copied from string
+        header_size = GetHeaderSize(data_size_compressed, enable_split_merge);
+        allocation = AllocateBlock(header_size + data_size_compressed,
+                                   cache_options_.memory_allocator.get());
+        data_ptr = allocation.get() + header_size;
+        // Ignore unpopulated tag on tagged_compressed_data; will only be
+        // populated on the new allocation.
+        std::memcpy(data_ptr, tagged_compressed_data.get() + kTagSize,
+                    data_size_compressed);
+        tagged_data =
+            Slice(data_ptr - kTagSize, data_size_compressed + kTagSize);
+        assert(tagged_data.data() >= allocation.get());
+      }
     }
   }
 
   PERF_COUNTER_ADD(compressed_sec_cache_insert_real_count, 1);
-  if (cache_options_.enable_custom_split_merge) {
+
+  // Save the tag fields
+  const_cast<char*>(tagged_data.data())[0] = lossless_cast<char>(source);
+  const_cast<char*>(tagged_data.data())[1] = lossless_cast<char>(
+      source == CacheTier::kVolatileCompressedTier ? to_type : from_type);
+
+  if (enable_split_merge) {
     size_t split_charge{0};
-    CacheValueChunk* value_chunks_head = SplitValueIntoChunks(
-        val, cache_options_.compression_type, split_charge);
-    return cache_->Insert(key, value_chunks_head, internal_helper,
-                          split_charge);
+    CacheValueChunk* value_chunks_head =
+        SplitValueIntoChunks(tagged_data, split_charge);
+    s = cache_->Insert(key, value_chunks_head, internal_helper, split_charge);
+    assert(s.ok());  // LRUCache::Insert() with handle==nullptr always OK
   } else {
+    // Save the size prefix
+    char* ptr = allocation.get();
+    ptr = EncodeVarint64(ptr, tagged_data.size());
+    assert(ptr == tagged_data.data());
 #ifdef ROCKSDB_MALLOC_USABLE_SIZE
-    size_t charge = malloc_usable_size(ptr.get());
+    size_t charge = malloc_usable_size(allocation.get());
 #else
-    size_t charge = total_size;
+    size_t charge = tagged_data.size();
 #endif
-    std::memcpy(ptr.get(), header, header_size);
-    CacheAllocationPtr* buf = new CacheAllocationPtr(std::move(ptr));
-    charge += sizeof(CacheAllocationPtr);
-    return cache_->Insert(key, buf, internal_helper, charge);
+    s = cache_->Insert(key, allocation.release(), internal_helper, charge);
+    assert(s.ok());  // LRUCache::Insert() with handle==nullptr always OK
   }
+  return Status::OK();
 }
 
 Status CompressedSecondaryCache::Insert(const Slice& key,
@@ -271,7 +302,17 @@ Status CompressedSecondaryCache::Insert(const Slice& key,
 Status CompressedSecondaryCache::InsertSaved(
     const Slice& key, const Slice& saved, CompressionType type = kNoCompression,
     CacheTier source = CacheTier::kVolatileTier) {
+  if (source == CacheTier::kVolatileCompressedTier) {
+    // Unexpected, would violate InsertInternal preconditions
+    assert(source != CacheTier::kVolatileCompressedTier);
+    return Status::OK();
+  }
   if (type == kNoCompression) {
+    // Not currently supported (why?)
+    return Status::OK();
+  }
+  if (cache_options_.enable_custom_split_merge) {
+    // We don't support custom split/merge for the tiered case (why?)
     return Status::OK();
   }
 
@@ -291,7 +332,7 @@ Status CompressedSecondaryCache::SetCapacity(size_t capacity) {
   MutexLock l(&capacity_mutex_);
   cache_options_.capacity = capacity;
   cache_->SetCapacity(capacity);
-  disable_cache_ = capacity == 0;
+  disable_cache_.StoreRelaxed(capacity == 0);
   return Status::OK();
 }
 
@@ -321,9 +362,14 @@ std::string CompressedSecondaryCache::GetPrintableOptions() const {
   return ret;
 }
 
+// FIXME: this could use a lot of attention, including:
+// * Use allocator
+// * We shouldn't be worse than non-split; be more pro-actively aware of
+// internal fragmentation
+// * Consider a unified object/chunk structure that may or may not split
+// * Optimize size overhead of chunks
 CompressedSecondaryCache::CacheValueChunk*
 CompressedSecondaryCache::SplitValueIntoChunks(const Slice& value,
-                                               CompressionType compression_type,
                                                size_t& charge) {
   assert(!value.empty());
   const char* src_ptr = value.data();
@@ -344,15 +390,14 @@ CompressedSecondaryCache::SplitValueIntoChunks(const Slice& value,
     // size, or there is no compression.
     if (upper == malloc_bin_sizes_.begin() ||
         upper == malloc_bin_sizes_.end() ||
-        *upper - predicted_chunk_size < malloc_bin_sizes_.front() ||
-        compression_type == kNoCompression) {
+        *upper - predicted_chunk_size < malloc_bin_sizes_.front()) {
       tmp_size = predicted_chunk_size;
     } else {
       tmp_size = *(--upper);
     }
 
     CacheValueChunk* new_chunk =
-        reinterpret_cast<CacheValueChunk*>(new char[tmp_size]);
+        static_cast<CacheValueChunk*>(static_cast<void*>(new char[tmp_size]));
     current_chunk->next = new_chunk;
     current_chunk = current_chunk->next;
     actual_chunk_size = tmp_size - sizeof(CacheValueChunk) + 1;
@@ -367,28 +412,24 @@ CompressedSecondaryCache::SplitValueIntoChunks(const Slice& value,
   return dummy_head.next;
 }
 
-CacheAllocationPtr CompressedSecondaryCache::MergeChunksIntoValue(
-    const void* chunks_head, size_t& charge) {
-  const CacheValueChunk* head =
-      reinterpret_cast<const CacheValueChunk*>(chunks_head);
+std::string CompressedSecondaryCache::MergeChunksIntoValue(
+    const CacheValueChunk* head) {
   const CacheValueChunk* current_chunk = head;
-  charge = 0;
+  size_t total_size = 0;
   while (current_chunk != nullptr) {
-    charge += current_chunk->size;
+    total_size += current_chunk->size;
     current_chunk = current_chunk->next;
   }
 
-  CacheAllocationPtr ptr =
-      AllocateBlock(charge, cache_options_.memory_allocator.get());
+  std::string result;
+  result.reserve(total_size);
   current_chunk = head;
-  size_t pos{0};
   while (current_chunk != nullptr) {
-    memcpy(ptr.get() + pos, current_chunk->data, current_chunk->size);
-    pos += current_chunk->size;
+    result.append(current_chunk->data, current_chunk->size);
     current_chunk = current_chunk->next;
   }
-
-  return ptr;
+  assert(result.size() == total_size);
+  return result;
 }
 
 const Cache::CacheItemHelper* CompressedSecondaryCache::GetHelper(
@@ -402,16 +443,16 @@ const Cache::CacheItemHelper* CompressedSecondaryCache::GetHelper(
             CacheValueChunk* tmp_chunk = chunks_head;
             chunks_head = chunks_head->next;
             tmp_chunk->Free();
-            obj = nullptr;
           }
         }};
     return &kHelper;
   } else {
     static const Cache::CacheItemHelper kHelper{
         CacheEntryRole::kMisc,
-        [](Cache::ObjectPtr obj, MemoryAllocator* /*alloc*/) {
-          delete static_cast<CacheAllocationPtr*>(obj);
-          obj = nullptr;
+        [](Cache::ObjectPtr obj, MemoryAllocator* alloc) {
+          if (obj != nullptr) {
+            CacheAllocationDeleter{alloc}(static_cast<char*>(obj));
+          }
         }};
     return &kHelper;
   }
@@ -422,12 +463,7 @@ size_t CompressedSecondaryCache::TEST_GetCharge(const Slice& key) {
   if (lru_handle == nullptr) {
     return 0;
   }
-
   size_t charge = cache_->GetCharge(lru_handle);
-  if (cache_->Value(lru_handle) != nullptr &&
-      !cache_options_.enable_custom_split_merge) {
-    charge -= 10;
-  }
   cache_->Release(lru_handle, /*erase_if_last_ref=*/false);
   return charge;
 }

cache/compressed_secondary_cache.h

@@ -10,13 +10,12 @@
 #include <memory>
 
 #include "cache/cache_reservation_manager.h"
-#include "cache/lru_cache.h"
 #include "memory/memory_allocator_impl.h"
+#include "rocksdb/advanced_compression.h"
 #include "rocksdb/secondary_cache.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"
-#include "util/compression.h"
-#include "util/mutexlock.h"
+#include "util/atomic.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -124,14 +123,9 @@ class CompressedSecondaryCache : public SecondaryCache {
   // Split value into chunks to better fit into jemalloc bins. The chunks
   // are stored in CacheValueChunk and extra charge is needed for each chunk,
   // so the cache charge is recalculated here.
-  CacheValueChunk* SplitValueIntoChunks(const Slice& value,
-                                        CompressionType compression_type,
-                                        size_t& charge);
+  CacheValueChunk* SplitValueIntoChunks(const Slice& value, size_t& charge);
 
-  // After merging chunks, the extra charge for each chunk is removed, so
-  // the charge is recalculated.
-  CacheAllocationPtr MergeChunksIntoValue(const void* chunks_head,
-                                          size_t& charge);
+  std::string MergeChunksIntoValue(const CacheValueChunk* head);
 
   bool MaybeInsertDummy(const Slice& key);
 
@@ -149,7 +143,7 @@ class CompressedSecondaryCache : public SecondaryCache {
   std::shared_ptr<Decompressor> decompressor_;
   mutable port::Mutex capacity_mutex_;
   std::shared_ptr<ConcurrentCacheReservationManager> cache_res_mgr_;
-  bool disable_cache_;
+  RelaxedAtomic<bool> disable_cache_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

cache/compressed_secondary_cache_test.cc

@@ -24,6 +24,14 @@ namespace ROCKSDB_NAMESPACE {
 using secondary_cache_test_util::GetTestingCacheTypes;
 using secondary_cache_test_util::WithCacheType;
 
+// Read and reset a statistic
+template <typename T>
+T Pop(T& var) {
+  T ret = var;
+  var = T();
+  return ret;
+}
+
 // 16 bytes for HCC compatibility
 const std::string key0 = "____    ____key0";
 const std::string key1 = "____    ____key1";
@@ -51,7 +59,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
 
     Random rnd(301);
     // Insert and Lookup the item k1 for the first time.
-    std::string str1(rnd.RandomString(1000));
+    std::string str1 = test::CompressibleString(&rnd, 0.5, 1000);
     TestItem item1(str1.data(), str1.length());
     // A dummy handle is inserted if the item is inserted for the first time.
     ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false));
@@ -68,7 +76,14 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1);
 
-    ASSERT_GT(comp_sec_cache->TEST_GetCharge(key1), 1000);
+    if (sec_cache_is_compressed) {
+      ASSERT_GT(comp_sec_cache->TEST_GetCharge(key1), str1.length() / 4);
+      ASSERT_LT(comp_sec_cache->TEST_GetCharge(key1), str1.length() * 3 / 4);
+    } else {
+      ASSERT_GE(comp_sec_cache->TEST_GetCharge(key1), str1.length());
+      // NOTE: split-merge is worse (1048 vs. 1024)
+      ASSERT_LE(comp_sec_cache->TEST_GetCharge(key1), 1048U);
+    }
 
     std::unique_ptr<SecondaryCacheResultHandle> handle1_2 =
         sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/true,
@@ -76,10 +91,13 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     ASSERT_NE(handle1_2, nullptr);
     ASSERT_FALSE(kept_in_sec_cache);
     if (sec_cache_is_compressed) {
-      ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
-                1000);
-      ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes,
-                1007);
+      ASSERT_EQ(
+          Pop(get_perf_context()->compressed_sec_cache_uncompressed_bytes),
+          str1.length());
+      ASSERT_LT(get_perf_context()->compressed_sec_cache_compressed_bytes,
+                str1.length() * 3 / 4);
+      ASSERT_GT(Pop(get_perf_context()->compressed_sec_cache_compressed_bytes),
+                str1.length() / 4);
     } else {
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
@@ -97,7 +115,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     ASSERT_EQ(handle1_3, nullptr);
 
     // Insert and Lookup the item k2.
-    std::string str2(rnd.RandomString(1000));
+    std::string str2 = test::CompressibleString(&rnd, 0.5, 1017);
     TestItem item2(str2.data(), str2.length());
     ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2);
@@ -109,10 +127,13 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2);
     if (sec_cache_is_compressed) {
-      ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
-                2000);
-      ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes,
-                2014);
+      ASSERT_EQ(
+          Pop(get_perf_context()->compressed_sec_cache_uncompressed_bytes),
+          str2.length());
+      ASSERT_LT(get_perf_context()->compressed_sec_cache_compressed_bytes,
+                str2.length() * 3 / 4);
+      ASSERT_GT(Pop(get_perf_context()->compressed_sec_cache_compressed_bytes),
+                str2.length() / 4);
     } else {
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
@@ -126,9 +147,48 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     ASSERT_NE(val2, nullptr);
     ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0);
 
+    // Release handles
     std::vector<SecondaryCacheResultHandle*> handles = {handle1_2.get(),
                                                         handle2_2.get()};
     sec_cache->WaitAll(handles);
+    handle1_2.reset();
+    handle2_2.reset();
+
+    // Insert and Lookup a non-compressible item k3.
+    std::string str3 = rnd.RandomBinaryString(480);
+    TestItem item3(str3.data(), str3.length());
+    ASSERT_OK(sec_cache->Insert(key3, &item3, GetHelper(), false));
+    ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 3);
+    std::unique_ptr<SecondaryCacheResultHandle> handle3_1 =
+        sec_cache->Lookup(key3, GetHelper(), this, true, /*advise_erase=*/false,
+                          /*stats=*/nullptr, kept_in_sec_cache);
+    ASSERT_EQ(handle3_1, nullptr);
+
+    ASSERT_OK(sec_cache->Insert(key3, &item3, GetHelper(), false));
+    ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 3);
+    if (sec_cache_is_compressed) {
+      // TODO: consider a compression rejected stat?
+      ASSERT_EQ(
+          Pop(get_perf_context()->compressed_sec_cache_uncompressed_bytes),
+          str3.length());
+      ASSERT_EQ(Pop(get_perf_context()->compressed_sec_cache_compressed_bytes),
+                str3.length());
+    } else {
+      ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
+      ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
+    }
+
+    std::unique_ptr<SecondaryCacheResultHandle> handle3_2 =
+        sec_cache->Lookup(key3, GetHelper(), this, true, /*advise_erase=*/false,
+                          /*stats=*/nullptr, kept_in_sec_cache);
+    ASSERT_NE(handle3_2, nullptr);
+    std::unique_ptr<TestItem> val3 =
+        std::unique_ptr<TestItem>(static_cast<TestItem*>(handle3_2->Value()));
+    ASSERT_NE(val3, nullptr);
+    ASSERT_EQ(memcmp(val3->Buf(), item3.Buf(), item3.Size()), 0);
+
+    EXPECT_GE(comp_sec_cache->TEST_GetCharge(key3), str3.length());
+    EXPECT_LE(comp_sec_cache->TEST_GetCharge(key3), 512);
 
     sec_cache.reset();
   }
@@ -178,8 +238,9 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
       secondary_cache_opts.compression_type = CompressionType::kNoCompression;
     }
 
-    secondary_cache_opts.capacity = 1100;
+    secondary_cache_opts.capacity = 1400;
     secondary_cache_opts.num_shard_bits = 0;
+    secondary_cache_opts.strict_capacity_limit = true;
     std::shared_ptr<SecondaryCache> sec_cache =
         NewCompressedSecondaryCache(secondary_cache_opts);
 
@@ -193,7 +254,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     ASSERT_OK(sec_cache->Insert(key1, &item1, GetHelper(), false));
 
     // Insert and Lookup the second item.
-    std::string str2(rnd.RandomString(200));
+    std::string str2(rnd.RandomString(500));
     TestItem item2(str2.data(), str2.length());
     // Insert a dummy handle, k1 is not evicted.
     ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false));
@@ -201,16 +262,23 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     std::unique_ptr<SecondaryCacheResultHandle> handle1 =
         sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false,
                           /*stats=*/nullptr, kept_in_sec_cache);
-    ASSERT_EQ(handle1, nullptr);
+    ASSERT_NE(handle1, nullptr);
+    std::unique_ptr<TestItem> val1{static_cast<TestItem*>(handle1->Value())};
+    ASSERT_NE(val1, nullptr);
+    ASSERT_EQ(val1->ToString(), str1);
+    handle1.reset();
 
     // Insert k2 and k1 is evicted.
     ASSERT_OK(sec_cache->Insert(key2, &item2, GetHelper(), false));
+    handle1 =
+        sec_cache->Lookup(key1, GetHelper(), this, true, /*advise_erase=*/false,
+                          /*stats=*/nullptr, kept_in_sec_cache);
+    ASSERT_EQ(handle1, nullptr);
     std::unique_ptr<SecondaryCacheResultHandle> handle2 =
         sec_cache->Lookup(key2, GetHelper(), this, true, /*advise_erase=*/false,
                           /*stats=*/nullptr, kept_in_sec_cache);
     ASSERT_NE(handle2, nullptr);
-    std::unique_ptr<TestItem> val2 =
-        std::unique_ptr<TestItem>(static_cast<TestItem*>(handle2->Value()));
+    std::unique_ptr<TestItem> val2{static_cast<TestItem*>(handle2->Value())};
     ASSERT_NE(val2, nullptr);
     ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0);
 
@@ -232,7 +300,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     // Save Fails.
     std::string str3 = rnd.RandomString(10);
     TestItem item3(str3.data(), str3.length());
-    // The Status is OK because a dummy handle is inserted.
+    // The first Status is OK because a dummy handle is inserted.
     ASSERT_OK(sec_cache->Insert(key3, &item3, GetHelperFail(), false));
     ASSERT_NOK(sec_cache->Insert(key3, &item3, GetHelperFail(), false));
 
@@ -265,11 +333,11 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
 
     get_perf_context()->Reset();
     Random rnd(301);
-    std::string str1 = rnd.RandomString(1001);
+    std::string str1 = test::CompressibleString(&rnd, 0.5, 1001);
     auto item1_1 = new TestItem(str1.data(), str1.length());
     ASSERT_OK(cache->Insert(key1, item1_1, GetHelper(), str1.length()));
 
-    std::string str2 = rnd.RandomString(1012);
+    std::string str2 = test::CompressibleString(&rnd, 0.5, 1012);
     auto item2_1 = new TestItem(str2.data(), str2.length());
     // After this Insert, primary cache contains k2 and secondary cache contains
     // k1's dummy item.
@@ -278,7 +346,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
 
-    std::string str3 = rnd.RandomString(1024);
+    std::string str3 = test::CompressibleString(&rnd, 0.5, 1024);
     auto item3_1 = new TestItem(str3.data(), str3.length());
     // After this Insert, primary cache contains k3 and secondary cache contains
     // k1's dummy item and k2's dummy item.
@@ -297,10 +365,13 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     ASSERT_OK(cache->Insert(key2, item2_2, GetHelper(), str2.length()));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1);
     if (sec_cache_is_compressed) {
-      ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
+      ASSERT_EQ(
+          Pop(get_perf_context()->compressed_sec_cache_uncompressed_bytes),
+          str1.length());
+      ASSERT_LT(get_perf_context()->compressed_sec_cache_compressed_bytes,
                 str1.length());
-      ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes,
-                1008);
+      ASSERT_GT(Pop(get_perf_context()->compressed_sec_cache_compressed_bytes),
+                str1.length() / 10);
     } else {
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
@@ -312,10 +383,13 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     ASSERT_OK(cache->Insert(key3, item3_2, GetHelper(), str3.length()));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2);
     if (sec_cache_is_compressed) {
-      ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
-                str1.length() + str2.length());
-      ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes,
-                2027);
+      ASSERT_EQ(
+          Pop(get_perf_context()->compressed_sec_cache_uncompressed_bytes),
+          str2.length());
+      ASSERT_LT(get_perf_context()->compressed_sec_cache_compressed_bytes,
+                str2.length());
+      ASSERT_GT(Pop(get_perf_context()->compressed_sec_cache_compressed_bytes),
+                str2.length() / 10);
     } else {
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
@@ -641,8 +715,7 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     size_t str_size{8500};
     std::string str = rnd.RandomString(static_cast<int>(str_size));
     size_t charge{0};
-    CacheValueChunk* chunks_head =
-        sec_cache->SplitValueIntoChunks(str, kLZ4Compression, charge);
+    CacheValueChunk* chunks_head = sec_cache->SplitValueIntoChunks(str, charge);
     ASSERT_EQ(charge, str_size + 3 * (sizeof(CacheValueChunk) - 1));
 
     CacheValueChunk* current_chunk = chunks_head;
@@ -688,12 +761,9 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     std::unique_ptr<CompressedSecondaryCache> sec_cache =
         std::make_unique<CompressedSecondaryCache>(
             CompressedSecondaryCacheOptions(1000, 0, true, 0.5, 0.0));
-    size_t charge{0};
-    CacheAllocationPtr value =
-        sec_cache->MergeChunksIntoValue(chunks_head, charge);
-    ASSERT_EQ(charge, size1 + size2 + size3);
-    std::string value_str{value.get(), charge};
-    ASSERT_EQ(strcmp(value_str.data(), str.data()), 0);
+    std::string value_str = sec_cache->MergeChunksIntoValue(chunks_head);
+    ASSERT_EQ(value_str.size(), size1 + size2 + size3);
+    ASSERT_EQ(value_str, str);
 
     while (chunks_head != nullptr) {
       CacheValueChunk* tmp_chunk = chunks_head;
@@ -725,15 +795,12 @@ class CompressedSecondaryCacheTestBase : public testing::Test,
     size_t str_size{8500};
     std::string str = rnd.RandomString(static_cast<int>(str_size));
     size_t charge{0};
-    CacheValueChunk* chunks_head =
-        sec_cache->SplitValueIntoChunks(str, kLZ4Compression, charge);
+    CacheValueChunk* chunks_head = sec_cache->SplitValueIntoChunks(str, charge);
     ASSERT_EQ(charge, str_size + 3 * (sizeof(CacheValueChunk) - 1));
 
-    CacheAllocationPtr value =
-        sec_cache->MergeChunksIntoValue(chunks_head, charge);
-    ASSERT_EQ(charge, str_size);
-    std::string value_str{value.get(), charge};
-    ASSERT_EQ(strcmp(value_str.data(), str.data()), 0);
+    std::string value_str = sec_cache->MergeChunksIntoValue(chunks_head);
+    ASSERT_EQ(value_str.size(), str_size);
+    ASSERT_EQ(value_str, str);
 
     sec_cache->GetHelper(true)->del_cb(chunks_head, /*alloc*/ nullptr);
   }
@@ -896,8 +963,8 @@ TEST_P(CompressedSecondaryCacheTestWithCompressionParam, EntryRoles) {
 
   std::shared_ptr<SecondaryCache> sec_cache = NewCompressedSecondaryCache(opts);
 
-  // Fixed seed to ensure consistent compressibility (doesn't compress)
-  std::string junk(Random(301).RandomString(1000));
+  Random rnd(301);
+  std::string junk = test::CompressibleString(&rnd, 0.5, 1000);
 
   for (uint32_t i = 0; i < kNumCacheEntryRoles; ++i) {
     CacheEntryRole role = static_cast<CacheEntryRole>(i);
@@ -930,9 +997,11 @@ TEST_P(CompressedSecondaryCacheTestWithCompressionParam, EntryRoles) {
         sec_cache_is_compressed_ && !do_not_compress.Contains(role);
     if (compressed) {
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
-                1000);
-      ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes,
-                1007);
+                junk.length());
+      ASSERT_LT(get_perf_context()->compressed_sec_cache_compressed_bytes,
+                junk.length() * 3 / 4);
+      ASSERT_GT(get_perf_context()->compressed_sec_cache_compressed_bytes,
+                junk.length() / 4);
     } else {
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);

cache/secondary_cache_adapter.cc

@@ -121,16 +121,13 @@ CacheWithSecondaryAdapter::~CacheWithSecondaryAdapter() {
     assert(s.ok());
     assert(placeholder_usage_ == 0);
     assert(reserved_usage_ == 0);
-    bool pri_cache_res_mismatch =
-        pri_cache_res_->GetTotalMemoryUsed() != sec_capacity;
-    if (pri_cache_res_mismatch) {
-      fprintf(stderr,
+    if (pri_cache_res_->GetTotalMemoryUsed() != sec_capacity) {
+      fprintf(stdout,
               "~CacheWithSecondaryAdapter: Primary cache reservation: "
               "%zu, Secondary cache capacity: %zu, "
               "Secondary cache reserved: %zu\n",
               pri_cache_res_->GetTotalMemoryUsed(), sec_capacity,
               sec_reserved_);
-      assert(!pri_cache_res_mismatch);
     }
   }
 #endif  // NDEBUG

db/arena_wrapped_db_iter.h

@@ -98,7 +98,7 @@ class ArenaWrappedDBIter : public Iterator {
 
   bool PrepareValue() override { return db_iter_->PrepareValue(); }
 
-  void Prepare(const std::vector<ScanOptions>& scan_opts) override {
+  void Prepare(const MultiScanArgs& scan_opts) override {
     db_iter_->Prepare(scan_opts);
   }
 

db/builder.cc

@@ -201,8 +201,7 @@ Status BuildTable(
     CompactionIterator c_iter(
         iter, ucmp, &merge, kMaxSequenceNumber, &snapshots, earliest_snapshot,
         earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env,
-        ShouldReportDetailedTime(env, ioptions.stats),
-        true /* internal key corruption is not ok */, range_del_agg.get(),
+        ShouldReportDetailedTime(env, ioptions.stats), range_del_agg.get(),
         blob_file_builder.get(), ioptions.allow_data_in_errors,
         ioptions.enforce_single_del_contracts,
         /*manual_compaction_canceled=*/kManualCompactionCanceledFalse,

db/c.cc

@@ -3639,6 +3639,11 @@ void rocksdb_options_set_write_buffer_manager(
   opt->rep.write_buffer_manager = wbm->rep;
 }
 
+void rocksdb_options_set_sst_file_manager(rocksdb_options_t* opt,
+                                          rocksdb_sst_file_manager_t* sfm) {
+  opt->rep.sst_file_manager = sfm->rep;
+}
+
 size_t rocksdb_options_get_write_buffer_size(rocksdb_options_t* opt) {
   return opt->rep.write_buffer_size;
 }
@@ -4746,6 +4751,15 @@ void rocksdb_options_add_compact_on_deletion_collector_factory_del_ratio(
   opt->rep.table_properties_collector_factories.emplace_back(compact_on_del);
 }
 
+void rocksdb_options_add_compact_on_deletion_collector_factory_min_file_size(
+    rocksdb_options_t* opt, size_t window_size, size_t num_dels_trigger,
+    double deletion_ratio, uint64_t min_file_size) {
+  std::shared_ptr<ROCKSDB_NAMESPACE::TablePropertiesCollectorFactory>
+      compact_on_del = NewCompactOnDeletionCollectorFactory(
+          window_size, num_dels_trigger, deletion_ratio, min_file_size);
+  opt->rep.table_properties_collector_factories.emplace_back(compact_on_del);
+}
+
 void rocksdb_set_perf_level(int v) {
   PerfLevel level = static_cast<PerfLevel>(v);
   SetPerfLevel(level);

db/column_family.cc

@@ -280,7 +280,8 @@ ColumnFamilyOptions SanitizeCfOptions(const ImmutableDBOptions& db_options,
   }
 
   if (result.compaction_style == kCompactionStyleUniversal &&
-      db_options.allow_ingest_behind && result.num_levels < 3) {
+      (db_options.allow_ingest_behind || result.cf_allow_ingest_behind) &&
+      result.num_levels < 3) {
     result.num_levels = 3;
   }
 
@@ -1331,7 +1332,7 @@ Compaction* ColumnFamilyData::CompactRange(
     const InternalKey* begin, const InternalKey* end,
     InternalKey** compaction_end, bool* conflict,
     uint64_t max_file_num_to_ignore, const std::string& trim_ts) {
-  auto* result = compaction_picker_->CompactRange(
+  auto* result = compaction_picker_->PickCompactionForCompactRange(
       GetName(), mutable_cf_options, mutable_db_options,
       current_->storage_info(), input_level, output_level,
       compact_range_options, begin, end, compaction_end, conflict,

db/column_family.h

@@ -600,6 +600,11 @@ class ColumnFamilyData {
     return (mem_->IsEmpty() ? 0 : 1) + imm_.NumNotFlushed();
   }
 
+  // thread-safe, DB mutex not needed.
+  bool AllowIngestBehind() const {
+    return ioptions_.cf_allow_ingest_behind || ioptions_.allow_ingest_behind;
+  }
+
  private:
   friend class ColumnFamilySet;
   ColumnFamilyData(

db/compact_files_test.cc

@@ -441,6 +441,10 @@ TEST_F(CompactFilesTest, SentinelCompressionType) {
 }
 
 TEST_F(CompactFilesTest, CompressionWithBlockAlign) {
+  if (!Snappy_Supported()) {
+    ROCKSDB_GTEST_SKIP("Test requires Snappy support");
+    return;
+  }
   Options options;
   options.compression = CompressionType::kNoCompression;
   options.create_if_missing = true;

db/compaction/compaction.cc

@@ -647,6 +647,8 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
     return true;
   } else if (output_level_ != 0 &&
              cfd_->ioptions().compaction_style == kCompactionStyleLevel) {
+    // TODO: apply the optimization here to other compaction styles and
+    // compaction/flush to L0.
     // Maybe use binary search to find right entry instead of linear search?
     const Comparator* user_cmp = cfd_->user_comparator();
     for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {

db/compaction/compaction_iterator.cc

@@ -28,7 +28,7 @@ CompactionIterator::CompactionIterator(
     SequenceNumber earliest_snapshot,
     SequenceNumber earliest_write_conflict_snapshot,
     SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker,
-    Env* env, bool report_detailed_time, bool expect_valid_internal_key,
+    Env* env, bool report_detailed_time,
     CompactionRangeDelAggregator* range_del_agg,
     BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
     bool enforce_single_del_contracts,
@@ -42,8 +42,8 @@ CompactionIterator::CompactionIterator(
     : CompactionIterator(
           input, cmp, merge_helper, last_sequence, snapshots, earliest_snapshot,
           earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env,
-          report_detailed_time, expect_valid_internal_key, range_del_agg,
-          blob_file_builder, allow_data_in_errors, enforce_single_del_contracts,
+          report_detailed_time, range_del_agg, blob_file_builder,
+          allow_data_in_errors, enforce_single_del_contracts,
           manual_compaction_canceled,
           compaction ? std::make_unique<RealCompaction>(compaction) : nullptr,
           must_count_input_entries, compaction_filter, shutting_down, info_log,
@@ -55,7 +55,7 @@ CompactionIterator::CompactionIterator(
     SequenceNumber earliest_snapshot,
     SequenceNumber earliest_write_conflict_snapshot,
     SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker,
-    Env* env, bool report_detailed_time, bool expect_valid_internal_key,
+    Env* env, bool report_detailed_time,
     CompactionRangeDelAggregator* range_del_agg,
     BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
     bool enforce_single_del_contracts,
@@ -76,16 +76,14 @@ CompactionIterator::CompactionIterator(
       env_(env),
       clock_(env_->GetSystemClock().get()),
       report_detailed_time_(report_detailed_time),
-      expect_valid_internal_key_(expect_valid_internal_key),
       range_del_agg_(range_del_agg),
       blob_file_builder_(blob_file_builder),
       compaction_(std::move(compaction)),
       compaction_filter_(compaction_filter),
       shutting_down_(shutting_down),
       manual_compaction_canceled_(manual_compaction_canceled),
-      bottommost_level_(!compaction_ ? false
-                                     : compaction_->bottommost_level() &&
-                                           !compaction_->allow_ingest_behind()),
+      bottommost_level_(compaction_ && compaction_->bottommost_level() &&
+                        !compaction_->allow_ingest_behind()),
       // snapshots_ cannot be nullptr, but we will assert later in the body of
       // the constructor.
       visible_at_tip_(snapshots_ ? snapshots_->empty() : false),
@@ -161,6 +159,7 @@ void CompactionIterator::Next() {
       // MergeUntil stops when it encounters a corrupt key and does not
       // include them in the result, so we expect the keys here to be valid.
       if (!s.ok()) {
+        // FIXME: should fail compaction after this fatal logging.
         ROCKS_LOG_FATAL(
             info_log_, "Invalid ikey %s in compaction. %s",
             allow_data_in_errors_ ? key_.ToString(true).c_str() : "hidden",
@@ -464,18 +463,9 @@ void CompactionIterator::NextFromInput() {
     if (!pik_status.ok()) {
       iter_stats_.num_input_corrupt_records++;
 
-      // If `expect_valid_internal_key_` is false, return the corrupted key
-      // and let the caller decide what to do with it.
-      if (expect_valid_internal_key_) {
-        status_ = pik_status;
-        return;
-      }
-      key_ = current_key_.SetInternalKey(key_);
-      has_current_user_key_ = false;
-      current_user_key_sequence_ = kMaxSequenceNumber;
-      current_user_key_snapshot_ = 0;
-      validity_info_.SetValid(ValidContext::kParseKeyError);
-      break;
+      // Always fail compaction when encountering corrupted internal keys
+      status_ = pik_status;
+      return;
     }
     TEST_SYNC_POINT_CALLBACK("CompactionIterator:ProcessKV", &ikey_);
     if (is_range_del_) {
@@ -642,7 +632,8 @@ void CompactionIterator::NextFromInput() {
     } else if (ikey_.type == kTypeSingleDeletion) {
       // We can compact out a SingleDelete if:
       // 1) We encounter the corresponding PUT -OR- we know that this key
-      //    doesn't appear past this output level
+      //    doesn't appear past this output level and  we are not in
+      //    ingest_behind mode.
       // =AND=
       // 2) We've already returned a record in this snapshot -OR-
       //    there are no earlier earliest_write_conflict_snapshot.
@@ -731,6 +722,8 @@ void CompactionIterator::NextFromInput() {
             "CompactionIterator::NextFromInput:SingleDelete:1",
             const_cast<Compaction*>(c));
         if (last_key_seq_zeroed_) {
+          // Drop SD and the next key since they are both in the last
+          // snapshot (since last key has seqno zeroed).
           ++iter_stats_.num_record_drop_hidden;
           ++iter_stats_.num_record_drop_obsolete;
           assert(bottommost_level_);
@@ -841,7 +834,7 @@ void CompactionIterator::NextFromInput() {
         // iteration. If the next key is corrupt, we return before the
         // comparison, so the value of has_current_user_key does not matter.
         has_current_user_key_ = false;
-        if (compaction_ != nullptr &&
+        if (compaction_ != nullptr && !compaction_->allow_ingest_behind() &&
             DefinitelyInSnapshot(ikey_.sequence, earliest_snapshot_) &&
             compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
                                                        &level_ptrs_) &&
@@ -854,6 +847,9 @@ void CompactionIterator::NextFromInput() {
             ++iter_stats_.num_optimized_del_drop_obsolete;
           }
         } else if (last_key_seq_zeroed_) {
+          // Sequence number zeroing requires bottommost_level_, which is
+          // false with ingest_behind.
+          assert(!compaction_->allow_ingest_behind());
           // Skip.
           ++iter_stats_.num_record_drop_hidden;
           ++iter_stats_.num_record_drop_obsolete;
@@ -870,6 +866,7 @@ void CompactionIterator::NextFromInput() {
     } else if (last_sequence != kMaxSequenceNumber &&
                (last_snapshot == current_user_key_snapshot_ ||
                 last_snapshot < current_user_key_snapshot_)) {
+      // rule (A):
       // If the earliest snapshot is which this key is visible in
       // is the same as the visibility of a previous instance of the
       // same key, then this kv is not visible in any snapshot.
@@ -878,6 +875,15 @@ void CompactionIterator::NextFromInput() {
       // Note: Dropping this key will not affect TransactionDB write-conflict
       // checking since there has already been a record returned for this key
       // in this snapshot.
+      // When ingest_behind is enabled, it's ok that we drop an overwritten
+      // Delete here. The overwritting key still covers whatever that will be
+      // ingested. Note that we will not drop SingleDelete here as SingleDelte
+      // is handled entirely in its own if clause. This is important, see
+      // example: from new to old: SingleDelete_1, PUT_1, SingleDelete_2, PUT_2,
+      // where all operations are on the same key and PUT_2 is ingested with
+      // ingest_behind=true. If SingleDelete_2 is dropped due to being compacted
+      // together with PUT_1, and then PUT_1 is compacted away together with
+      // SingleDelete_1, PUT_2 can incorrectly becomes visible.
       if (last_sequence < current_user_key_sequence_) {
         ROCKS_LOG_FATAL(info_log_,
                         "key %s, last_sequence (%" PRIu64
@@ -887,12 +893,13 @@ void CompactionIterator::NextFromInput() {
         assert(false);
       }
 
-      ++iter_stats_.num_record_drop_hidden;  // rule (A)
+      ++iter_stats_.num_record_drop_hidden;
       AdvanceInputIter();
     } else if (compaction_ != nullptr &&
                (ikey_.type == kTypeDeletion ||
                 (ikey_.type == kTypeDeletionWithTimestamp &&
                  cmp_with_history_ts_low_ < 0)) &&
+               !compaction_->allow_ingest_behind() &&
                DefinitelyInSnapshot(ikey_.sequence, earliest_snapshot_) &&
                compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
                                                           &level_ptrs_)) {
@@ -928,11 +935,13 @@ void CompactionIterator::NextFromInput() {
                 (ikey_.type == kTypeDeletionWithTimestamp &&
                  cmp_with_history_ts_low_ < 0)) &&
                bottommost_level_) {
+      assert(compaction_);
+      assert(!compaction_->allow_ingest_behind());  // bottommost_level_ is true
       // Handle the case where we have a delete key at the bottom most level
       // We can skip outputting the key iff there are no subsequent puts for
       // this key
-      assert(!compaction_ || compaction_->KeyNotExistsBeyondOutputLevel(
-                                 ikey_.user_key, &level_ptrs_));
+      assert(compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
+                                                        &level_ptrs_));
       ParsedInternalKey next_ikey;
       AdvanceInputIter();
 #ifndef NDEBUG
@@ -974,6 +983,12 @@ void CompactionIterator::NextFromInput() {
                 (compaction_ != nullptr &&
                  compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
                                                             &level_ptrs_)))) {
+      // FIXME: it's possible that we are setting sequence number to 0 as
+      // preferred sequence number here. If cf_ingest_behind is enabled, this
+      // may fail ingestions since they expect all keys above the last level
+      // to have non-zero sequence number. We should probably not allow seqno
+      // zeroing here.
+      //
       // This section that attempts to swap preferred sequence number will not
       // be invoked if this is a CompactionIterator created for flush, since
       // `compaction_` will be nullptr and it's not bottommost either.
@@ -1274,11 +1289,11 @@ void CompactionIterator::PrepareOutput() {
     //
     // Can we do the same for levels above bottom level as long as
     // KeyNotExistsBeyondOutputLevel() return true?
-    if (Valid() && compaction_ != nullptr &&
-        !compaction_->allow_ingest_behind() && bottommost_level_ &&
+    if (Valid() && bottommost_level_ &&
         DefinitelyInSnapshot(ikey_.sequence, earliest_snapshot_) &&
         ikey_.type != kTypeMerge && current_key_committed_ &&
         ikey_.sequence <= preserve_seqno_after_ && !is_range_del_) {
+      assert(compaction_ != nullptr && !compaction_->allow_ingest_behind());
       if (ikey_.type == kTypeDeletion ||
           (ikey_.type == kTypeSingleDeletion && timestamp_size_ == 0)) {
         ROCKS_LOG_FATAL(

db/compaction/compaction_iterator.h

@@ -145,7 +145,8 @@ class CompactionIterator {
     }
 
     bool allow_ingest_behind() const override {
-      return compaction_->immutable_options().allow_ingest_behind;
+      return compaction_->immutable_options().cf_allow_ingest_behind ||
+             compaction_->immutable_options().allow_ingest_behind;
     }
 
     bool allow_mmap_reads() const override {
@@ -192,7 +193,7 @@ class CompactionIterator {
       SequenceNumber earliest_snapshot,
       SequenceNumber earliest_write_conflict_snapshot,
       SequenceNumber job_snapshot, const SnapshotChecker* snapshot_checker,
-      Env* env, bool report_detailed_time, bool expect_valid_internal_key,
+      Env* env, bool report_detailed_time,
       CompactionRangeDelAggregator* range_del_agg,
       BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
       bool enforce_single_del_contracts,
@@ -212,7 +213,7 @@ class CompactionIterator {
                      SequenceNumber earliest_write_conflict_snapshot,
                      SequenceNumber job_snapshot,
                      const SnapshotChecker* snapshot_checker, Env* env,
-                     bool report_detailed_time, bool expect_valid_internal_key,
+                     bool report_detailed_time,
                      CompactionRangeDelAggregator* range_del_agg,
                      BlobFileBuilder* blob_file_builder,
                      bool allow_data_in_errors,
@@ -347,7 +348,6 @@ class CompactionIterator {
   Env* env_;
   SystemClock* clock_;
   const bool report_detailed_time_;
-  const bool expect_valid_internal_key_;
   CompactionRangeDelAggregator* range_del_agg_;
   BlobFileBuilder* blob_file_builder_;
   std::unique_ptr<CompactionProxy> compaction_;
@@ -417,13 +417,15 @@ class CompactionIterator {
   // NextFromInput()).
   ParsedInternalKey ikey_;
 
-  // Stores whether ikey_.user_key is valid. If set to false, the user key is
-  // not compared against the current key in the underlying iterator.
+  // Stores whether current_user_key_ is valid. If so, current_user_key_
+  // stores the user key of the last key seen by the iterator.
+  // If false, treat the next key to read as a new user key.
   bool has_current_user_key_ = false;
   // If false, the iterator holds a copy of the current compaction iterator
   // output (or current key in the underlying iterator during NextFromInput()).
   bool at_next_ = false;
 
+  // A copy of the current internal key.
   IterKey current_key_;
   Slice current_user_key_;
   std::string curr_ts_;
@@ -433,8 +435,9 @@ class CompactionIterator {
   // True if the iterator has already returned a record for the current key.
   bool has_outputted_key_ = false;
 
-  // truncated the value of the next key and output it without applying any
-  // compaction rules.  This is used for outputting a put after a single delete.
+  // Truncate the value of the next key and output it without applying any
+  // compaction rules. This is an optimization for outputting a put after
+  // a single delete. See more in `NextFromInput()` under Optimization 3.
   bool clear_and_output_next_key_ = false;
 
   MergeOutputIterator merge_out_iter_;

db/compaction/compaction_iterator_test.cc

@@ -294,7 +294,7 @@ class CompactionIteratorTest : public testing::TestWithParam<bool> {
         snapshots_.empty() ? kMaxSequenceNumber : snapshots_.at(0),
         earliest_write_conflict_snapshot, kMaxSequenceNumber,
         snapshot_checker_.get(), Env::Default(),
-        false /* report_detailed_time */, false, range_del_agg_.get(),
+        false /* report_detailed_time */, range_del_agg_.get(),
         nullptr /* blob_file_builder */, true /*allow_data_in_errors*/,
         true /*enforce_single_del_contracts*/,
         /*manual_compaction_canceled=*/kManualCompactionCanceledFalse_,
@@ -374,8 +374,7 @@ TEST_P(CompactionIteratorTest, EmptyResult) {
   ASSERT_FALSE(c_iter_->Valid());
 }
 
-// If there is a corruption after a single deletion, the corrupted key should
-// be preserved.
+// If there is a corruption after a single deletion, the compaction should fail.
 TEST_P(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
   InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion),
                  test::KeyStr("a", 3, kTypeValue, true),
@@ -386,14 +385,10 @@ TEST_P(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
   ASSERT_EQ(test::KeyStr("a", 5, kTypeSingleDeletion),
             c_iter_->key().ToString());
   c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("a", 3, kTypeValue, true), c_iter_->key().ToString());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("b", 10, kTypeValue), c_iter_->key().ToString());
-  c_iter_->Next();
-  ASSERT_OK(c_iter_->status());
+  // The iterator should now fail when encountering the corrupted key
   ASSERT_FALSE(c_iter_->Valid());
+  ASSERT_FALSE(c_iter_->status().ok());
+  ASSERT_TRUE(c_iter_->status().IsCorruption());
 }
 
 // Tests compatibility of TimedPut and SingleDelete. TimedPut should act as if

db/compaction/compaction_job.cc

@@ -668,16 +668,17 @@ void CompactionJob::GenSubcompactionBoundaries() {
                extra_num_subcompaction_threads_reserved_));
 }
 
-Status CompactionJob::Run() {
+void CompactionJob::InitializeCompactionRun() {
   AutoThreadOperationStageUpdater stage_updater(
       ThreadStatus::STAGE_COMPACTION_RUN);
   TEST_SYNC_POINT("CompactionJob::Run():Start");
   log_buffer_->FlushBufferToLog();
   LogCompaction();
+}
 
+void CompactionJob::RunSubcompactions() {
   const size_t num_threads = compact_->sub_compact_states.size();
   assert(num_threads > 0);
-  const uint64_t start_micros = db_options_.clock->NowMicros();
   compact_->compaction->GetOrInitInputTableProperties();
 
   // Launch a thread for each of subcompactions 1...num_threads-1
@@ -696,25 +697,43 @@ Status CompactionJob::Run() {
   for (auto& thread : thread_pool) {
     thread.join();
   }
+  RemoveEmptyOutputs();
 
+  ReleaseSubcompactionResources();
+  TEST_SYNC_POINT("CompactionJob::ReleaseSubcompactionResources");
+}
+
+void CompactionJob::UpdateTimingStats(uint64_t start_micros) {
   internal_stats_.SetMicros(db_options_.clock->NowMicros() - start_micros);
 
   for (auto& state : compact_->sub_compact_states) {
     internal_stats_.AddCpuMicros(state.compaction_job_stats.cpu_micros);
-    state.RemoveLastEmptyOutput();
   }
 
   RecordTimeToHistogram(stats_, COMPACTION_TIME,
                         internal_stats_.output_level_stats.micros);
   RecordTimeToHistogram(stats_, COMPACTION_CPU_TIME,
                         internal_stats_.output_level_stats.cpu_micros);
+}
 
-  TEST_SYNC_POINT("CompactionJob::Run:BeforeVerify");
+void CompactionJob::RemoveEmptyOutputs() {
+  for (auto& state : compact_->sub_compact_states) {
+    state.RemoveLastEmptyOutput();
+  }
+}
 
-  // Check if any thread encountered an error during execution
+bool CompactionJob::HasNewBlobFiles() const {
+  for (const auto& state : compact_->sub_compact_states) {
+    if (state.Current().HasBlobFileAdditions()) {
+      return true;
+    }
+  }
+  return false;
+}
+
+Status CompactionJob::CollectSubcompactionErrors() {
   Status status;
   IOStatus io_s;
-  bool wrote_new_blob_files = false;
 
   for (const auto& state : compact_->sub_compact_states) {
     if (!state.status.ok()) {
@@ -722,125 +741,131 @@ Status CompactionJob::Run() {
       io_s = state.io_status;
       break;
     }
-
-    if (state.Current().HasBlobFileAdditions()) {
-      wrote_new_blob_files = true;
-    }
   }
 
   if (io_status_.ok()) {
     io_status_ = io_s;
   }
-  if (status.ok()) {
-    constexpr IODebugContext* dbg = nullptr;
 
-    if (output_directory_) {
-      io_s = output_directory_->FsyncWithDirOptions(
-          IOOptions(), dbg,
-          DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced));
-    }
+  return status;
+}
 
-    if (io_s.ok() && wrote_new_blob_files && blob_output_directory_ &&
-        blob_output_directory_ != output_directory_) {
-      io_s = blob_output_directory_->FsyncWithDirOptions(
-          IOOptions(), dbg,
-          DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced));
-    }
+Status CompactionJob::SyncOutputDirectories() {
+  Status status;
+  IOStatus io_s;
+  constexpr IODebugContext* dbg = nullptr;
+  const bool wrote_new_blob_files = HasNewBlobFiles();
+  if (output_directory_) {
+    io_s = output_directory_->FsyncWithDirOptions(
+        IOOptions(), dbg,
+        DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced));
   }
+
+  if (io_s.ok() && wrote_new_blob_files && blob_output_directory_ &&
+      blob_output_directory_ != output_directory_) {
+    io_s = blob_output_directory_->FsyncWithDirOptions(
+        IOOptions(), dbg,
+        DirFsyncOptions(DirFsyncOptions::FsyncReason::kNewFileSynced));
+  }
+
   if (io_status_.ok()) {
     io_status_ = io_s;
   }
   if (status.ok()) {
     status = io_s;
   }
-  if (status.ok()) {
-    thread_pool.clear();
-    std::vector<const CompactionOutputs::Output*> files_output;
-    for (const auto& state : compact_->sub_compact_states) {
-      for (const auto& output : state.GetOutputs()) {
-        files_output.emplace_back(&output);
-      }
+
+  return status;
+}
+
+Status CompactionJob::VerifyOutputFiles() {
+  Status status;
+  std::vector<port::Thread> thread_pool;
+  std::vector<const CompactionOutputs::Output*> files_output;
+  for (const auto& state : compact_->sub_compact_states) {
+    for (const auto& output : state.GetOutputs()) {
+      files_output.emplace_back(&output);
     }
-    ColumnFamilyData* cfd = compact_->compaction->column_family_data();
-    std::atomic<size_t> next_file_idx(0);
-    auto verify_table = [&](Status& output_status) {
-      while (true) {
-        size_t file_idx = next_file_idx.fetch_add(1);
-        if (file_idx >= files_output.size()) {
-          break;
-        }
-        // Verify that the table is usable
-        // We set for_compaction to false and don't
-        // OptimizeForCompactionTableRead here because this is a special case
-        // after we finish the table building No matter whether
-        // use_direct_io_for_flush_and_compaction is true, we will regard this
-        // verification as user reads since the goal is to cache it here for
-        // further user reads
-        ReadOptions verify_table_read_options(Env::IOActivity::kCompaction);
-        verify_table_read_options.rate_limiter_priority =
-            GetRateLimiterPriority();
-        InternalIterator* iter = cfd->table_cache()->NewIterator(
-            verify_table_read_options, file_options_,
-            cfd->internal_comparator(), files_output[file_idx]->meta,
-            /*range_del_agg=*/nullptr,
-            compact_->compaction->mutable_cf_options(),
-            /*table_reader_ptr=*/nullptr,
-            cfd->internal_stats()->GetFileReadHist(
-                compact_->compaction->output_level()),
-            TableReaderCaller::kCompactionRefill, /*arena=*/nullptr,
-            /*skip_filters=*/false, compact_->compaction->output_level(),
-            MaxFileSizeForL0MetaPin(compact_->compaction->mutable_cf_options()),
-            /*smallest_compaction_key=*/nullptr,
-            /*largest_compaction_key=*/nullptr,
-            /*allow_unprepared_value=*/false);
-        auto s = iter->status();
+  }
+  ColumnFamilyData* cfd = compact_->compaction->column_family_data();
+  std::atomic<size_t> next_file_idx(0);
+  auto verify_table = [&](Status& output_status) {
+    while (true) {
+      size_t file_idx = next_file_idx.fetch_add(1);
+      if (file_idx >= files_output.size()) {
+        break;
+      }
+      // Verify that the table is usable
+      // We set for_compaction to false and don't
+      // OptimizeForCompactionTableRead here because this is a special case
+      // after we finish the table building No matter whether
+      // use_direct_io_for_flush_and_compaction is true, we will regard this
+      // verification as user reads since the goal is to cache it here for
+      // further user reads
+      ReadOptions verify_table_read_options(Env::IOActivity::kCompaction);
+      verify_table_read_options.rate_limiter_priority =
+          GetRateLimiterPriority();
+      InternalIterator* iter = cfd->table_cache()->NewIterator(
+          verify_table_read_options, file_options_, cfd->internal_comparator(),
+          files_output[file_idx]->meta,
+          /*range_del_agg=*/nullptr, compact_->compaction->mutable_cf_options(),
+          /*table_reader_ptr=*/nullptr,
+          cfd->internal_stats()->GetFileReadHist(
+              compact_->compaction->output_level()),
+          TableReaderCaller::kCompactionRefill, /*arena=*/nullptr,
+          /*skip_filters=*/false, compact_->compaction->output_level(),
+          MaxFileSizeForL0MetaPin(compact_->compaction->mutable_cf_options()),
+          /*smallest_compaction_key=*/nullptr,
+          /*largest_compaction_key=*/nullptr,
+          /*allow_unprepared_value=*/false);
+      auto s = iter->status();
 
-        if (s.ok() && paranoid_file_checks_) {
-          OutputValidator validator(cfd->internal_comparator(),
-                                    /*_enable_hash=*/true);
-          for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-            s = validator.Add(iter->key(), iter->value());
-            if (!s.ok()) {
-              break;
-            }
-          }
-          if (s.ok()) {
-            s = iter->status();
-          }
-          if (s.ok() &&
-              !validator.CompareValidator(files_output[file_idx]->validator)) {
-            s = Status::Corruption("Paranoid checksums do not match");
+      if (s.ok() && paranoid_file_checks_) {
+        OutputValidator validator(cfd->internal_comparator(),
+                                  /*_enable_hash=*/true);
+        for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+          s = validator.Add(iter->key(), iter->value());
+          if (!s.ok()) {
+            break;
           }
         }
-
-        delete iter;
-
-        if (!s.ok()) {
-          output_status = s;
-          break;
+        if (s.ok()) {
+          s = iter->status();
+        }
+        if (s.ok() &&
+            !validator.CompareValidator(files_output[file_idx]->validator)) {
+          s = Status::Corruption("Paranoid checksums do not match");
         }
       }
-    };
-    for (size_t i = 1; i < compact_->sub_compact_states.size(); i++) {
-      thread_pool.emplace_back(
-          verify_table, std::ref(compact_->sub_compact_states[i].status));
-    }
-    verify_table(compact_->sub_compact_states[0].status);
-    for (auto& thread : thread_pool) {
-      thread.join();
-    }
 
-    for (const auto& state : compact_->sub_compact_states) {
-      if (!state.status.ok()) {
-        status = state.status;
+      delete iter;
+
+      if (!s.ok()) {
+        output_status = s;
         break;
       }
     }
+  };
+  for (size_t i = 1; i < compact_->sub_compact_states.size(); i++) {
+    thread_pool.emplace_back(verify_table,
+                             std::ref(compact_->sub_compact_states[i].status));
+  }
+  verify_table(compact_->sub_compact_states[0].status);
+  for (auto& thread : thread_pool) {
+    thread.join();
   }
 
-  ReleaseSubcompactionResources();
-  TEST_SYNC_POINT("CompactionJob::ReleaseSubcompactionResources");
+  for (const auto& state : compact_->sub_compact_states) {
+    if (!state.status.ok()) {
+      status = state.status;
+      break;
+    }
+  }
 
+  return status;
+}
+
+void CompactionJob::SetOutputTableProperties() {
   for (const auto& state : compact_->sub_compact_states) {
     for (const auto& output : state.GetOutputs()) {
       auto fn =
@@ -850,7 +875,9 @@ Status CompactionJob::Run() {
                                                      output.table_properties);
     }
   }
+}
 
+void CompactionJob::AggregateSubcompactionStats() {
   // Before the compaction starts, is_remote_compaction was set to true if
   // compaction_service is set. We now know whether each sub_compaction was
   // done remotely or not. Reset is_remote_compaction back to false and allow
@@ -859,68 +886,86 @@ Status CompactionJob::Run() {
 
   // Finish up all bookkeeping to unify the subcompaction results.
   compact_->AggregateCompactionStats(internal_stats_, *job_stats_);
+}
 
-  uint64_t num_input_range_del = 0;
-  bool ok = BuildStatsFromInputTableProperties(&num_input_range_del);
-  // (Sub)compactions returned ok, do sanity check on the number of input
-  // keys.
-  if (status.ok() && ok) {
-    if (job_stats_->has_num_input_records) {
-      status = VerifyInputRecordCount(num_input_range_del);
-      if (!status.ok()) {
-        ROCKS_LOG_WARN(
-            db_options_.info_log, "[%s] [JOB %d] Compaction with status: %s",
-            compact_->compaction->column_family_data()->GetName().c_str(),
-            job_context_->job_id, status.ToString().c_str());
-      }
+Status CompactionJob::VerifyCompactionRecordCounts(
+    bool stats_built_from_input_table_prop, uint64_t num_input_range_del) {
+  Status status;
+  if (stats_built_from_input_table_prop && job_stats_->has_num_input_records) {
+    status = VerifyInputRecordCount(num_input_range_del);
+    if (!status.ok()) {
+      return status;
     }
+  }
+
+  const auto& mutable_cf_options = compact_->compaction->mutable_cf_options();
+  if ((mutable_cf_options.table_factory->IsInstanceOf(
+           TableFactory::kBlockBasedTableName()) ||
+       mutable_cf_options.table_factory->IsInstanceOf(
+           TableFactory::kPlainTableName()))) {
+    status = VerifyOutputRecordCount();
+    if (!status.ok()) {
+      return status;
+    }
+  }
+  return status;
+}
+
+void CompactionJob::FinalizeCompactionRun(
+    const Status& input_status, bool stats_built_from_input_table_prop,
+    uint64_t num_input_range_del) {
+  if (stats_built_from_input_table_prop) {
     UpdateCompactionJobInputStats(internal_stats_, num_input_range_del);
   }
   UpdateCompactionJobOutputStats(internal_stats_);
-
-  // Verify number of output records
-  // Only verify on table with format collects table properties
-  const auto& mutable_cf_options = compact_->compaction->mutable_cf_options();
-  if (status.ok() &&
-      (mutable_cf_options.table_factory->IsInstanceOf(
-           TableFactory::kBlockBasedTableName()) ||
-       mutable_cf_options.table_factory->IsInstanceOf(
-           TableFactory::kPlainTableName())) &&
-      db_options_.compaction_verify_record_count) {
-    uint64_t total_output_num = 0;
-    for (const auto& state : compact_->sub_compact_states) {
-      for (const auto& output : state.GetOutputs()) {
-        total_output_num += output.table_properties->num_entries -
-                            output.table_properties->num_range_deletions;
-      }
-    }
-
-    uint64_t expected = internal_stats_.output_level_stats.num_output_records;
-    if (internal_stats_.has_proximal_level_output) {
-      expected += internal_stats_.proximal_level_stats.num_output_records;
-    }
-    if (expected != total_output_num) {
-      char scratch[2345];
-      compact_->compaction->Summary(scratch, sizeof(scratch));
-      std::string msg =
-          "Number of keys in compaction output SST files does not match "
-          "number of keys added. Expected " +
-          std::to_string(expected) + " but there are " +
-          std::to_string(total_output_num) +
-          " in output SST files. Compaction summary: " + scratch;
-      ROCKS_LOG_WARN(
-          db_options_.info_log, "[%s] [JOB %d] Compaction with status: %s",
-          compact_->compaction->column_family_data()->GetName().c_str(),
-          job_context_->job_id, msg.c_str());
-      status = Status::Corruption(msg);
-    }
-  }
-
   RecordCompactionIOStats();
+
   LogFlush(db_options_.info_log);
   TEST_SYNC_POINT("CompactionJob::Run():End");
-  compact_->status = status;
-  TEST_SYNC_POINT_CALLBACK("CompactionJob::Run():EndStatusSet", &status);
+  compact_->status = input_status;
+  TEST_SYNC_POINT_CALLBACK("CompactionJob::Run():EndStatusSet",
+                           const_cast<Status*>(&input_status));
+}
+
+Status CompactionJob::Run() {
+  InitializeCompactionRun();
+
+  const uint64_t start_micros = db_options_.clock->NowMicros();
+
+  RunSubcompactions();
+
+  UpdateTimingStats(start_micros);
+
+  TEST_SYNC_POINT("CompactionJob::Run:BeforeVerify");
+
+  Status status = CollectSubcompactionErrors();
+
+  if (status.ok()) {
+    status = SyncOutputDirectories();
+  }
+
+  if (status.ok()) {
+    status = VerifyOutputFiles();
+  }
+
+  if (status.ok()) {
+    SetOutputTableProperties();
+  }
+
+  AggregateSubcompactionStats();
+
+  uint64_t num_input_range_del = 0;
+  bool stats_built_from_input_table_prop =
+      BuildStatsFromInputFiles(&num_input_range_del);
+
+  if (status.ok()) {
+    status = VerifyCompactionRecordCounts(stats_built_from_input_table_prop,
+                                          num_input_range_del);
+  }
+
+  FinalizeCompactionRun(status, stats_built_from_input_table_prop,
+                        num_input_range_del);
+
   return status;
 }
 
@@ -1365,8 +1410,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
       &(job_context_->snapshot_seqs), earliest_snapshot_,
       job_context_->earliest_write_conflict_snapshot, job_snapshot_seq,
       job_context_->snapshot_checker, env_,
-      ShouldReportDetailedTime(env_, stats_),
-      /*expect_valid_internal_key=*/true, sub_compact->RangeDelAgg(),
+      ShouldReportDetailedTime(env_, stats_), sub_compact->RangeDelAgg(),
       blob_file_builder.get(), db_options_.allow_data_in_errors,
       db_options_.enforce_single_del_contracts, manual_compaction_canceled_,
       sub_compact->compaction
@@ -1736,14 +1780,11 @@ Status CompactionJob::FinishCompactionOutputFile(
   if (s.ok()) {
     tp = outputs.GetTableProperties();
   }
-
   if (s.ok() && current_entries == 0 && tp.num_range_deletions == 0) {
     // If there is nothing to output, no necessary to generate a sst file.
     // This happens when the output level is bottom level, at the same time
     // the sub_compact output nothing.
-    std::string fname =
-        TableFileName(sub_compact->compaction->immutable_options().cf_paths,
-                      meta->fd.GetNumber(), meta->fd.GetPathId());
+    std::string fname = GetTableFileName(meta->fd.GetNumber());
 
     // TODO(AR) it is not clear if there are any larger implications if
     // DeleteFile fails here
@@ -1942,6 +1983,10 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
 
   // no need to lock because VersionSet::next_file_number_ is atomic
   uint64_t file_number = versions_->NewFileNumber();
+#ifndef NDEBUG
+  TEST_SYNC_POINT_CALLBACK(
+      "CompactionJob::OpenCompactionOutputFile::NewFileNumber", &file_number);
+#endif
   std::string fname = GetTableFileName(file_number);
   // Fire events.
   ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
@@ -2105,8 +2150,7 @@ void CopyPrefix(const Slice& src, size_t prefix_length, std::string* dst) {
 }
 }  // namespace
 
-bool CompactionJob::BuildStatsFromInputTableProperties(
-    uint64_t* num_input_range_del) {
+bool CompactionJob::BuildStatsFromInputFiles(uint64_t* num_input_range_del) {
   assert(compact_);
 
   Compaction* compaction = compact_->compaction;
@@ -2377,6 +2421,11 @@ Status CompactionJob::VerifyInputRecordCount(
           "number of keys processed. Expected " +
           std::to_string(expected) + " but processed " +
           std::to_string(actual) + ". Compaction summary: " + scratch;
+      ROCKS_LOG_WARN(
+          db_options_.info_log,
+          "[%s] [JOB %d] VerifyInputRecordCount() Status: %s",
+          compact_->compaction->column_family_data()->GetName().c_str(),
+          job_context_->job_id, msg.c_str());
       if (db_options_.compaction_verify_record_count) {
         return Status::Corruption(msg);
       }
@@ -2385,4 +2434,38 @@ Status CompactionJob::VerifyInputRecordCount(
   return Status::OK();
 }
 
+Status CompactionJob::VerifyOutputRecordCount() const {
+  uint64_t total_output_num = 0;
+  for (const auto& state : compact_->sub_compact_states) {
+    for (const auto& output : state.GetOutputs()) {
+      total_output_num += output.table_properties->num_entries -
+                          output.table_properties->num_range_deletions;
+    }
+  }
+
+  uint64_t expected = internal_stats_.output_level_stats.num_output_records;
+  if (internal_stats_.has_proximal_level_output) {
+    expected += internal_stats_.proximal_level_stats.num_output_records;
+  }
+  if (expected != total_output_num) {
+    char scratch[2345];
+    compact_->compaction->Summary(scratch, sizeof(scratch));
+    std::string msg =
+        "Number of keys in compaction output SST files does not match "
+        "number of keys added. Expected " +
+        std::to_string(expected) + " but there are " +
+        std::to_string(total_output_num) +
+        " in output SST files. Compaction summary: " + scratch;
+    ROCKS_LOG_WARN(
+        db_options_.info_log,
+        "[%s] [JOB %d] VerifyOutputRecordCount() status: %s",
+        compact_->compaction->column_family_data()->GetName().c_str(),
+        job_context_->job_id, msg.c_str());
+    if (db_options_.compaction_verify_record_count) {
+      return Status::Corruption(msg);
+    }
+  }
+  return Status::OK();
+}
+
 }  // namespace ROCKSDB_NAMESPACE

db/compaction/compaction_job.h

@@ -225,7 +225,7 @@ class CompactionJob {
  private:
   friend class CompactionJobTestBase;
 
-  // Collect the following stats from Input Table Properties
+  // Collect the following stats from input files and table properties
   // - num_input_files_in_non_output_levels
   // - num_input_files_in_output_level
   // - bytes_read_non_output_levels
@@ -242,14 +242,14 @@ class CompactionJob {
   // num_input_range_del are calculated successfully.
   //
   // This should be called only once for compactions (not per subcompaction)
-  bool BuildStatsFromInputTableProperties(
-      uint64_t* num_input_range_del = nullptr);
+  bool BuildStatsFromInputFiles(uint64_t* num_input_range_del = nullptr);
 
   void UpdateCompactionJobInputStats(
       const InternalStats::CompactionStatsFull& internal_stats,
       uint64_t num_input_range_del) const;
 
   Status VerifyInputRecordCount(uint64_t num_input_range_del) const;
+  Status VerifyOutputRecordCount() const;
 
   // Generates a histogram representing potential divisions of key ranges from
   // the input. It adds the starting and/or ending keys of certain input files
@@ -278,6 +278,22 @@ class CompactionJob {
   // Release all reserved threads and update the compaction limits.
   void ReleaseSubcompactionResources();
 
+  void InitializeCompactionRun();
+  void RunSubcompactions();
+  void UpdateTimingStats(uint64_t start_micros);
+  void RemoveEmptyOutputs();
+  bool HasNewBlobFiles() const;
+  Status CollectSubcompactionErrors();
+  Status SyncOutputDirectories();
+  Status VerifyOutputFiles();
+  void SetOutputTableProperties();
+  void AggregateSubcompactionStats();
+  Status VerifyCompactionRecordCounts(bool stats_built_from_input_table_prop,
+                                      uint64_t num_input_range_del);
+  void FinalizeCompactionRun(const Status& status,
+                             bool stats_built_from_input_table_prop,
+                             uint64_t num_input_range_del);
+
   CompactionServiceJobStatus ProcessKeyValueCompactionWithCompactionService(
       SubcompactionState* sub_compact);
 

db/compaction/compaction_picker.cc

@@ -333,11 +333,13 @@ bool CompactionPicker::AreFilesInCompaction(
   return false;
 }
 
-Compaction* CompactionPicker::CompactFiles(
+Compaction* CompactionPicker::PickCompactionForCompactFiles(
     const CompactionOptions& compact_options,
     const std::vector<CompactionInputFiles>& input_files, int output_level,
     VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options,
-    const MutableDBOptions& mutable_db_options, uint32_t output_path_id) {
+    const MutableDBOptions& mutable_db_options, uint32_t output_path_id,
+    std::optional<SequenceNumber> earliest_snapshot,
+    const SnapshotChecker* snapshot_checker) {
 #ifndef NDEBUG
   assert(input_files.size());
   // This compaction output should not overlap with a running compaction as
@@ -380,8 +382,8 @@ Compaction* CompactionPicker::CompactFiles(
       GetCompressionOptions(mutable_cf_options, vstorage, output_level),
       mutable_cf_options.default_write_temperature,
       compact_options.max_subcompactions,
-      /* grandparents */ {}, /* earliest_snapshot */ std::nullopt,
-      /* snapshot_checker */ nullptr, CompactionReason::kManualCompaction);
+      /* grandparents */ {}, earliest_snapshot, snapshot_checker,
+      CompactionReason::kManualCompaction);
   RegisterCompaction(c);
   return c;
 }
@@ -601,7 +603,7 @@ void CompactionPicker::GetGrandparents(
   }
 }
 
-Compaction* CompactionPicker::CompactRange(
+Compaction* CompactionPicker::PickCompactionForCompactRange(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
     int input_level, int output_level,
@@ -617,8 +619,8 @@ Compaction* CompactionPicker::CompactRange(
     // Universal compaction with more than one level always compacts all the
     // files together to the last level.
     assert(vstorage->num_levels() > 1);
-    int max_output_level =
-        vstorage->MaxOutputLevel(ioptions_.allow_ingest_behind);
+    int max_output_level = vstorage->MaxOutputLevel(
+        ioptions_.cf_allow_ingest_behind || ioptions_.allow_ingest_behind);
     // DBImpl::CompactRange() set output level to be the last level
     assert(output_level == max_output_level);
     // DBImpl::RunManualCompaction will make full range for universal compaction

db/compaction/compaction_picker.h

@@ -75,7 +75,7 @@ class CompactionPicker {
   // *compaction_end should point to valid InternalKey!
   // REQUIRES: If not compacting all levels (input_level == kCompactAllLevels),
   // then levels between input_level and output_level should be empty.
-  virtual Compaction* CompactRange(
+  virtual Compaction* PickCompactionForCompactRange(
       const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
       const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
       int input_level, int output_level,
@@ -117,12 +117,17 @@ class CompactionPicker {
   // Caller must provide a set of input files that has been passed through
   // `SanitizeAndConvertCompactionInputFiles` earlier. The lock should not be
   // released between that call and this one.
-  Compaction* CompactFiles(const CompactionOptions& compact_options,
-                           const std::vector<CompactionInputFiles>& input_files,
-                           int output_level, VersionStorageInfo* vstorage,
-                           const MutableCFOptions& mutable_cf_options,
-                           const MutableDBOptions& mutable_db_options,
-                           uint32_t output_path_id);
+  //
+  //  TODO - Remove default values for earliest_snapshot and snapshot_checker
+  //  and require all callers to pass them in so that DB::CompactFiles() can
+  //  also benefit from Standalone Range Tombstone Optimization
+  Compaction* PickCompactionForCompactFiles(
+      const CompactionOptions& compact_options,
+      const std::vector<CompactionInputFiles>& input_files, int output_level,
+      VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options,
+      const MutableDBOptions& mutable_db_options, uint32_t output_path_id,
+      std::optional<SequenceNumber> earliest_snapshot = std::nullopt,
+      const SnapshotChecker* snapshot_checker = nullptr);
 
   // Converts a set of compaction input file numbers into
   // a list of CompactionInputFiles.
@@ -278,18 +283,17 @@ class NullCompactionPicker : public CompactionPicker {
   }
 
   // Always return "nullptr"
-  Compaction* CompactRange(const std::string& /*cf_name*/,
-                           const MutableCFOptions& /*mutable_cf_options*/,
-                           const MutableDBOptions& /*mutable_db_options*/,
-                           VersionStorageInfo* /*vstorage*/,
-                           int /*input_level*/, int /*output_level*/,
-                           const CompactRangeOptions& /*compact_range_options*/,
-                           const InternalKey* /*begin*/,
-                           const InternalKey* /*end*/,
-                           InternalKey** /*compaction_end*/,
-                           bool* /*manual_conflict*/,
-                           uint64_t /*max_file_num_to_ignore*/,
-                           const std::string& /*trim_ts*/) override {
+  Compaction* PickCompactionForCompactRange(
+      const std::string& /*cf_name*/,
+      const MutableCFOptions& /*mutable_cf_options*/,
+      const MutableDBOptions& /*mutable_db_options*/,
+      VersionStorageInfo* /*vstorage*/, int /*input_level*/,
+      int /*output_level*/,
+      const CompactRangeOptions& /*compact_range_options*/,
+      const InternalKey* /*begin*/, const InternalKey* /*end*/,
+      InternalKey** /*compaction_end*/, bool* /*manual_conflict*/,
+      uint64_t /*max_file_num_to_ignore*/,
+      const std::string& /*trim_ts*/) override {
     return nullptr;
   }
 

db/compaction/compaction_picker_fifo.cc

@@ -258,6 +258,9 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
     // better serves a major type of FIFO use cases where smaller keys are
     // associated with older data.
     for (const auto& f : last_level_files) {
+      if (f->being_compacted) {
+        continue;
+      }
       total_size -= f->fd.file_size;
       inputs[0].files.push_back(f);
       char tmp_fsize[16];
@@ -440,7 +443,7 @@ Compaction* FIFOCompactionPicker::PickCompaction(
   return c;
 }
 
-Compaction* FIFOCompactionPicker::CompactRange(
+Compaction* FIFOCompactionPicker::PickCompactionForCompactRange(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
     int input_level, int output_level,

db/compaction/compaction_picker_fifo.h

@@ -26,16 +26,14 @@ class FIFOCompactionPicker : public CompactionPicker {
       VersionStorageInfo* version, LogBuffer* log_buffer,
       bool /* require_max_output_level*/ = false) override;
 
-  Compaction* CompactRange(const std::string& cf_name,
-                           const MutableCFOptions& mutable_cf_options,
-                           const MutableDBOptions& mutable_db_options,
-                           VersionStorageInfo* vstorage, int input_level,
-                           int output_level,
-                           const CompactRangeOptions& compact_range_options,
-                           const InternalKey* begin, const InternalKey* end,
-                           InternalKey** compaction_end, bool* manual_conflict,
-                           uint64_t max_file_num_to_ignore,
-                           const std::string& trim_ts) override;
+  Compaction* PickCompactionForCompactRange(
+      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+      const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
+      int input_level, int output_level,
+      const CompactRangeOptions& compact_range_options,
+      const InternalKey* begin, const InternalKey* end,
+      InternalKey** compaction_end, bool* manual_conflict,
+      uint64_t max_file_num_to_ignore, const std::string& trim_ts) override;
 
   // The maximum allowed output level.  Always returns 0.
   int MaxOutputLevel() const override { return 0; }

db/compaction/compaction_picker_test.cc

@@ -544,41 +544,48 @@ TEST_F(CompactionPickerTest, NeedsCompactionUniversal) {
 }
 
 TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {
-  const uint64_t kFileSize = 100000;
-  NewVersionStorage(3 /* num_levels */, kCompactionStyleUniversal);
-  ioptions_.allow_ingest_behind = true;
-  ioptions_.num_levels = 3;
-  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
-  UpdateVersionStorageInfo();
-  // must return false when there's no files.
-  ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
-            false);
+  for (bool cf_option : {false, true}) {
+    SCOPED_TRACE("cf_option = " + std::to_string(cf_option));
+    const uint64_t kFileSize = 100000;
+    NewVersionStorage(3 /* num_levels */, kCompactionStyleUniversal);
+    if (cf_option) {
+      ioptions_.cf_allow_ingest_behind = true;
+    } else {
+      ioptions_.allow_ingest_behind = true;
+    }
+    ioptions_.num_levels = 3;
+    UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
+    UpdateVersionStorageInfo();
+    // must return false when there's no files.
+    ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
+              false);
 
-  NewVersionStorage(3, kCompactionStyleUniversal);
+    NewVersionStorage(3, kCompactionStyleUniversal);
 
-  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
-  Add(0, 2U, "201", "250", kFileSize, 0, 401, 450);
-  Add(0, 4U, "260", "300", kFileSize, 0, 260, 300);
-  Add(1, 5U, "100", "151", kFileSize, 0, 200, 251);
-  Add(1, 3U, "301", "350", kFileSize, 0, 101, 150);
-  Add(2, 6U, "120", "200", kFileSize, 0, 20, 100);
+    Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
+    Add(0, 2U, "201", "250", kFileSize, 0, 401, 450);
+    Add(0, 4U, "260", "300", kFileSize, 0, 260, 300);
+    Add(1, 5U, "100", "151", kFileSize, 0, 200, 251);
+    Add(1, 3U, "301", "350", kFileSize, 0, 101, 150);
+    Add(2, 6U, "120", "200", kFileSize, 0, 20, 100);
 
-  UpdateVersionStorageInfo();
+    UpdateVersionStorageInfo();
 
-  std::unique_ptr<Compaction> compaction(
-      universal_compaction_picker.PickCompaction(
-          cf_name_, mutable_cf_options_, mutable_db_options_,
-          /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
-          vstorage_.get(), &log_buffer_));
+    std::unique_ptr<Compaction> compaction(
+        universal_compaction_picker.PickCompaction(
+            cf_name_, mutable_cf_options_, mutable_db_options_,
+            /*existing_snapshots=*/{}, /* snapshot_checker */ nullptr,
+            vstorage_.get(), &log_buffer_));
 
-  // output level should be the one above the bottom-most
-  ASSERT_EQ(1, compaction->output_level());
+    // output level should be the one above the bottom-most
+    ASSERT_EQ(1, compaction->output_level());
 
-  // input should not include the reserved level
-  const std::vector<CompactionInputFiles>* inputs = compaction->inputs();
-  for (const auto& compaction_input : *inputs) {
-    if (!compaction_input.empty()) {
-      ASSERT_LT(compaction_input.level, 2);
+    // input should not include the reserved level
+    const std::vector<CompactionInputFiles>* inputs = compaction->inputs();
+    for (const auto& compaction_input : *inputs) {
+      if (!compaction_input.empty()) {
+        ASSERT_LT(compaction_input.level, 2);
+      }
     }
   }
 }
@@ -2689,13 +2696,14 @@ TEST_F(CompactionPickerTest, CompactRangeMaxCompactionBytes) {
   bool manual_conflict = false;
   InternalKey manual_end;
   InternalKey* manual_end_ptr = &manual_end;
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.CompactRange(
-      cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
-      /*input_level=*/1, /*output_level=*/2,
-      /*compact_range_options*/ {}, /*begin=*/nullptr, /*end=*/nullptr,
-      &manual_end_ptr, &manual_conflict,
-      /*max_file_num_to_ignore=*/std::numeric_limits<uint64_t>::max(),
-      /*trim_ts=*/""));
+  std::unique_ptr<Compaction> compaction(
+      level_compaction_picker.PickCompactionForCompactRange(
+          cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
+          /*input_level=*/1, /*output_level=*/2,
+          /*compact_range_options*/ {}, /*begin=*/nullptr, /*end=*/nullptr,
+          &manual_end_ptr, &manual_conflict,
+          /*max_file_num_to_ignore=*/std::numeric_limits<uint64_t>::max(),
+          /*trim_ts=*/""));
   ASSERT_TRUE(compaction.get() != nullptr);
   ASSERT_EQ(1U, compaction->num_input_levels());
   ASSERT_EQ(2, compaction->output_level());
@@ -3627,7 +3635,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedManualCompaction) {
   bool manual_conflict = false;
   InternalKey* manual_end = nullptr;
   std::unique_ptr<Compaction> compaction(
-      universal_compaction_picker.CompactRange(
+      universal_compaction_picker.PickCompactionForCompactRange(
           cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
           ColumnFamilyData::kCompactAllLevels, 6, CompactRangeOptions(),
           nullptr, nullptr, &manual_end, &manual_conflict,
@@ -3831,9 +3839,10 @@ TEST_F(CompactionPickerU64TsTest, Overlap) {
   std::vector<CompactionInputFiles> input_files;
   ASSERT_OK(level_compaction_picker.GetCompactionInputsFromFileNumbers(
       &input_files, &input, vstorage_.get(), CompactionOptions()));
-  std::unique_ptr<Compaction> comp1(level_compaction_picker.CompactFiles(
-      CompactionOptions(), input_files, level, vstorage_.get(),
-      mutable_cf_options_, mutable_db_options_, /*output_path_id=*/0));
+  std::unique_ptr<Compaction> comp1(
+      level_compaction_picker.PickCompactionForCompactFiles(
+          CompactionOptions(), input_files, level, vstorage_.get(),
+          mutable_cf_options_, mutable_db_options_, /*output_path_id=*/0));
 
   {
     // [600, ts=50000] to [600, ts=50000] is the range to check.
@@ -3942,9 +3951,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest, OverlapWithNormalCompaction) {
   ASSERT_OK(level_compaction_picker.GetCompactionInputsFromFileNumbers(
       &input_files, &input_set, vstorage_.get(), comp_options));
 
-  std::unique_ptr<Compaction> comp1(level_compaction_picker.CompactFiles(
-      comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_,
-      mutable_db_options_, 0));
+  std::unique_ptr<Compaction> comp1(
+      level_compaction_picker.PickCompactionForCompactFiles(
+          comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_,
+          mutable_db_options_, 0));
 
   input_set.clear();
   input_files.clear();
@@ -3988,9 +3998,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest, NormalCompactionOverlap) {
   ASSERT_OK(level_compaction_picker.GetCompactionInputsFromFileNumbers(
       &input_files, &input_set, vstorage_.get(), comp_options));
 
-  std::unique_ptr<Compaction> comp1(level_compaction_picker.CompactFiles(
-      comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
-      mutable_db_options_, 0));
+  std::unique_ptr<Compaction> comp1(
+      level_compaction_picker.PickCompactionForCompactFiles(
+          comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
+          mutable_db_options_, 0));
 
   input_set.clear();
   input_files.clear();
@@ -4030,9 +4041,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
   ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers(
       &input_files, &input_set, vstorage_.get(), comp_options));
 
-  std::unique_ptr<Compaction> comp1(universal_compaction_picker.CompactFiles(
-      comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_,
-      mutable_db_options_, 0));
+  std::unique_ptr<Compaction> comp1(
+      universal_compaction_picker.PickCompactionForCompactFiles(
+          comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_,
+          mutable_db_options_, 0));
 
   input_set.clear();
   input_files.clear();
@@ -4077,9 +4089,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest, NormalCompactionOverlapUniversal) {
   ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers(
       &input_files, &input_set, vstorage_.get(), comp_options));
 
-  std::unique_ptr<Compaction> comp1(universal_compaction_picker.CompactFiles(
-      comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
-      mutable_db_options_, 0));
+  std::unique_ptr<Compaction> comp1(
+      universal_compaction_picker.PickCompactionForCompactFiles(
+          comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
+          mutable_db_options_, 0));
 
   input_set.clear();
   input_files.clear();
@@ -4125,9 +4138,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest, ProximalOverlapUniversal) {
   ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers(
       &input_files, &input_set, vstorage_.get(), comp_options));
 
-  std::unique_ptr<Compaction> comp1(universal_compaction_picker.CompactFiles(
-      comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
-      mutable_db_options_, 0));
+  std::unique_ptr<Compaction> comp1(
+      universal_compaction_picker.PickCompactionForCompactFiles(
+          comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
+          mutable_db_options_, 0));
 
   input_set.clear();
   input_files.clear();
@@ -4176,9 +4190,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest, LastLevelOnlyOverlapUniversal) {
   ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers(
       &input_files, &input_set, vstorage_.get(), comp_options));
 
-  std::unique_ptr<Compaction> comp1(universal_compaction_picker.CompactFiles(
-      comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
-      mutable_db_options_, 0));
+  std::unique_ptr<Compaction> comp1(
+      universal_compaction_picker.PickCompactionForCompactFiles(
+          comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
+          mutable_db_options_, 0));
 
   // cannot compact file 41 if the preclude_last_level feature is on, otherwise
   // compact file 41 is okay.
@@ -4234,9 +4249,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
   ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers(
       &input_files, &input_set, vstorage_.get(), comp_options));
 
-  std::unique_ptr<Compaction> comp1(universal_compaction_picker.CompactFiles(
-      comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
-      mutable_db_options_, 0));
+  std::unique_ptr<Compaction> comp1(
+      universal_compaction_picker.PickCompactionForCompactFiles(
+          comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
+          mutable_db_options_, 0));
 
   ASSERT_TRUE(comp1);
   ASSERT_EQ(comp1->GetProximalLevel(), Compaction::kInvalidLevel);
@@ -4252,9 +4268,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
   ASSERT_FALSE(universal_compaction_picker.FilesRangeOverlapWithCompaction(
       input_files, 5, Compaction::kInvalidLevel));
 
-  std::unique_ptr<Compaction> comp2(universal_compaction_picker.CompactFiles(
-      comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_,
-      mutable_db_options_, 0));
+  std::unique_ptr<Compaction> comp2(
+      universal_compaction_picker.PickCompactionForCompactFiles(
+          comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_,
+          mutable_db_options_, 0));
   ASSERT_TRUE(comp2);
   ASSERT_EQ(Compaction::kInvalidLevel, comp2->GetProximalLevel());
 }
@@ -4290,9 +4307,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
   ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers(
       &input_files, &input_set, vstorage_.get(), comp_options));
 
-  std::unique_ptr<Compaction> comp1(universal_compaction_picker.CompactFiles(
-      comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_,
-      mutable_db_options_, 0));
+  std::unique_ptr<Compaction> comp1(
+      universal_compaction_picker.PickCompactionForCompactFiles(
+          comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_,
+          mutable_db_options_, 0));
 
   ASSERT_TRUE(comp1);
   ASSERT_EQ(comp1->GetProximalLevel(), Compaction::kInvalidLevel);
@@ -4310,9 +4328,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
                     vstorage_.get(), mutable_cf_options_, ioptions_, 6, 6)));
 
   if (!enable_per_key_placement_) {
-    std::unique_ptr<Compaction> comp2(universal_compaction_picker.CompactFiles(
-        comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
-        mutable_db_options_, 0));
+    std::unique_ptr<Compaction> comp2(
+        universal_compaction_picker.PickCompactionForCompactFiles(
+            comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
+            mutable_db_options_, 0));
     ASSERT_TRUE(comp2);
     ASSERT_EQ(Compaction::kInvalidLevel, comp2->GetProximalLevel());
   }
@@ -4350,9 +4369,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
   ASSERT_OK(universal_compaction_picker.GetCompactionInputsFromFileNumbers(
       &input_files, &input_set, vstorage_.get(), comp_options));
 
-  std::unique_ptr<Compaction> comp1(universal_compaction_picker.CompactFiles(
-      comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_,
-      mutable_db_options_, 0));
+  std::unique_ptr<Compaction> comp1(
+      universal_compaction_picker.PickCompactionForCompactFiles(
+          comp_options, input_files, 5, vstorage_.get(), mutable_cf_options_,
+          mutable_db_options_, 0));
 
   ASSERT_TRUE(comp1);
   ASSERT_EQ(comp1->GetProximalLevel(), Compaction::kInvalidLevel);
@@ -4370,9 +4390,10 @@ TEST_P(PerKeyPlacementCompactionPickerTest,
                                         ioptions_, 6, 6)));
 
   // 2 compactions can be run in parallel
-  std::unique_ptr<Compaction> comp2(universal_compaction_picker.CompactFiles(
-      comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
-      mutable_db_options_, 0));
+  std::unique_ptr<Compaction> comp2(
+      universal_compaction_picker.PickCompactionForCompactFiles(
+          comp_options, input_files, 6, vstorage_.get(), mutable_cf_options_,
+          mutable_db_options_, 0));
   ASSERT_TRUE(comp2);
   if (enable_per_key_placement_) {
     ASSERT_NE(Compaction::kInvalidLevel, comp2->GetProximalLevel());

db/compaction/compaction_picker_universal.cc

@@ -48,7 +48,9 @@ class UniversalCompactionBuilder {
         vstorage_(vstorage),
         picker_(picker),
         log_buffer_(log_buffer),
-        require_max_output_level_(require_max_output_level) {
+        require_max_output_level_(require_max_output_level),
+        allow_ingest_behind_(ioptions.cf_allow_ingest_behind ||
+                             ioptions.allow_ingest_behind) {
     assert(icmp_);
     const auto* ucmp = icmp_->user_comparator();
     assert(ucmp);
@@ -422,8 +424,7 @@ class UniversalCompactionBuilder {
   bool MeetsOutputLevelRequirements(int output_level) const {
     return !require_max_output_level_ ||
            Compaction::OutputToNonZeroMaxOutputLevel(
-               output_level,
-               vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind));
+               output_level, vstorage_->MaxOutputLevel(allow_ingest_behind_));
   }
 
   const ImmutableOptions& ioptions_;
@@ -437,7 +438,6 @@ class UniversalCompactionBuilder {
   VersionStorageInfo* vstorage_;
   UniversalCompactionPicker* picker_;
   LogBuffer* log_buffer_;
-  bool require_max_output_level_;
   // Optional earliest snapshot at time of compaction picking. This is only
   // provided if the column family doesn't enable user-defined timestamps.
   // And this information is only passed to `Compaction` picked by deletion
@@ -448,6 +448,8 @@ class UniversalCompactionBuilder {
   // marked for compaction. This is only populated when snapshot info is
   // populated.
   std::map<uint64_t, size_t> file_marked_for_compaction_to_sorted_run_index_;
+  bool require_max_output_level_;
+  bool allow_ingest_behind_;
 
   std::vector<UniversalCompactionBuilder::SortedRun> CalculateSortedRuns(
       const VersionStorageInfo& vstorage, int last_level,
@@ -733,8 +735,7 @@ bool UniversalCompactionBuilder::ShouldSkipMarkedFile(
 Compaction* UniversalCompactionBuilder::PickCompaction() {
   const int kLevel0 = 0;
   score_ = vstorage_->CompactionScore(kLevel0);
-  const int max_output_level =
-      vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind);
+  const int max_output_level = vstorage_->MaxOutputLevel(allow_ingest_behind_);
   const int file_num_compaction_trigger =
       mutable_cf_options_.level0_file_num_compaction_trigger;
   const unsigned int ratio =
@@ -781,8 +782,7 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
         "UniversalCompactionBuilder::PickCompaction:Return", nullptr);
     return nullptr;
   }
-  assert(c->output_level() <=
-         vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind));
+  assert(c->output_level() <= vstorage_->MaxOutputLevel(allow_ingest_behind_));
   assert(MeetsOutputLevelRequirements(c->output_level()));
 
   if (mutable_cf_options_.compaction_options_universal.allow_trivial_move ==
@@ -1024,8 +1024,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSortedRuns(
   int start_level = sorted_runs_[start_index].level;
   int output_level;
   // last level is reserved for the files ingested behind
-  int max_output_level =
-      vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind);
+  int max_output_level = vstorage_->MaxOutputLevel(allow_ingest_behind_);
   if (first_index_after == sorted_runs_.size()) {
     output_level = max_output_level;
   } else if (sorted_runs_[first_index_after].level == 0) {
@@ -1517,8 +1516,7 @@ Compaction* UniversalCompactionBuilder::PickDeleteTriggeredCompaction() {
       return nullptr;
     }
 
-    int max_output_level =
-        vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind);
+    int max_output_level = vstorage_->MaxOutputLevel(allow_ingest_behind_);
     // Pick the first non-empty level after the start_level
     for (output_level = start_level + 1; output_level <= max_output_level;
          output_level++) {
@@ -1621,8 +1619,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionWithSortedRunRange(
   uint32_t path_id =
       GetPathId(ioptions_, mutable_cf_options_, estimated_total_size);
   int start_level = sorted_runs_[start_index].level;
-  int max_output_level =
-      vstorage_->MaxOutputLevel(ioptions_.allow_ingest_behind);
+  int max_output_level = vstorage_->MaxOutputLevel(allow_ingest_behind_);
   std::vector<CompactionInputFiles> inputs(max_output_level + 1);
   for (size_t i = 0; i < inputs.size(); ++i) {
     inputs[i].level = start_level + static_cast<int>(i);

db/compaction/compaction_service_test.cc

@@ -461,6 +461,97 @@ TEST_F(CompactionServiceTest, ManualCompaction) {
   ASSERT_EQ(handles_[1]->GetName(), info.cf_name);
 }
 
+TEST_F(CompactionServiceTest, StandaloneDeleteRangeTombstoneOptimization) {
+  Options options = CurrentOptions();
+  options.compaction_style = CompactionStyle::kCompactionStyleUniversal;
+  ReopenWithCompactionService(&options);
+
+  size_t num_files_after_filtered = 0;
+  SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::MakeInputIterator:NewCompactionMergingIterator",
+      [&](void* arg) {
+        num_files_after_filtered = *static_cast<size_t*>(arg);
+      });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  std::vector<std::string> files;
+  {
+    // Writes first version of data in range partitioned files.
+    SstFileWriter sst_file_writer(EnvOptions(), options);
+    std::string file1 = dbname_ + "file1.sst";
+    ASSERT_OK(sst_file_writer.Open(file1));
+    ASSERT_OK(sst_file_writer.Put("a", "a1"));
+    ASSERT_OK(sst_file_writer.Put("b", "b1"));
+    ExternalSstFileInfo file1_info;
+    ASSERT_OK(sst_file_writer.Finish(&file1_info));
+    files.push_back(std::move(file1));
+
+    std::string file2 = dbname_ + "file2.sst";
+    ASSERT_OK(sst_file_writer.Open(file2));
+    ASSERT_OK(sst_file_writer.Put("x", "x1"));
+    ASSERT_OK(sst_file_writer.Put("y", "y1"));
+    ExternalSstFileInfo file2_info;
+    ASSERT_OK(sst_file_writer.Finish(&file2_info));
+    files.push_back(std::move(file2));
+  }
+
+  IngestExternalFileOptions ifo;
+  ASSERT_OK(db_->IngestExternalFile(files, ifo));
+  ASSERT_EQ(Get("a"), "a1");
+  ASSERT_EQ(Get("b"), "b1");
+  ASSERT_EQ(Get("x"), "x1");
+  ASSERT_EQ(Get("y"), "y1");
+  ASSERT_EQ(2, NumTableFilesAtLevel(6));
+
+  auto my_cs = GetCompactionService();
+  uint64_t comp_num = my_cs->GetCompactionNum();
+
+  {
+    // Atomically delete old version of data with one range delete file.
+    // And a new batch of range partitioned files with new version of data.
+    files.clear();
+    SstFileWriter sst_file_writer(EnvOptions(), options);
+    std::string file2 = dbname_ + "file2.sst";
+    ASSERT_OK(sst_file_writer.Open(file2));
+    ASSERT_OK(sst_file_writer.DeleteRange("a", "z"));
+    ExternalSstFileInfo file2_info;
+    ASSERT_OK(sst_file_writer.Finish(&file2_info));
+    files.push_back(std::move(file2));
+
+    std::string file3 = dbname_ + "file3.sst";
+    ASSERT_OK(sst_file_writer.Open(file3));
+    ASSERT_OK(sst_file_writer.Put("a", "a2"));
+    ASSERT_OK(sst_file_writer.Put("b", "b2"));
+    ExternalSstFileInfo file3_info;
+    ASSERT_OK(sst_file_writer.Finish(&file3_info));
+    files.push_back(std::move(file3));
+
+    std::string file4 = dbname_ + "file4.sst";
+    ASSERT_OK(sst_file_writer.Open(file4));
+    ASSERT_OK(sst_file_writer.Put("x", "x2"));
+    ASSERT_OK(sst_file_writer.Put("y", "y2"));
+    ExternalSstFileInfo file4_info;
+    ASSERT_OK(sst_file_writer.Finish(&file4_info));
+    files.push_back(std::move(file4));
+  }
+
+  ASSERT_OK(db_->IngestExternalFile(files, ifo));
+  ASSERT_OK(db_->WaitForCompact(WaitForCompactOptions()));
+  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
+
+  CompactionServiceResult result;
+  my_cs->GetResult(&result);
+  ASSERT_OK(result.status);
+  ASSERT_TRUE(result.stats.is_manual_compaction);
+  ASSERT_TRUE(result.stats.is_remote_compaction);
+
+  ASSERT_EQ(num_files_after_filtered, 1);
+
+  Close();
+  SyncPoint::GetInstance()->DisableProcessing();
+}
+
 TEST_F(CompactionServiceTest, CompactionOutputFileIOError) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
@@ -793,6 +884,79 @@ TEST_F(CompactionServiceTest, VerifyInputRecordCount) {
   SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
+TEST_F(CompactionServiceTest, EmptyResult) {
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  ReopenWithCompactionService(&options);
+  GenerateTestData();
+
+  auto my_cs = GetCompactionService();
+
+  uint64_t comp_num = my_cs->GetCompactionNum();
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_GE(my_cs->GetCompactionNum(), comp_num + 1);
+
+  // Delete range to cover entire range
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), "key", "keyz"));
+  ASSERT_OK(Flush());
+
+  // In this unit test, both remote compaction and primary db instance are
+  // running in the same process, so NewFileNumber will never have a collision.
+  // In the real-world remote compactions, when the compaction is indeed running
+  // in another process, this is not going to be the case.
+  // To simulate the SST file with the same name created in the tmp directory,
+  // override the file number in remote compaction to re-use old SST file
+  // number.
+  bool need_to_override_file_number = false;
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImplSecondary::OpenAndCompact::BeforeLoadingOptions:0",
+      [&](void*) { need_to_override_file_number = true; });
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::OpenCompactionOutputFile::NewFileNumber",
+      [&](void* file_number) {
+        if (need_to_override_file_number) {
+          auto n = static_cast<uint64_t*>(file_number);
+          ColumnFamilyMetaData cf_meta;
+          db_->GetColumnFamilyMetaData(&cf_meta);
+          for (const auto& level : cf_meta.levels) {
+            for (const auto& file : level.files) {
+              // Use one of the existing file name
+              *n = test::GetFileNumber(file.name);
+              need_to_override_file_number = false;
+              return;
+            }
+          }
+        }
+      });
+
+  // Inject failure, so that the remote compaction fails after
+  // ProcessKeyValueCompaction()
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImplSecondary::CompactWithoutInstallation::End", [&](void* status) {
+        // override job status
+        auto s = static_cast<Status*>(status);
+        *s = Status::Aborted("MyTestCompactionService failed to compact!");
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  // Compaction should fail and SST files in the primary db should exist
+  {
+    ASSERT_NOK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+    ColumnFamilyMetaData meta;
+    db_->GetColumnFamilyMetaData(&meta);
+    for (const auto& level : meta.levels) {
+      for (const auto& file : level.files) {
+        std::string fname = file.db_path + "/" + file.name;
+        ASSERT_OK(db_->GetEnv()->FileExists(fname));
+      }
+    }
+  }
+  Close();
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 TEST_F(CompactionServiceTest, CorruptedOutput) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;

db/db_compaction_test.cc

@@ -74,6 +74,43 @@ class CompactionStatsCollector : public EventListener {
   std::vector<std::atomic<int>> compaction_completed_;
 };
 
+class DeletionTriggeredCompactionWithMinFileSizeTestListener
+    : public EventListener {
+ public:
+  explicit DeletionTriggeredCompactionWithMinFileSizeTestListener(
+      uint64_t min_file_size)
+      : min_file_size_(min_file_size) {}
+
+  void OnCompactionBegin(DB* db, const CompactionJobInfo& ci) override {
+    ASSERT_EQ(ci.compaction_reason,
+              CompactionReason::kFilesMarkedForCompaction);
+
+    auto env = db->GetEnv();
+    const std::vector<DbPath>& db_paths = db->GetOptions().db_paths;
+    for (const auto& file : ci.input_file_infos) {
+      uint64_t file_size = GetSstFileSize(env, db_paths, file.file_number);
+
+      // Assert that the file size respects the minimum threshold
+      ASSERT_GE(file_size, min_file_size_);
+    }
+  }
+
+ private:
+  static uint64_t GetSstFileSize(Env* env, const std::vector<DbPath>& db_paths,
+                                 uint64_t file_number) {
+    uint32_t path_id = 0;  // since only one path
+    std::string sst_file_name = TableFileName(db_paths, file_number, path_id);
+    uint64_t file_size = 0;
+    Status s = env->GetFileSize(sst_file_name, &file_size);
+    if (!s.ok()) {
+      return 0;
+    }
+    return file_size;
+  }
+
+  uint64_t min_file_size_;
+};
+
 class DBCompactionTest : public DBTestBase {
  public:
   DBCompactionTest()
@@ -1371,6 +1408,89 @@ TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) {
   } while (ChangeOptions());
 }
 
+TEST_F(DBCompactionTest, CompactionWithDeletionsAndMinFileSize) {
+  const uint64_t kMinFileSize = 32 * 1024;  // 32KB
+  const int kDeletionTriggerCount = 50;
+  const int kInitialKeyCount = 100;
+  const int kAdditionalKeyCount = 50;
+  const int kValueSize = 1024;
+  const int kSmallValueSize = 512;
+  const int kSeed = 301;
+
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleLevel;
+  options.write_buffer_size = 1024 * 1024;  // 1MB
+  options.level0_file_num_compaction_trigger = 100;
+
+  options.table_properties_collector_factories = {
+      NewCompactOnDeletionCollectorFactory(
+          kInitialKeyCount /* sliding window size */, kDeletionTriggerCount,
+          0.5 /* deletion ratio */, kMinFileSize)};
+  auto listener =
+      new DeletionTriggeredCompactionWithMinFileSizeTestListener(kMinFileSize);
+  options.listeners.emplace_back(listener);
+
+  DestroyAndReopen(options);
+  Random rnd(kSeed);
+
+  // Create a large file that will be subject to DTC later
+  for (int i = 0; i < kInitialKeyCount; i++) {
+    ASSERT_OK(Put(Key(i), rnd.RandomString(kValueSize)));
+  }
+  ASSERT_OK(Flush());
+
+  std::vector<LiveFileMetaData> initial_metadata;
+  db_->GetLiveFilesMetaData(&initial_metadata);
+  ASSERT_EQ(initial_metadata.size(), 1);
+
+  // Create small files that should not trigger compaction
+  ASSERT_OK(Put("small_file_key1", rnd.RandomString(kSmallValueSize)));
+  ASSERT_OK(Put("small_file_key2", rnd.RandomString(kSmallValueSize)));
+  ASSERT_OK(Flush());
+  ASSERT_OK(Delete("small_file_key1"));
+  ASSERT_OK(Flush());
+
+  // Create a file with enough deletions and size to trigger DTC
+  // Delete keys from the large file to reach deletion threshold
+  for (int i = 0; i < kDeletionTriggerCount; i++) {
+    ASSERT_OK(Delete(Key(i)));
+  }
+
+  // Add new keys to ensure the deletion file meets the min_file_size threshold
+  for (int i = kInitialKeyCount; i < kInitialKeyCount + kAdditionalKeyCount;
+       i++) {
+    ASSERT_OK(Put(Key(i), rnd.RandomString(kValueSize)));
+  }
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+
+  // Verify file count after compaction
+  ASSERT_EQ(NumTableFilesAtLevel(0), 2);  // Small file and deletion file
+  ASSERT_EQ(NumTableFilesAtLevel(1), 1);  // Compacted large file
+
+  // Verify deleted keys are gone
+  for (int i = 0; i < kDeletionTriggerCount; i++) {
+    std::string value;
+    ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound());
+  }
+
+  // Verify non-deleted keys from large file are still accessible
+  for (int i = kDeletionTriggerCount; i < kInitialKeyCount; i++) {
+    std::string value;
+    ASSERT_OK(db_->Get(ReadOptions(), Key(i), &value));
+    ASSERT_EQ(value.size(), kValueSize);
+  }
+
+  // Verify new keys are accessible
+  for (int i = kInitialKeyCount; i < kInitialKeyCount + kAdditionalKeyCount;
+       i++) {
+    std::string value;
+    ASSERT_OK(db_->Get(ReadOptions(), Key(i), &value));
+    ASSERT_EQ(value.size(), kValueSize);
+  }
+}
+
 TEST_P(DBCompactionTestWithParam, TrivialMoveOneFile) {
   int32_t trivial_move = 0;
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
@@ -6748,7 +6868,11 @@ INSTANTIATE_TEST_CASE_P(RoundRobinSubcompactionsAgainstPressureToken,
                         RoundRobinSubcompactionsAgainstPressureToken,
                         testing::Bool());
 
-TEST_P(RoundRobinSubcompactionsAgainstResources, SubcompactionsUsingResources) {
+// FIXME: the test is flaky and failing the assertion
+// ASSERT_EQ(actual_reserved_threads, expected_reserved_threads);
+// It's likely a test set up issue, fix if we are to use RoubdRobin compaction.
+TEST_P(RoundRobinSubcompactionsAgainstResources,
+       DISABLED_SubcompactionsUsingResources) {
   const int kKeysPerBuffer = 200;
   Options options = CurrentOptions();
   options.num_levels = 4;
@@ -7004,6 +7128,70 @@ TEST_F(DBCompactionTest, PartialManualCompaction) {
   ASSERT_OK(dbfull()->CompactRange(cro, nullptr, nullptr));
 }
 
+TEST_F(DBCompactionTest, ConcurrentFIFOPickingSameFileBug) {
+  Options opts = CurrentOptions();
+  opts.compaction_style = CompactionStyle::kCompactionStyleLevel;
+  opts.num_levels = 3;
+  opts.disable_auto_compactions = true;
+  opts.max_background_jobs = 3;
+
+  DestroyAndReopen(opts);
+
+  ASSERT_OK(Put("k1", "v1"));
+  ASSERT_OK(Flush());
+
+  // Create a non-L0 SST file for multi-level FIFO size-based compaction later
+  MoveFilesToLevel(2);
+
+  Options opts_new(opts);
+  opts_new.compaction_style = CompactionStyle::kCompactionStyleFIFO;
+  opts_new.max_open_files = -1;
+  // Set a low threshold to trigger multi-level size-based compaction
+  opts_new.compaction_options_fifo.max_table_files_size = 1;
+
+  Reopen(opts_new);
+
+  const CompactRangeOptions cro;
+  const Slice begin_key("k1");
+  const Slice end_key("k2");
+
+  std::unique_ptr<port::Thread> concurrent_compaction;
+
+  bool within_first_compaction = true;
+  SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::LogAndApply:WriteManifestStart", [&](void* /*arg*/) {
+        if (!within_first_compaction) {
+          return;
+        }
+        within_first_compaction = false;
+
+        // To allow the second/concurrent compaction to still see the non-L0
+        // SST file and coerce the bug of picking that file
+        SyncPoint::GetInstance()->LoadDependency({
+            {"DBImpl::BackgroundCompaction:BeforeCompaction",
+             "VersionSet::LogAndApply:WriteManifest"},
+        });
+
+        concurrent_compaction.reset(new port::Thread([&]() {
+          // Before the fix, the second CompactRange() will either fail the
+          // assertion of double file picking `being_compacted !=
+          // inputs_[i][j]->being_compacted` in debug mode or cause LSM shape
+          // corruption "Cannot delete table file XXX from level 2 since it is
+          // not in the LSM tree" in release mode
+          Status s = db_->CompactRange(cro, &begin_key, &end_key);
+          ASSERT_OK(s);
+        }));
+      });
+
+  SyncPoint::GetInstance()->EnableProcessing();
+  Status s = db_->CompactRange(cro, &begin_key, &end_key);
+  SyncPoint::GetInstance()->DisableProcessing();
+
+  ASSERT_OK(s);
+
+  concurrent_compaction->join();
+}
+
 TEST_F(DBCompactionTest, ManualCompactionFailsInReadOnlyMode) {
   // Regression test for bug where manual compaction hangs forever when the DB
   // is in read-only mode. Verify it now at least returns, despite failing.

db/db_impl/db_impl.cc

@@ -3835,7 +3835,7 @@ bool DBImpl::KeyMayExist(const ReadOptions& read_options,
 
 std::unique_ptr<MultiScan> DBImpl::NewMultiScan(
     const ReadOptions& _read_options, ColumnFamilyHandle* column_family,
-    const std::vector<ScanOptions>& scan_opts) {
+    const MultiScanArgs& scan_opts) {
   std::unique_ptr<MultiScan> ms_iter = std::make_unique<MultiScan>(
       _read_options, scan_opts, this, column_family);
   return ms_iter;
@@ -4345,7 +4345,7 @@ void DBImpl::ReleaseSnapshot(const Snapshot* s) {
     CfdList cf_scheduled;
     if (oldest_snapshot > bottommost_files_mark_threshold_) {
       for (auto* cfd : *versions_->GetColumnFamilySet()) {
-        if (!cfd->ioptions().allow_ingest_behind) {
+        if (!cfd->AllowIngestBehind()) {
           cfd->current()->storage_info()->UpdateOldestSnapshot(
               oldest_snapshot, /*allow_ingest_behind=*/false);
           if (!cfd->current()
@@ -4365,8 +4365,7 @@ void DBImpl::ReleaseSnapshot(const Snapshot* s) {
       // inaccurate.
       SequenceNumber new_bottommost_files_mark_threshold = kMaxSequenceNumber;
       for (auto* cfd : *versions_->GetColumnFamilySet()) {
-        if (CfdListContains(cf_scheduled, cfd) ||
-            cfd->ioptions().allow_ingest_behind) {
+        if (CfdListContains(cf_scheduled, cfd) || cfd->AllowIngestBehind()) {
           continue;
         }
         new_bottommost_files_mark_threshold = std::min(
@@ -5761,10 +5760,6 @@ Status DBImpl::IngestExternalFiles(
   for (const auto& arg : args) {
     const IngestExternalFileOptions& ingest_opts = arg.options;
     if (ingest_opts.ingest_behind) {
-      if (!immutable_db_options_.allow_ingest_behind) {
-        return Status::InvalidArgument(
-            "can't ingest_behind file in DB with allow_ingest_behind=false");
-      }
       auto ucmp = arg.column_family->GetComparator();
       assert(ucmp);
       if (ucmp->timestamp_size() > 0) {
@@ -5772,6 +5767,14 @@ Status DBImpl::IngestExternalFiles(
             "Column family with user-defined "
             "timestamps enabled doesn't support ingest behind.");
       }
+
+      if (!static_cast<ColumnFamilyHandleImpl*>(arg.column_family)
+               ->cfd()
+               ->AllowIngestBehind()) {
+        return Status::InvalidArgument(
+            "Can't ingest_behind file in ColumnFamily %s with "
+            "cf_allow_ingest_behind=false");
+      }
     }
     if (arg.atomic_replace_range.has_value()) {
       if (ingest_opts.ingest_behind) {
@@ -6006,18 +6009,19 @@ Status DBImpl::IngestExternalFiles(
       // mutex when persisting MANIFEST file, and the snapshots taken during
       // that period will not be stable if VersionSet last seqno is updated
       // before LogAndApply.
-      int consumed_seqno_count =
-          ingestion_jobs[0].ConsumedSequenceNumbersCount();
+      SequenceNumber max_assigned_seqno =
+          ingestion_jobs[0].MaxAssignedSequenceNumber();
       for (size_t i = 1; i != num_cfs; ++i) {
-        consumed_seqno_count =
-            std::max(consumed_seqno_count,
-                     ingestion_jobs[i].ConsumedSequenceNumbersCount());
+        max_assigned_seqno = std::max(
+            max_assigned_seqno, ingestion_jobs[i].MaxAssignedSequenceNumber());
       }
-      if (consumed_seqno_count > 0) {
+      if (max_assigned_seqno > 0) {
         const SequenceNumber last_seqno = versions_->LastSequence();
-        versions_->SetLastAllocatedSequence(last_seqno + consumed_seqno_count);
-        versions_->SetLastPublishedSequence(last_seqno + consumed_seqno_count);
-        versions_->SetLastSequence(last_seqno + consumed_seqno_count);
+        if (max_assigned_seqno > last_seqno) {
+          versions_->SetLastAllocatedSequence(max_assigned_seqno);
+          versions_->SetLastPublishedSequence(max_assigned_seqno);
+          versions_->SetLastSequence(max_assigned_seqno);
+        }
       }
     }
 

db/db_impl/db_impl.h

@@ -386,7 +386,7 @@ class DBImpl : public DB {
   using DB::NewMultiScan;
   std::unique_ptr<MultiScan> NewMultiScan(
       const ReadOptions& _read_options, ColumnFamilyHandle* column_family,
-      const std::vector<ScanOptions>& scan_opts) override;
+      const MultiScanArgs& scan_opts) override;
 
   const Snapshot* GetSnapshot() override;
   void ReleaseSnapshot(const Snapshot* snapshot) override;
@@ -1388,6 +1388,9 @@ class DBImpl : public DB {
   // logs_, cur_wal_number_. Refer to the definition of each variable below for
   // more description.
   //
+  // Protects access to most ColumnFamilyData methods, see more in comment for
+  // each method.
+  //
   // `mutex_` can be a hot lock in some workloads, so it deserves dedicated
   // cachelines.
   mutable CacheAlignedInstrumentedMutex mutex_;

db/db_impl/db_impl_compaction_flush.cc

@@ -1111,8 +1111,7 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
       cfd->NumberLevels() > 1) {
     // Always compact all files together.
     final_output_level = cfd->NumberLevels() - 1;
-    // if bottom most level is reserved
-    if (immutable_db_options_.allow_ingest_behind) {
+    if (cfd->AllowIngestBehind()) {
       final_output_level--;
     }
     s = RunManualCompaction(cfd, ColumnFamilyData::kCompactAllLevels,
@@ -1460,7 +1459,7 @@ Status DBImpl::CompactFilesImpl(
     }
   }
 
-  if (cfd->ioptions().allow_ingest_behind &&
+  if (cfd->AllowIngestBehind() &&
       output_level >= cfd->ioptions().num_levels - 1) {
     return Status::InvalidArgument(
         "Exceed the maximum output level defined by "
@@ -1500,7 +1499,7 @@ Status DBImpl::CompactFilesImpl(
 
   std::unique_ptr<Compaction> c;
   assert(cfd->compaction_picker());
-  c.reset(cfd->compaction_picker()->CompactFiles(
+  c.reset(cfd->compaction_picker()->PickCompactionForCompactFiles(
       compact_options, input_files, output_level, version->storage_info(),
       cfd->GetLatestMutableCFOptions(), mutable_db_options_, output_path_id));
   // we already sanitized the set of input files and checked for conflicts
@@ -4155,6 +4154,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
                      ->current()
                      ->storage_info()
                      ->MaxOutputLevel(
+                         c->immutable_options().cf_allow_ingest_behind ||
                          immutable_db_options_.allow_ingest_behind)) &&
              env_->GetBackgroundThreads(Env::Priority::BOTTOM) > 0) {
     assert(thread_pri == Env::Priority::LOW);
@@ -4660,7 +4660,7 @@ void DBImpl::InstallSuperVersionAndScheduleWork(
   bottommost_files_mark_threshold_ = kMaxSequenceNumber;
   standalone_range_deletion_files_mark_threshold_ = kMaxSequenceNumber;
   for (auto* my_cfd : *versions_->GetColumnFamilySet()) {
-    if (!my_cfd->ioptions().allow_ingest_behind) {
+    if (!my_cfd->AllowIngestBehind()) {
       bottommost_files_mark_threshold_ = std::min(
           bottommost_files_mark_threshold_,
           my_cfd->current()->storage_info()->bottommost_files_mark_threshold());

db/db_impl/db_impl_open.cc

@@ -599,7 +599,7 @@ Status DBImpl::Recover(
         // allow_ingest_behind does not support Level Compaction,
         // and per_key_placement can have infinite compaction loop for Level
         // Compaction. Adjust to_level here just to be safe.
-        if (cfd->ioptions().allow_ingest_behind ||
+        if (cfd->AllowIngestBehind() ||
             moptions.preclude_last_level_data_seconds > 0) {
           to_level -= 1;
         }
@@ -1755,8 +1755,12 @@ Status DBImpl::MaybeHandleStopReplayForCorruptionForInconsistency(
         ROCKS_LOG_ERROR(immutable_db_options_.info_log,
                         "Column family inconsistency: SST file contains data"
                         " beyond the point of corruption.");
-        status = Status::Corruption("SST file is ahead of WALs in CF " +
-                                    cfd->GetName());
+        status = Status::Corruption(
+            "Column family inconsistency: SST file contains data"
+            " beyond the point of corruption in CF " +
+            cfd->GetName() +
+            ". WAL recovery stopped at corruption point, but SST files"
+            " contain newer data.");
         return status;
       }
     }

db/db_impl/db_impl_secondary.cc

@@ -848,7 +848,6 @@ Status DBImplSecondary::CompactWithoutInstallation(
 
   VersionStorageInfo* vstorage = version->storage_info();
 
-  // Use comp_options to reuse some CompactFiles functions
   CompactionOptions comp_options;
   comp_options.compression = kDisableCompressionOption;
   comp_options.output_file_size_limit = MaxFileSizeForLevel(
@@ -867,13 +866,27 @@ Status DBImplSecondary::CompactWithoutInstallation(
     return s;
   }
 
+  const int job_id = next_job_id_.fetch_add(1);
+  JobContext job_context(job_id, true /*create_superversion*/);
+  std::vector<SequenceNumber> snapshots = input.snapshots;
+
+  // TODO - snapshot_checker support in Remote Compaction
+  job_context.InitSnapshotContext(/*checker=*/nullptr,
+                                  /*managed_snapshot=*/nullptr,
+                                  kMaxSequenceNumber, std::move(snapshots));
+
+  // TODO - consider serializing the entire Compaction object and using it as
+  // input instead of recreating it in the remote worker
   std::unique_ptr<Compaction> c;
   assert(cfd->compaction_picker());
-  c.reset(cfd->compaction_picker()->CompactFiles(
+  c.reset(cfd->compaction_picker()->PickCompactionForCompactFiles(
       comp_options, input_files, input.output_level, vstorage,
-      cfd->GetLatestMutableCFOptions(), mutable_db_options_, 0));
+      cfd->GetLatestMutableCFOptions(), mutable_db_options_, 0,
+      /*earliest_snapshot=*/job_context.snapshot_seqs.empty()
+          ? kMaxSequenceNumber
+          : job_context.snapshot_seqs.front(),
+      job_context.snapshot_checker));
   assert(c != nullptr);
-
   c->FinalizeInputInfo(version);
 
   // Create output directory if it's not existed yet
@@ -886,11 +899,6 @@ Status DBImplSecondary::CompactWithoutInstallation(
   LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL,
                        immutable_db_options_.info_log.get());
 
-  const int job_id = next_job_id_.fetch_add(1);
-  JobContext job_context(0, true /*create_superversion*/);
-  std::vector<SequenceNumber> snapshots = input.snapshots;
-  job_context.InitSnapshotContext(nullptr, nullptr, kMaxSequenceNumber,
-                                  std::move(snapshots));
   // use primary host's db_id for running the compaction, but db_session_id is
   // using the local one, which is to make sure the unique id is unique from
   // the remote compactors. Because the id is generated from db_id,

db/db_iter.h

@@ -240,8 +240,8 @@ class DBIter final : public Iterator {
 
   bool PrepareValue() override;
 
-  void Prepare(const std::vector<ScanOptions>& scan_opts) override {
-    std::optional<std::vector<ScanOptions>> new_scan_opts;
+  void Prepare(const MultiScanArgs& scan_opts) override {
+    std::optional<MultiScanArgs> new_scan_opts;
     new_scan_opts.emplace(scan_opts);
     scan_opts_.swap(new_scan_opts);
     if (!scan_opts.empty()) {
@@ -505,7 +505,7 @@ class DBIter final : public Iterator {
   const Slice* const timestamp_lb_;
   const size_t timestamp_size_;
   std::string saved_timestamp_;
-  std::optional<std::vector<ScanOptions>> scan_opts_;
+  std::optional<MultiScanArgs> scan_opts_;
   ReadOnlyMemTable* const active_mem_;
   SequenceNumber memtable_seqno_lb_;
   uint32_t memtable_op_scan_flush_trigger_;

db/db_iterator_test.cc

@@ -4142,13 +4142,17 @@ TEST_P(DBIteratorTest, AverageMemtableOpsScanFlushTriggerByOverwrites) {
   ASSERT_EQ(1, NumTableFilesAtLevel(0));
 }
 
-class DBMultiScanIteratorTest : public DBTestBase {
+class DBMultiScanIteratorTest : public DBTestBase,
+                                public ::testing::WithParamInterface<bool> {
  public:
   DBMultiScanIteratorTest()
       : DBTestBase("db_multi_scan_iterator_test", /*env_do_fsync=*/true) {}
 };
 
-TEST_F(DBMultiScanIteratorTest, BasicTest) {
+INSTANTIATE_TEST_CASE_P(DBMultiScanIteratorTest, DBMultiScanIteratorTest,
+                        ::testing::Bool());
+
+TEST_P(DBMultiScanIteratorTest, BasicTest) {
   // Create a file
   for (int i = 0; i < 100; ++i) {
     std::stringstream ss;
@@ -4159,9 +4163,10 @@ TEST_F(DBMultiScanIteratorTest, BasicTest) {
 
   std::vector<std::string> key_ranges({"k03", "k10", "k25", "k50"});
   ReadOptions ro;
-  std::vector<ScanOptions> scan_options(
-      {ScanOptions(key_ranges[0], key_ranges[1]),
-       ScanOptions(key_ranges[2], key_ranges[3])});
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
   ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
   std::unique_ptr<MultiScan> iter =
       dbfull()->NewMultiScan(ro, cfh, scan_options);
@@ -4190,7 +4195,10 @@ TEST_F(DBMultiScanIteratorTest, BasicTest) {
 
   // Test the overlapping scan case
   key_ranges[1] = "k30";
-  scan_options[0] = ScanOptions(key_ranges[0], key_ranges[1]);
+  scan_options = MultiScanArgs(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+
   iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
   try {
     int idx = 0;
@@ -4216,8 +4224,9 @@ TEST_F(DBMultiScanIteratorTest, BasicTest) {
   iter.reset();
 
   // Test the no limit scan case
-  scan_options[0] = ScanOptions(key_ranges[0]);
-  scan_options[1] = ScanOptions(key_ranges[2]);
+  scan_options = MultiScanArgs(BytewiseComparator());
+  scan_options.insert(key_ranges[0]);
+  scan_options.insert(key_ranges[2]);
   iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
   try {
     int idx = 0;
@@ -4245,7 +4254,7 @@ TEST_F(DBMultiScanIteratorTest, BasicTest) {
   iter.reset();
 }
 
-TEST_F(DBMultiScanIteratorTest, MixedBoundsTest) {
+TEST_P(DBMultiScanIteratorTest, MixedBoundsTest) {
   // Create a file
   for (int i = 0; i < 100; ++i) {
     std::stringstream ss;
@@ -4257,9 +4266,11 @@ TEST_F(DBMultiScanIteratorTest, MixedBoundsTest) {
   std::vector<std::string> key_ranges(
       {"k03", "k10", "k25", "k50", "k75", "k90"});
   ReadOptions ro;
-  std::vector<ScanOptions> scan_options(
-      {ScanOptions(key_ranges[0], key_ranges[1]), ScanOptions(key_ranges[2]),
-       ScanOptions(key_ranges[4], key_ranges[5])});
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2]);
+  scan_options.insert(key_ranges[4], key_ranges[5]);
   ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
   std::unique_ptr<MultiScan> iter =
       dbfull()->NewMultiScan(ro, cfh, scan_options);
@@ -4268,13 +4279,15 @@ TEST_F(DBMultiScanIteratorTest, MixedBoundsTest) {
     int count = 0;
     for (auto range : *iter) {
       for (auto it : range) {
-        ASSERT_GE(it.first.ToString().compare(
-                      scan_options[idx].range.start->ToString()),
-                  0);
-        if (scan_options[idx].range.limit) {
-          ASSERT_LT(it.first.ToString().compare(
-                        scan_options[idx].range.limit->ToString()),
-                    0);
+        ASSERT_GE(
+            it.first.ToString().compare(
+                scan_options.GetScanRanges()[idx].range.start->ToString()),
+            0);
+        if (scan_options.GetScanRanges()[idx].range.limit) {
+          ASSERT_LT(
+              it.first.ToString().compare(
+                  scan_options.GetScanRanges()[idx].range.limit->ToString()),
+              0);
         }
         count++;
       }
@@ -4291,23 +4304,25 @@ TEST_F(DBMultiScanIteratorTest, MixedBoundsTest) {
     abort();
   }
   iter.reset();
-
-  scan_options[0] = ScanOptions(key_ranges[0]);
-  scan_options[1] = ScanOptions(key_ranges[2], key_ranges[3]);
-  scan_options[2] = ScanOptions(key_ranges[4]);
+  scan_options = MultiScanArgs(BytewiseComparator());
+  scan_options.insert(key_ranges[0]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+  scan_options.insert(key_ranges[4]);
   iter = dbfull()->NewMultiScan(ro, cfh, scan_options);
   try {
     int idx = 0;
     int count = 0;
     for (auto range : *iter) {
       for (auto it : range) {
-        ASSERT_GE(it.first.ToString().compare(
-                      scan_options[idx].range.start->ToString()),
-                  0);
-        if (scan_options[idx].range.limit) {
-          ASSERT_LT(it.first.ToString().compare(
-                        scan_options[idx].range.limit->ToString()),
-                    0);
+        ASSERT_GE(
+            it.first.ToString().compare(
+                scan_options.GetScanRanges()[idx].range.start->ToString()),
+            0);
+        if (scan_options.GetScanRanges()[idx].range.limit) {
+          ASSERT_LT(
+              it.first.ToString().compare(
+                  scan_options.GetScanRanges()[idx].range.limit->ToString()),
+              0);
         }
         count++;
       }
@@ -4325,6 +4340,41 @@ TEST_F(DBMultiScanIteratorTest, MixedBoundsTest) {
   }
   iter.reset();
 }
+
+TEST_P(DBMultiScanIteratorTest, RangeAcrossFiles) {
+  auto options = CurrentOptions();
+  options.target_file_size_base = 100 << 10;  // 20KB
+  options.compaction_style = kCompactionStyleUniversal;
+  options.num_levels = 50;
+  options.compression = kNoCompression;
+  DestroyAndReopen(options);
+
+  auto rnd = Random::GetTLSInstance();
+  // Write ~200KB data
+  for (int i = 0; i < 100; ++i) {
+    ASSERT_OK(Put(Key(i), rnd->RandomString(2 << 10)));
+  }
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(db_->CompactRange({}, nullptr, nullptr));
+  ASSERT_EQ(2, NumTableFilesAtLevel(49));
+  std::vector<std::string> key_ranges({Key(10), Key(90)});
+  ReadOptions ro;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+  int i = 10;
+  for (auto range : *iter) {
+    for (auto it : range) {
+      ASSERT_EQ(it.first.ToString(), Key(i));
+      ++i;
+    }
+  }
+  ASSERT_EQ(i, 90);
+}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_sst_test.cc

@@ -1937,6 +1937,70 @@ TEST_F(DBSSTTest, DBWithSFMForBlobFilesAtomicFlush) {
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
+TEST_F(DBSSTTest, SstGetFileSizeFails) {
+  // Build an SST file
+  ASSERT_OK(Put("x", "zaphod"));
+  ASSERT_OK(Flush());
+  std::vector<LiveFileMetaData> metadata;
+  db_->GetLiveFilesMetaData(&metadata);
+  ASSERT_EQ(1U, metadata.size());
+  std::string filename = dbname_ + metadata[0].name;
+
+  // Prepare for fault injection
+  std::shared_ptr<FaultInjectionTestFS> fault_fs =
+      std::make_shared<FaultInjectionTestFS>(
+          CurrentOptions().env->GetFileSystem());
+  std::unique_ptr<Env> fault_fs_env(NewCompositeEnv(fault_fs));
+  Options options = CurrentOptions();
+  options.env = fault_fs_env.get();
+  options.paranoid_checks = false;  // don't check file sizes on open
+
+  for (int i = 0; i < 4; i++) {
+    SCOPED_TRACE("Iteration = " + std::to_string(i));
+    fault_fs->SetFailRandomAccessGetFileSizeSst(false);
+    fault_fs->SetFailFilesystemGetFileSizeSst(false);
+    Close();
+
+    if (i == 1) {
+      // Just FSRandomAccessFile::GetFileSize fails, which should be worked
+      // around
+      fault_fs->SetFailRandomAccessGetFileSizeSst(true);
+    } else if (i == 2) {
+      // FileSystem::GetFileSize fails, which should be worked around if
+      // FSRandomAccessFile::GetFileSize is supported
+      fault_fs->SetFailFilesystemGetFileSizeSst(true);
+    } else if (i == 3) {
+      // Both GetFileSize APIs fail with an IOError
+      fault_fs->SetFailRandomAccessGetFileSizeSst(true);
+      fault_fs->SetFailFilesystemGetFileSizeSst(true);
+    }
+
+    ASSERT_OK(TryReopen(options));
+    std::string value;
+    Status get_status = db_->Get({}, "x", &value);
+    if (i < 2) {
+      ASSERT_OK(get_status);
+    } else if (i == 2) {
+      if (encrypted_env_) {
+        // Can't recover because RandomAccessFile::GetFileSize is not supported
+        // on EncryptedEnv
+        // Fail with propagated IOError. (Not Corruption nor NotSupported!)
+        ASSERT_EQ(get_status.code(), Status::Code::kIOError);
+        ASSERT_STREQ(get_status.getState(), "FileSystem::GetFileSize failed");
+      } else {
+        // Never sees the FileSystem::GetFileSize failure
+        ASSERT_OK(get_status);
+      }
+    } else {
+      ASSERT_EQ(i, 3);
+      // Fail with propagated IOError. (Not Corruption nor NotSupported!)
+      ASSERT_EQ(get_status.code(), Status::Code::kIOError);
+      ASSERT_STREQ(get_status.getState(), "FileSystem::GetFileSize failed");
+    }
+  }
+  Close();
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_test2.cc

@@ -7466,11 +7466,27 @@ TEST_F(DBTest2, GetFileChecksumsFromCurrentManifest_CRC32) {
   FlushOptions fopts;
   fopts.wait = true;
   Random rnd(test::RandomSeed());
+
+  // Write 4 files into the default column family.
   for (int i = 0; i < 4; i++) {
     ASSERT_OK(db->Put(wopts, Key(i), rnd.RandomString(100)));
     ASSERT_OK(db->Flush(fopts));
   }
 
+  // Create a new column family, write 1 file into it and drop it.
+  ColumnFamilyHandle* cf;
+  ASSERT_OK(
+      db->CreateColumnFamily(ColumnFamilyOptions(), "soon_to_be_deleted", &cf));
+  ASSERT_OK(db->Put(wopts, cf, "some_key", "some_value"));
+  ASSERT_OK(db->Flush(fopts, cf));
+
+  // Drop column family should generate corresponding version edit
+  // in manifest, which we expect to be correctly interpreted by
+  // GetFileChecksumsFromCurrentManifest API after db close.
+  ASSERT_OK(db->DropColumnFamily(cf));
+  delete cf;
+  cf = nullptr;
+
   // Obtain rich files metadata for source of truth.
   std::vector<LiveFileMetaData> live_files;
   db->GetLiveFilesMetaData(&live_files);

db/db_test_util.cc

@@ -366,11 +366,6 @@ Options DBTestBase::GetOptions(
     table_options.block_cache = NewLRUCache(/* too small */ 1);
   }
 
-  // Test anticipated new default as much as reasonably possible (and remove
-  // this code when obsolete)
-  assert(!table_options.decouple_partitioned_filters);
-  table_options.decouple_partitioned_filters = true;
-
   bool can_allow_mmap = IsMemoryMappedAccessSupported();
   switch (option_config) {
     case kHashSkipList:
@@ -1159,16 +1154,18 @@ size_t DBTestBase::CountLiveFiles() {
 }
 
 int DBTestBase::NumTableFilesAtLevel(int level, int cf) {
-  std::string property;
-  if (cf == 0) {
-    // default cfd
-    EXPECT_TRUE(db_->GetProperty(
-        "rocksdb.num-files-at-level" + std::to_string(level), &property));
-  } else {
-    EXPECT_TRUE(db_->GetProperty(
-        handles_[cf], "rocksdb.num-files-at-level" + std::to_string(level),
-        &property));
+  return NumTableFilesAtLevel(level,
+                              cf ? handles_[cf] : db_->DefaultColumnFamily());
+}
+
+int DBTestBase::NumTableFilesAtLevel(int level, ColumnFamilyHandle* cfh,
+                                     DB* db) {
+  if (!db) {
+    db = db_;
   }
+  std::string property;
+  EXPECT_TRUE(db->GetProperty(
+      cfh, "rocksdb.num-files-at-level" + std::to_string(level), &property));
   return atoi(property.c_str());
 }
 
@@ -1201,12 +1198,22 @@ int DBTestBase::TotalTableFiles(int cf, int levels) {
 
 // Return spread of files per level
 std::string DBTestBase::FilesPerLevel(int cf) {
-  int num_levels =
-      (cf == 0) ? db_->NumberLevels() : db_->NumberLevels(handles_[cf]);
+  if (cf == 0) {
+    return FilesPerLevel(db_->DefaultColumnFamily());
+  } else {
+    return FilesPerLevel(handles_[cf]);
+  }
+}
+
+std::string DBTestBase::FilesPerLevel(ColumnFamilyHandle* cfh, DB* db) {
+  if (!db) {
+    db = db_;
+  }
+  int num_levels = db->NumberLevels(cfh);
   std::string result;
   size_t last_non_zero_offset = 0;
   for (int level = 0; level < num_levels; level++) {
-    int f = NumTableFilesAtLevel(level, cf);
+    int f = NumTableFilesAtLevel(level, cfh, db);
     char buf[100];
     snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
     result += buf;
@@ -1339,12 +1346,14 @@ void DBTestBase::FillLevels(const std::string& smallest,
 }
 
 void DBTestBase::MoveFilesToLevel(int level, int cf) {
+  MoveFilesToLevel(level, cf ? handles_[cf] : db_->DefaultColumnFamily());
+}
+
+void DBTestBase::MoveFilesToLevel(int level, ColumnFamilyHandle* column_family,
+                                  DB* db) {
+  DBImpl* db_impl = db ? static_cast<DBImpl*>(db) : dbfull();
   for (int l = 0; l < level; ++l) {
-    if (cf > 0) {
-      EXPECT_OK(dbfull()->TEST_CompactRange(l, nullptr, nullptr, handles_[cf]));
-    } else {
-      EXPECT_OK(dbfull()->TEST_CompactRange(l, nullptr, nullptr));
-    }
+    EXPECT_OK(db_impl->TEST_CompactRange(l, nullptr, nullptr, column_family));
   }
 }
 

db/db_test_util.h

@@ -1280,6 +1280,9 @@ class DBTestBase : public testing::Test {
 
   int NumTableFilesAtLevel(int level, int cf = 0);
 
+  int NumTableFilesAtLevel(int level, ColumnFamilyHandle* column_family,
+                           DB* db = nullptr);
+
   double CompressionRatioAtLevel(int level, int cf = 0);
 
   int TotalTableFiles(int cf = 0, int levels = -1);
@@ -1289,6 +1292,8 @@ class DBTestBase : public testing::Test {
   // Return spread of files per level
   std::string FilesPerLevel(int cf = 0);
 
+  std::string FilesPerLevel(ColumnFamilyHandle* cfh, DB* db = nullptr);
+
   size_t CountFiles();
 
   Status CountFiles(size_t* count);
@@ -1320,6 +1325,9 @@ class DBTestBase : public testing::Test {
 
   void MoveFilesToLevel(int level, int cf = 0);
 
+  void MoveFilesToLevel(int level, ColumnFamilyHandle* column_family,
+                        DB* db = nullptr);
+
   void DumpFileCounts(const char* label);
 
   std::string DumpSSTableList();

db/db_universal_compaction_test.cc

@@ -2106,46 +2106,79 @@ TEST_F(DBTestUniversalCompaction2, OverlappingL0) {
 }
 
 TEST_F(DBTestUniversalCompaction2, IngestBehind) {
-  const int kNumKeys = 3000;
-  const int kWindowSize = 100;
-  const int kNumDelsTrigger = 90;
+  for (bool cf_option : {false, true}) {
+    SCOPED_TRACE("cf_option = " + std::to_string(cf_option));
+    const int kNumKeys = 3000;
+    const int kWindowSize = 100;
+    const int kNumDelsTrigger = 90;
 
-  Options opts = CurrentOptions();
-  opts.table_properties_collector_factories.emplace_back(
-      NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger));
-  opts.compaction_style = kCompactionStyleUniversal;
-  opts.level0_file_num_compaction_trigger = 2;
-  opts.compression = kNoCompression;
-  opts.allow_ingest_behind = true;
-  opts.compaction_options_universal.size_ratio = 10;
-  opts.compaction_options_universal.min_merge_width = 2;
-  opts.compaction_options_universal.max_size_amplification_percent = 200;
-  Reopen(opts);
-
-  // add an L1 file to prevent tombstones from dropping due to obsolescence
-  // during flush
-  int i;
-  for (i = 0; i < 2000; ++i) {
-    ASSERT_OK(Put(Key(i), "val"));
-  }
-  ASSERT_OK(Flush());
-  //  MoveFilesToLevel(6);
-  ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-
-  for (i = 1999; i < kNumKeys; ++i) {
-    if (i >= kNumKeys - kWindowSize &&
-        i < kNumKeys - kWindowSize + kNumDelsTrigger) {
-      ASSERT_OK(Delete(Key(i)));
+    Options opts = CurrentOptions();
+    opts.table_properties_collector_factories.emplace_back(
+        NewCompactOnDeletionCollectorFactory(kWindowSize, kNumDelsTrigger));
+    opts.compaction_style = kCompactionStyleUniversal;
+    opts.level0_file_num_compaction_trigger = 2;
+    opts.compression = kNoCompression;
+    if (cf_option) {
+      opts.cf_allow_ingest_behind = true;
     } else {
+      opts.allow_ingest_behind = true;
+    }
+    opts.compaction_options_universal.size_ratio = 10;
+    opts.compaction_options_universal.min_merge_width = 2;
+    opts.compaction_options_universal.max_size_amplification_percent = 200;
+    Reopen(opts);
+
+    // add an L1 file to prevent tombstones from dropping due to obsolescence
+    // during flush
+    int i;
+    for (i = 0; i < 2000; ++i) {
       ASSERT_OK(Put(Key(i), "val"));
     }
-  }
-  ASSERT_OK(Flush());
+    ASSERT_OK(Flush());
+    //  MoveFilesToLevel(6);
+    ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr));
 
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  ASSERT_EQ(0, NumTableFilesAtLevel(0));
-  ASSERT_EQ(0, NumTableFilesAtLevel(6));
-  ASSERT_GT(NumTableFilesAtLevel(5), 0);
+    for (i = 1999; i < kNumKeys; ++i) {
+      if (i >= kNumKeys - kWindowSize &&
+          i < kNumKeys - kWindowSize + kNumDelsTrigger) {
+        ASSERT_OK(Delete(Key(i)));
+      } else {
+        ASSERT_OK(Put(Key(i), "val"));
+      }
+    }
+    ASSERT_OK(Flush());
+
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_EQ(0, NumTableFilesAtLevel(0));
+    ASSERT_EQ(0, NumTableFilesAtLevel(6));
+    ASSERT_GT(NumTableFilesAtLevel(5), 0);
+
+    if (cf_option) {
+      // Test that another CF does not allow ingest behind
+      ColumnFamilyHandle* new_cfh;
+      Options new_cf_option;
+      new_cf_option.compaction_style = kCompactionStyleUniversal;
+      new_cf_option.num_levels = 7;
+      // CreateColumnFamilies({"new_cf"}, new_cf_option);
+      ASSERT_OK(db_->CreateColumnFamily(new_cf_option, "new_cf", &new_cfh));
+      // handles_.push_back(new_cfh);
+      for (i = 0; i < 10; ++i) {
+        // ASSERT_OK(Put(1, Key(i), "val"));
+        ASSERT_OK(db_->Put(WriteOptions(), new_cfh, Key(i), "val"));
+      }
+      ASSERT_OK(
+          db_->CompactRange(CompactRangeOptions(), new_cfh, nullptr, nullptr));
+      // This CF can use the last leve.
+      std::string property;
+      EXPECT_TRUE(db_->GetProperty(
+          new_cfh, "rocksdb.num-files-at-level" + std::to_string(6),
+          &property));
+      ASSERT_EQ(1, atoi(property.c_str()));
+
+      ASSERT_OK(db_->DropColumnFamily(new_cfh));
+      ASSERT_OK(db_->DestroyColumnFamilyHandle(new_cfh));
+    }
+  }
 }
 
 TEST_F(DBTestUniversalCompaction2, PeriodicCompactionDefault) {

db/experimental.cc

@@ -88,11 +88,12 @@ Status GetFileChecksumsFromCurrentManifest(FileSystem* fs,
 
   // Read all records from the manifest file...
   uint64_t manifest_file_size = std::numeric_limits<uint64_t>::max();
-  FileChecksumRetriever retriever(read_options, manifest_file_size,
-                                  *checksum_list);
+  FileChecksumRetriever retriever(read_options, manifest_file_size);
   retriever.Iterate(reader, &s);
-
-  return retriever.status();
+  if (!retriever.status().ok()) {
+    return retriever.status();
+  }
+  return retriever.FetchFileChecksumList(*checksum_list);
 }
 
 Status UpdateManifestForFilesState(

db/external_sst_file_basic_test.cc

@@ -2567,7 +2567,14 @@ TEST_F(ExternalSSTFileBasicTest, IngestWithTemperature) {
     options.default_write_temperature = Temperature::kHot;
     SstFileWriter sst_file_writer(EnvOptions(), options);
     options.level0_file_num_compaction_trigger = 2;
-    options.allow_ingest_behind = (mode == "ingest_behind");
+    bool cf_option = Random::GetTLSInstance()->OneIn(2);
+    SCOPED_TRACE(std::string("Use ") + (cf_option ? "CF" : "DB") +
+                 " option for ingest behind");
+    if (cf_option) {
+      options.cf_allow_ingest_behind = (mode == "ingest_behind");
+    } else {
+      options.allow_ingest_behind = (mode == "ingest_behind");
+    }
     Reopen(options);
     Defer destroyer([&]() { Destroy(options); });
 

db/external_sst_file_ingestion_job.cc

@@ -122,24 +122,28 @@ Status ExternalSstFileIngestionJob::Prepare(
     }
   }
 
-  if (ingestion_options_.ingest_behind && files_overlap_) {
-    return Status::NotSupported(
-        "Files with overlapping ranges cannot be ingested with ingestion "
-        "behind mode.");
-  }
+  if (files_overlap_) {
+    if (ingestion_options_.ingest_behind) {
+      return Status::NotSupported(
+          "Files with overlapping ranges cannot be ingested with ingestion "
+          "behind mode.");
+    }
 
-  // Overlapping files need at least two different sequence numbers. If settings
-  // disables global seqno, ingestion will fail anyway, so fail fast in prepare.
-  if (!ingestion_options_.allow_global_seqno && files_overlap_) {
-    return Status::InvalidArgument(
-        "Global seqno is required, but disabled (because external files key "
-        "range overlaps).");
-  }
+    // Overlapping files need at least two different sequence numbers. If
+    // settings disables global seqno, ingestion will fail anyway, so fail
+    // fast in prepare.
+    if (!ingestion_options_.allow_global_seqno &&
+        !ingestion_options_.allow_db_generated_files) {
+      return Status::InvalidArgument(
+          "Global seqno is required, but disabled (because external files key "
+          "range overlaps).");
+    }
 
-  if (ucmp_->timestamp_size() > 0 && files_overlap_) {
-    return Status::NotSupported(
-        "Files with overlapping ranges cannot be ingested to column "
-        "family with user-defined timestamp enabled.");
+    if (ucmp_->timestamp_size() > 0) {
+      return Status::NotSupported(
+          "Files with overlapping ranges cannot be ingested to column "
+          "family with user-defined timestamp enabled.");
+    }
   }
 
   // Copy/Move external files into DB
@@ -540,6 +544,8 @@ Status ExternalSstFileIngestionJob::Run() {
 
   // Find levels to ingest into
   std::optional<int> prev_batch_uppermost_level;
+  // batches at the front of file_batches_to_ingest_ contains older updates and
+  // are placed in smaller levels.
   for (auto& batch : file_batches_to_ingest_) {
     int batch_uppermost_level = 0;
     status = AssignLevelsForOneBatch(batch, super_version, force_global_seqno,
@@ -590,6 +596,15 @@ Status ExternalSstFileIngestionJob::AssignLevelsForOneBatch(
     if (!status.ok()) {
       return status;
     }
+
+    // If any ingested file overlaps with the DB, it will fail here.
+    if (ingestion_options_.allow_db_generated_files && assigned_seqno != 0) {
+      return Status::InvalidArgument(
+          "An ingested file overlaps with existing data in the DB and has been "
+          "assigned a non-zero sequence number, which is not allowed when "
+          "'allow_db_generated_files' is enabled.");
+    }
+
     if (smallest_parsed.sequence == 0 && assigned_seqno != 0) {
       UpdateInternalKey(file->smallest_internal_key.rep(), assigned_seqno,
                         smallest_parsed.type);
@@ -608,8 +623,8 @@ Status ExternalSstFileIngestionJob::AssignLevelsForOneBatch(
     assert(assigned_seqno == 0 || assigned_seqno == *last_seqno + 1);
     if (assigned_seqno > *last_seqno) {
       *last_seqno = assigned_seqno;
-      ++consumed_seqno_count_;
     }
+    max_assigned_seqno_ = std::max(max_assigned_seqno_, assigned_seqno);
 
     status = GenerateChecksumForIngestedFile(file);
     if (!status.ok()) {
@@ -632,15 +647,24 @@ Status ExternalSstFileIngestionJob::AssignLevelsForOneBatch(
         file->table_properties.num_range_deletions == 1 &&
         (file->table_properties.num_entries ==
          file->table_properties.num_range_deletions);
+    SequenceNumber smallest_seqno = file->assigned_seqno;
+    SequenceNumber largest_seqno = file->assigned_seqno;
+    if (ingestion_options_.allow_db_generated_files) {
+      assert(file->assigned_seqno == 0);
+      assert(file->smallest_seqno != kMaxSequenceNumber);
+      assert(file->largest_seqno != kMaxSequenceNumber);
+      smallest_seqno = file->smallest_seqno;
+      largest_seqno = file->largest_seqno;
+      max_assigned_seqno_ = std::max(max_assigned_seqno_, file->largest_seqno);
+    }
     FileMetaData f_metadata(
         file->fd.GetNumber(), file->fd.GetPathId(), file->fd.GetFileSize(),
-        file->smallest_internal_key, file->largest_internal_key,
-        file->assigned_seqno, file->assigned_seqno, false,
-        file->file_temperature, kInvalidBlobFileNumber, oldest_ancester_time,
-        current_time,
+        file->smallest_internal_key, file->largest_internal_key, smallest_seqno,
+        largest_seqno, false, file->file_temperature, kInvalidBlobFileNumber,
+        oldest_ancester_time, current_time,
         ingestion_options_.ingest_behind
             ? kReservedEpochNumberForFileIngestedBehind
-            : cfd_->NewEpochNumber(),
+            : cfd_->NewEpochNumber(),  // orders files ingested to L0
         file->file_checksum, file->file_checksum_func_name, file->unique_id, 0,
         tail_size, file->user_defined_timestamps_persisted);
     f_metadata.temperature = file->file_temperature;
@@ -787,7 +811,6 @@ void ExternalSstFileIngestionJob::Cleanup(const Status& status) {
     // We failed to add the files to the database
     // remove all the files we copied
     DeleteInternalFiles();
-    consumed_seqno_count_ = 0;
     files_overlap_ = false;
   } else if (status.ok() && ingestion_options_.move_files) {
     // The files were moved and added successfully, remove original file links
@@ -1001,6 +1024,32 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
     return status;
   }
 
+  const bool allow_data_in_errors = db_options_.allow_data_in_errors;
+  ParsedInternalKey key;
+  if (ingestion_options_.allow_db_generated_files) {
+    // We are ingesting a DB generated SST file for which we don't reassign
+    // sequence numbers. We need its smallest sequence number and largest
+    // sequence number for FileMetaData.
+    Status seqno_status = GetSeqnoBoundaryForFile(
+        table_reader.get(), sv, file_to_ingest, allow_data_in_errors);
+
+    if (!seqno_status.ok()) {
+      return seqno_status;
+    }
+    assert(file_to_ingest->smallest_seqno <= file_to_ingest->largest_seqno);
+    assert(file_to_ingest->largest_seqno < kMaxSequenceNumber);
+  } else {
+    SequenceNumber largest_seqno =
+        table_reader.get()->GetTableProperties()->key_largest_seqno;
+    // UINT64_MAX means unknown and the file is generated before table property
+    // `key_largest_seqno` is introduced.
+    if (largest_seqno != UINT64_MAX && largest_seqno > 0) {
+      return Status::Corruption(
+          "External file has non zero largest sequence number " +
+          std::to_string(largest_seqno));
+    }
+  }
+
   if (ingestion_options_.verify_checksums_before_ingest) {
     // If customized readahead size is needed, we can pass a user option
     // all the way to here. Right now we just rely on the default readahead
@@ -1016,7 +1065,6 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
     }
   }
 
-  ParsedInternalKey key;
   // TODO: plumb Env::IOActivity, Env::IOPriority
   ReadOptions ro;
   ro.fill_cache = ingestion_options_.fill_cache;
@@ -1025,7 +1073,6 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
       /*skip_filters=*/false, TableReaderCaller::kExternalSSTIngestion));
 
   // Get first (smallest) and last (largest) key from file.
-  bool allow_data_in_errors = db_options_.allow_data_in_errors;
   iter->SeekToFirst();
   if (iter->Valid()) {
     Status pik_status =
@@ -1034,7 +1081,7 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
       return Status::Corruption("Corrupted key in external file. ",
                                 pik_status.getState());
     }
-    if (key.sequence != 0) {
+    if (key.sequence != 0 && !ingestion_options_.allow_db_generated_files) {
       return Status::Corruption("External file has non zero sequence number");
     }
     file_to_ingest->smallest_internal_key.SetFrom(key);
@@ -1071,41 +1118,13 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
       return Status::Corruption("Corrupted key in external file. ",
                                 pik_status.getState());
     }
-    if (key.sequence != 0) {
+    if (key.sequence != 0 && !ingestion_options_.allow_db_generated_files) {
       return Status::Corruption("External file has non zero sequence number");
     }
     file_to_ingest->largest_internal_key.SetFrom(key);
   } else if (!iter->status().ok()) {
     return iter->status();
   }
-  SequenceNumber largest_seqno =
-      table_reader.get()->GetTableProperties()->key_largest_seqno;
-  // UINT64_MAX means unknown and the file is generated before table property
-  // `key_largest_seqno` is introduced.
-  if (largest_seqno != UINT64_MAX && largest_seqno > 0) {
-    return Status::Corruption(
-        "External file has non zero largest sequence number " +
-        std::to_string(largest_seqno));
-  }
-  if (ingestion_options_.allow_db_generated_files &&
-      largest_seqno == UINT64_MAX) {
-    // Need to verify that all keys have seqno zero.
-    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-      Status pik_status =
-          ParseInternalKey(iter->key(), &key, allow_data_in_errors);
-      if (!pik_status.ok()) {
-        return Status::Corruption("Corrupted key in external file. ",
-                                  pik_status.getState());
-      }
-      if (key.sequence != 0) {
-        return Status::NotSupported(
-            "External file has a key with non zero sequence number.");
-      }
-    }
-    if (!iter->status().ok()) {
-      return iter->status();
-    }
-  }
 
   std::unique_ptr<InternalIterator> range_del_iter(
       table_reader->NewRangeTombstoneIterator(ro));
@@ -1120,7 +1139,7 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
         return Status::Corruption("Corrupted key in external file. ",
                                   pik_status.getState());
       }
-      if (key.sequence != 0) {
+      if (key.sequence != 0 && !ingestion_options_.allow_db_generated_files) {
         return Status::Corruption(
             "External file has a range deletion with non zero sequence "
             "number.");
@@ -1168,12 +1187,14 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
   const size_t ts_sz = ucmp_->timestamp_size();
   assert(!prev_batch_uppermost_level.has_value() ||
          prev_batch_uppermost_level.value() < cfd_->NumberLevels());
-  bool must_assign_to_l0 = prev_batch_uppermost_level.has_value() &&
-                           prev_batch_uppermost_level.value() == 0;
-  if (force_global_seqno || files_overlap_ ||
-      compaction_style == kCompactionStyleFIFO || must_assign_to_l0) {
+  bool must_assign_to_l0 = (prev_batch_uppermost_level.has_value() &&
+                            prev_batch_uppermost_level.value() == 0) ||
+                           compaction_style == kCompactionStyleFIFO;
+
+  if (force_global_seqno || (!ingestion_options_.allow_db_generated_files &&
+                             (files_overlap_ || must_assign_to_l0))) {
     *assigned_seqno = last_seqno + 1;
-    if (compaction_style == kCompactionStyleFIFO || must_assign_to_l0) {
+    if (must_assign_to_l0) {
       assert(ts_sz == 0);
       file_to_ingest->picked_level = 0;
       if (ingestion_options_.fail_if_not_bottommost_level &&
@@ -1194,16 +1215,26 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
   ro.total_order_seek = true;
   int target_level = 0;
   auto* vstorage = cfd_->current()->storage_info();
-  assert(!must_assign_to_l0);
-  int exclusive_end_level = prev_batch_uppermost_level.has_value()
-                                ? prev_batch_uppermost_level.value()
-                                : cfd_->NumberLevels();
+  assert(!must_assign_to_l0 || ingestion_options_.allow_db_generated_files);
+  int assigned_level_exclusive_end = cfd_->NumberLevels();
+  if (must_assign_to_l0) {
+    assigned_level_exclusive_end = 0;
+  } else if (prev_batch_uppermost_level.has_value()) {
+    assigned_level_exclusive_end = prev_batch_uppermost_level.value();
+  }
 
-  for (int lvl = 0; lvl < exclusive_end_level; lvl++) {
+  // When ingesting db generated files, we require that ingested files do not
+  // overlap with any file in the DB. So we need to check all levels.
+  int overlap_checking_exclusive_end =
+      ingestion_options_.allow_db_generated_files
+          ? cfd_->NumberLevels()
+          : assigned_level_exclusive_end;
+  for (int lvl = 0; lvl < overlap_checking_exclusive_end; lvl++) {
     if (lvl > 0 && lvl < vstorage->base_level()) {
       continue;
     }
-    if (atomic_replace_range_.has_value()) {
+    if (lvl < assigned_level_exclusive_end &&
+        atomic_replace_range_.has_value()) {
       target_level = lvl;
       continue;
     }
@@ -1234,7 +1265,8 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
 
     // We don't overlap with any keys in this level, but we still need to check
     // if our file can fit in it
-    if (IngestedFileFitInLevel(file_to_ingest, lvl)) {
+    if (lvl < assigned_level_exclusive_end &&
+        IngestedFileFitInLevel(file_to_ingest, lvl)) {
       target_level = lvl;
     }
   }
@@ -1243,8 +1275,9 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
       target_level < cfd_->NumberLevels() - 1) {
     status = Status::TryAgain(
         "Files cannot be ingested to Lmax. Please make sure key range of Lmax "
-        "and ongoing compaction's output to Lmax"
-        "does not overlap with files to ingest.");
+        "and ongoing compaction's output to Lmax does not overlap with files "
+        "to ingest. Input files overlapping with each other can cause some "
+        "file to be assigned to non Lmax level.");
     return status;
   }
 
@@ -1265,11 +1298,6 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
     }
   }
 
-  if (ingestion_options_.allow_db_generated_files && *assigned_seqno != 0) {
-    return Status::InvalidArgument(
-        "An ingested file is assigned to a non-zero sequence number, which is "
-        "incompatible with ingestion option allow_db_generated_files.");
-  }
   return status;
 }
 
@@ -1286,13 +1314,13 @@ Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
         "at the last level!");
   }
 
-  // Second, check if despite allow_ingest_behind=true we still have 0 seqnums
-  // at some upper level
+  // Second, check if despite cf_allow_ingest_behind=true we still have 0
+  // seqnums at some upper level
   for (int lvl = 0; lvl < cfd_->NumberLevels() - 1; lvl++) {
     for (auto file : vstorage->LevelFiles(lvl)) {
       if (file->fd.smallest_seqno == 0) {
         return Status::InvalidArgument(
-            "Can't ingest_behind file as despite allow_ingest_behind=true "
+            "Can't ingest_behind file as despite cf_allow_ingest_behind=true "
             "there are files with 0 seqno in database at upper levels!");
       }
     }
@@ -1304,8 +1332,12 @@ Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
 
 Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
     IngestedFileInfo* file_to_ingest, SequenceNumber seqno) {
+  if (ingestion_options_.allow_db_generated_files) {
+    assert(seqno == 0);
+    assert(file_to_ingest->original_seqno == 0);
+  }
   if (file_to_ingest->original_seqno == seqno) {
-    // This file already have the correct global seqno
+    // This file already has the correct global seqno.
     return Status::OK();
   } else if (!ingestion_options_.allow_global_seqno) {
     return Status::InvalidArgument("Global seqno is required, but disabled");
@@ -1419,4 +1451,86 @@ Status ExternalSstFileIngestionJob::SyncIngestedFile(TWritableFile* file) {
   }
 }
 
+Status ExternalSstFileIngestionJob::GetSeqnoBoundaryForFile(
+    TableReader* table_reader, SuperVersion* sv,
+    IngestedFileInfo* file_to_ingest, bool allow_data_in_errors) {
+  const bool has_largest_seqno =
+      table_reader->GetTableProperties()->HasKeyLargestSeqno();
+  SequenceNumber largest_seqno =
+      table_reader->GetTableProperties()->key_largest_seqno;
+  if (has_largest_seqno && largest_seqno == 0) {
+    file_to_ingest->largest_seqno = 0;
+    file_to_ingest->smallest_seqno = 0;
+    return Status::OK();
+  }
+  // The following file scan is only executed when ingesting files with
+  // non-zero seqno.
+  // TODO: record smallest_seqno in table properties to avoid the
+  // file scan here.
+  SequenceNumber smallest_seqno = kMaxSequenceNumber;
+
+  SequenceNumber largest_seqno_from_iter = 0;
+  ReadOptions ro;
+  ro.fill_cache = ingestion_options_.fill_cache;
+  std::unique_ptr<InternalIterator> iter(table_reader->NewIterator(
+      ro, sv->mutable_cf_options.prefix_extractor.get(), /*arena=*/nullptr,
+      /*skip_filters=*/false, TableReaderCaller::kExternalSSTIngestion));
+  ParsedInternalKey key;
+  iter->SeekToFirst();
+  while (iter->Valid()) {
+    Status pik_status =
+        ParseInternalKey(iter->key(), &key, allow_data_in_errors);
+    if (!pik_status.ok()) {
+      return Status::Corruption("Corrupted key in external file. ",
+                                pik_status.getState());
+    }
+    smallest_seqno = std::min(smallest_seqno, key.sequence);
+    largest_seqno_from_iter = std::max(largest_seqno_from_iter, key.sequence);
+    iter->Next();
+  }
+  if (!iter->status().ok()) {
+    return iter->status();
+  }
+
+  if (table_reader->GetTableProperties()->num_range_deletions > 0) {
+    std::unique_ptr<InternalIterator> range_del_iter(
+        table_reader->NewRangeTombstoneIterator(ro));
+    if (range_del_iter != nullptr) {
+      for (range_del_iter->SeekToFirst(); range_del_iter->Valid();
+           range_del_iter->Next()) {
+        Status pik_status =
+            ParseInternalKey(range_del_iter->key(), &key, allow_data_in_errors);
+        if (!pik_status.ok()) {
+          return Status::Corruption("Corrupted key in external file. ",
+                                    pik_status.getState());
+        }
+        smallest_seqno = std::min(smallest_seqno, key.sequence);
+        largest_seqno_from_iter =
+            std::max(largest_seqno_from_iter, key.sequence);
+      }
+      if (!range_del_iter->status().ok()) {
+        return range_del_iter->status();
+      }
+    }
+  }
+
+  file_to_ingest->smallest_seqno = smallest_seqno;
+  if (!has_largest_seqno) {
+    file_to_ingest->largest_seqno = largest_seqno_from_iter;
+  } else {
+    assert(largest_seqno == largest_seqno_from_iter);
+    file_to_ingest->largest_seqno = largest_seqno;
+  }
+
+  if (file_to_ingest->largest_seqno == kMaxSequenceNumber) {
+    return Status::InvalidArgument(
+        "Unknown smallest seqno for db generated file.");
+  }
+  if (file_to_ingest->smallest_seqno == kMaxSequenceNumber) {
+    return Status::InvalidArgument(
+        "Unknown largest seqno for db generated file.");
+  }
+  return Status::OK();
+}
+
 }  // namespace ROCKSDB_NAMESPACE

db/external_sst_file_ingestion_job.h

@@ -180,6 +180,9 @@ struct IngestedFileInfo : public KeyRangeInfo {
   // the user key's format in the external file matches the column family's
   // setting.
   bool user_defined_timestamps_persisted = true;
+
+  SequenceNumber largest_seqno = kMaxSequenceNumber;
+  SequenceNumber smallest_seqno = kMaxSequenceNumber;
 };
 
 // A batch of files.
@@ -230,7 +233,7 @@ class ExternalSstFileIngestionJob {
         directories_(directories),
         event_logger_(event_logger),
         job_start_time_(clock_->NowMicros()),
-        consumed_seqno_count_(0),
+        max_assigned_seqno_(0),
         io_tracer_(io_tracer) {
     assert(directories != nullptr);
     assert(cfd_);
@@ -287,8 +290,16 @@ class ExternalSstFileIngestionJob {
     return files_to_ingest_;
   }
 
-  // How many sequence numbers did we consume as part of the ingestion job?
-  int ConsumedSequenceNumbersCount() const { return consumed_seqno_count_; }
+  // Return the maximum assigned sequence number for all files in this job.
+  // When allow_db_generated_files = false, we may assign global sequence
+  // numbers to ingested files. The global sequence numbers are sequence numbers
+  // following versions_->LastSequence().
+  // When allow_db_generated_files = true, we ingest files that already have
+  // sequence numbers assigned. max_assigned_seqno_ will be the max sequence
+  // number among ingested files.
+  SequenceNumber MaxAssignedSequenceNumber() const {
+    return max_assigned_seqno_;
+  }
 
  private:
   Status ResetTableReader(const std::string& external_file,
@@ -349,7 +360,7 @@ class ExternalSstFileIngestionJob {
       std::optional<int> prev_batch_uppermost_level);
 
   // File that we want to ingest behind always goes to the lowest level;
-  // we just check that it fits in the level, that DB allows ingest_behind,
+  // we just check that it fits in the level, that the CF allows ingest_behind,
   // and that we don't have 0 seqnums at the upper levels.
   // REQUIRES: Mutex held
   Status CheckLevelForIngestedBehindFile(IngestedFileInfo* file_to_ingest);
@@ -369,6 +380,13 @@ class ExternalSstFileIngestionJob {
   template <typename TWritableFile>
   Status SyncIngestedFile(TWritableFile* file);
 
+  // Helper function to obtain the smallest and largest sequence number from a
+  // file. When OK is returned, file_to_ingest->smallest_seqno and
+  // file_to_ingest->largest_seqno will be updated.
+  Status GetSeqnoBoundaryForFile(TableReader* table_reader, SuperVersion* sv,
+                                 IngestedFileInfo* file_to_ingest,
+                                 bool allow_data_in_errors);
+
   // Create equivalent `Compaction` objects to this file ingestion job
   // , which will be used to check range conflict with other ongoing
   // compactions.
@@ -395,7 +413,7 @@ class ExternalSstFileIngestionJob {
   EventLogger* event_logger_;
   VersionEdit edit_;
   uint64_t job_start_time_;
-  int consumed_seqno_count_;
+  SequenceNumber max_assigned_seqno_;
   // Set in ExternalSstFileIngestionJob::Prepare(), if true all files are
   // ingested in L0
   bool files_overlap_{false};

db/external_sst_file_test.cc

@@ -7,6 +7,7 @@
 
 #include <functional>
 #include <memory>
+#include <sstream>
 
 #include "db/db_test_util.h"
 #include "db/dbformat.h"
@@ -2417,102 +2418,130 @@ TEST_F(ExternalSSTFileTest, SnapshotInconsistencyBug) {
 }
 
 TEST_P(ExternalSSTFileTest, IngestBehind) {
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleUniversal;
-  options.num_levels = 3;
-  options.disable_auto_compactions = false;
-  DestroyAndReopen(options);
-  std::vector<std::pair<std::string, std::string>> file_data;
-  std::map<std::string, std::string> true_data;
+  for (bool cf_option : {false, true}) {
+    SCOPED_TRACE("cf_option = " + std::to_string(cf_option));
+    Options options = CurrentOptions();
+    options.compaction_style = kCompactionStyleUniversal;
+    options.num_levels = 3;
+    options.disable_auto_compactions = false;
+    DestroyAndReopen(options);
+    std::vector<std::pair<std::string, std::string>> file_data;
+    std::map<std::string, std::string> true_data;
 
-  // Insert 100 -> 200 into the memtable
-  for (int i = 100; i <= 200; i++) {
-    ASSERT_OK(Put(Key(i), "memtable"));
-  }
-
-  // Insert 100 -> 200 using IngestExternalFile
-  file_data.clear();
-  for (int i = 0; i <= 20; i++) {
-    file_data.emplace_back(Key(i), "ingest_behind");
-    true_data[Key(i)] = "ingest_behind";
-  }
-
-  bool allow_global_seqno = true;
-  bool ingest_behind = true;
-  bool write_global_seqno = std::get<0>(GetParam());
-  bool verify_checksums_before_ingest = std::get<1>(GetParam());
-
-  // Can't ingest behind since allow_ingest_behind isn't set to true
-  ASSERT_NOK(GenerateAndAddExternalFile(
-      options, file_data, -1, allow_global_seqno, write_global_seqno,
-      verify_checksums_before_ingest, ingest_behind, false /*sort_data*/,
-      &true_data));
-
-  options.allow_ingest_behind = true;
-  // check that we still can open the DB, as num_levels should be
-  // sanitized to 3
-  options.num_levels = 2;
-  DestroyAndReopen(options);
-
-  options.num_levels = 3;
-  DestroyAndReopen(options);
-  true_data.clear();
-  // Insert 100 -> 200 into the memtable
-  for (int i = 100; i <= 200; i++) {
-    ASSERT_OK(Put(Key(i), "memtable"));
-    true_data[Key(i)] = "memtable";
-  }
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  // Universal picker should go at second from the bottom level
-  ASSERT_EQ("0,1", FilesPerLevel());
-  ASSERT_OK(GenerateAndAddExternalFile(
-      options, file_data, -1, allow_global_seqno, write_global_seqno,
-      verify_checksums_before_ingest, true /*ingest_behind*/,
-      false /*sort_data*/, &true_data));
-  ASSERT_EQ("0,1,1", FilesPerLevel());
-  // this time ingest should fail as the file doesn't fit to the bottom level
-  ASSERT_NOK(GenerateAndAddExternalFile(
-      options, file_data, -1, allow_global_seqno, write_global_seqno,
-      verify_checksums_before_ingest, true /*ingest_behind*/,
-      false /*sort_data*/, &true_data));
-  ASSERT_EQ("0,1,1", FilesPerLevel());
-  std::vector<std::vector<FileMetaData>> level_to_files;
-  dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &level_to_files);
-  uint64_t ingested_file_number = level_to_files[2][0].fd.GetNumber();
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  // Last level should not be compacted
-  ASSERT_EQ("0,1,1", FilesPerLevel());
-  dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &level_to_files);
-  ASSERT_EQ(ingested_file_number, level_to_files[2][0].fd.GetNumber());
-  size_t kcnt = 0;
-  VerifyDBFromMap(true_data, &kcnt, false);
-
-  // Auto-compaction should not include the last level.
-  // Trigger compaction if size amplification exceeds 110%.
-  options.compaction_options_universal.max_size_amplification_percent = 110;
-  options.level0_file_num_compaction_trigger = 4;
-  ASSERT_OK(TryReopen(options));
-  Random rnd(301);
-  for (int i = 0; i < 4; ++i) {
-    for (int j = 0; j < 10; j++) {
-      true_data[Key(j)] = rnd.RandomString(1000);
-      ASSERT_OK(Put(Key(j), true_data[Key(j)]));
+    // Insert 100 -> 200 into the memtable
+    for (int i = 100; i <= 200; i++) {
+      ASSERT_OK(Put(Key(i), "memtable"));
     }
-    ASSERT_OK(Flush());
-  }
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &level_to_files);
-  ASSERT_EQ(1, level_to_files[2].size());
-  ASSERT_EQ(ingested_file_number, level_to_files[2][0].fd.GetNumber());
 
-  // Turning off the option allows DB to compact ingested files.
-  options.allow_ingest_behind = false;
-  ASSERT_OK(TryReopen(options));
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &level_to_files);
-  ASSERT_EQ(1, level_to_files[2].size());
-  ASSERT_NE(ingested_file_number, level_to_files[2][0].fd.GetNumber());
-  VerifyDBFromMap(true_data, &kcnt, false);
+    // Insert 100 -> 200 using IngestExternalFile
+    file_data.clear();
+    for (int i = 0; i <= 20; i++) {
+      file_data.emplace_back(Key(i), "ingest_behind");
+      true_data[Key(i)] = "ingest_behind";
+    }
+
+    bool allow_global_seqno = true;
+    bool ingest_behind = true;
+    bool write_global_seqno = std::get<0>(GetParam());
+    bool verify_checksums_before_ingest = std::get<1>(GetParam());
+
+    // Can't ingest behind since allow_ingest_behind isn't set to true
+    ASSERT_NOK(GenerateAndAddExternalFile(
+        options, file_data, -1, allow_global_seqno, write_global_seqno,
+        verify_checksums_before_ingest, ingest_behind, false /*sort_data*/,
+        &true_data));
+
+    if (cf_option) {
+      options.cf_allow_ingest_behind = true;
+    } else {
+      options.allow_ingest_behind = true;
+    }
+    // check that we still can open the DB, as num_levels should be
+    // sanitized to 3
+    options.num_levels = 2;
+    DestroyAndReopen(options);
+
+    options.num_levels = 3;
+    DestroyAndReopen(options);
+    true_data.clear();
+    // Insert 100 -> 200 into the memtable
+    for (int i = 100; i <= 200; i++) {
+      ASSERT_OK(Put(Key(i), "memtable"));
+      true_data[Key(i)] = "memtable";
+    }
+    ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+    // Universal picker should go at second from the bottom level
+    ASSERT_EQ("0,1", FilesPerLevel());
+    ASSERT_OK(GenerateAndAddExternalFile(
+        options, file_data, -1, allow_global_seqno, write_global_seqno,
+        verify_checksums_before_ingest, true /*ingest_behind*/,
+        false /*sort_data*/, &true_data));
+    ASSERT_EQ("0,1,1", FilesPerLevel());
+    // this time ingest should fail as the file doesn't fit to the bottom level
+    ASSERT_NOK(GenerateAndAddExternalFile(
+        options, file_data, -1, allow_global_seqno, write_global_seqno,
+        verify_checksums_before_ingest, true /*ingest_behind*/,
+        false /*sort_data*/, &true_data));
+    ASSERT_EQ("0,1,1", FilesPerLevel());
+    std::vector<std::vector<FileMetaData>> level_to_files;
+    dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(),
+                                    &level_to_files);
+    uint64_t ingested_file_number = level_to_files[2][0].fd.GetNumber();
+    ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+    // Last level should not be compacted
+    ASSERT_EQ("0,1,1", FilesPerLevel());
+    dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(),
+                                    &level_to_files);
+    ASSERT_EQ(ingested_file_number, level_to_files[2][0].fd.GetNumber());
+    size_t kcnt = 0;
+    VerifyDBFromMap(true_data, &kcnt, false);
+
+    // Auto-compaction should not include the last level.
+    // Trigger compaction if size amplification exceeds 110%.
+    options.compaction_options_universal.max_size_amplification_percent = 110;
+    options.level0_file_num_compaction_trigger = 4;
+    ASSERT_OK(TryReopen(options));
+    Random rnd(301);
+    for (int i = 0; i < 4; ++i) {
+      for (int j = 0; j < 10; j++) {
+        true_data[Key(j)] = rnd.RandomString(1000);
+        ASSERT_OK(Put(Key(j), true_data[Key(j)]));
+      }
+      ASSERT_OK(Flush());
+    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(),
+                                    &level_to_files);
+    ASSERT_EQ(1, level_to_files[2].size());
+    ASSERT_EQ(ingested_file_number, level_to_files[2][0].fd.GetNumber());
+
+    // Turning off the option allows DB to compact ingested files.
+    if (cf_option) {
+      // Test that another CF does not allow ingest behind
+      ColumnFamilyHandle* new_cfh;
+      Options new_cf_option;
+      ASSERT_OK(db_->CreateColumnFamily(new_cf_option, "new_cf", &new_cfh));
+      ASSERT_TRUE(GenerateAndAddExternalFile(
+                      new_cf_option, file_data, -1, allow_global_seqno,
+                      write_global_seqno, verify_checksums_before_ingest,
+                      true /*ingest_behind*/, false /*sort_data*/, nullptr,
+                      /*cfh=*/new_cfh)
+                      .IsInvalidArgument());
+      ASSERT_OK(db_->DropColumnFamily(new_cfh));
+      ASSERT_OK(db_->DestroyColumnFamilyHandle(new_cfh));
+
+      options.cf_allow_ingest_behind = false;
+    } else {
+      options.allow_ingest_behind = false;
+    }
+    ASSERT_OK(TryReopen(options));
+    ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+    dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(),
+                                    &level_to_files);
+    ASSERT_EQ(1, level_to_files[2].size());
+    ASSERT_NE(ingested_file_number, level_to_files[2][0].fd.GetNumber());
+    VerifyDBFromMap(true_data, &kcnt, false);
+  }
 }
 
 TEST_F(ExternalSSTFileTest, SkipBloomFilter) {
@@ -3514,19 +3543,26 @@ TEST_F(ExternalSSTFileWithTimestampTest, SanityCheck) {
   // overlapping key ranges.
   ASSERT_TRUE(IngestExternalUDTFile({file1, file2}).IsNotSupported());
 
-  options.allow_ingest_behind = true;
-  DestroyAndReopen(options);
-  IngestExternalFileOptions opts;
+  for (bool cf_option : {false, true}) {
+    SCOPED_TRACE("cf_option = " + std::to_string(cf_option));
+    if (cf_option) {
+      options.cf_allow_ingest_behind = true;
+    } else {
+      options.allow_ingest_behind = true;
+    }
+    DestroyAndReopen(options);
+    IngestExternalFileOptions opts;
 
-  // TODO(yuzhangyu): support ingestion behind for user-defined timestamps?
-  // Ingesting external files with user-defined timestamps requires searching
-  // through the whole lsm tree to make sure there is no key range overlap with
-  // the db. Ingestion behind currently is doing a simply placing it at the
-  // bottom level step without a search, so we don't allow it either.
-  opts.ingest_behind = true;
-  ASSERT_TRUE(db_->IngestExternalFile({file1}, opts).IsNotSupported());
+    // TODO(yuzhangyu): support ingestion behind for user-defined timestamps?
+    // Ingesting external files with user-defined timestamps requires searching
+    // through the whole lsm tree to make sure there is no key range overlap
+    // with the db. Ingestion behind currently is doing a simply placing it at
+    // the bottom level step without a search, so we don't allow it either.
+    opts.ingest_behind = true;
+    ASSERT_TRUE(db_->IngestExternalFile({file1}, opts).IsNotSupported());
 
-  DestroyAndRecreateExternalSSTFilesDir();
+    DestroyAndRecreateExternalSSTFilesDir();
+  }
 }
 
 TEST_F(ExternalSSTFileWithTimestampTest, UDTSettingsCompatibilityCheck) {
@@ -3818,99 +3854,32 @@ TEST_P(IngestDBGeneratedFileTest, FailureCase) {
       ASSERT_OK(Put(1, Key(k), "cf1_" + Key(k)));
     }
     ASSERT_OK(Flush(/*cf=*/1));
-    {
-      // Verify that largest key of the file has non-zero seqno.
-      std::vector<std::vector<FileMetaData>> metadata;
-      dbfull()->TEST_GetFilesMetaData(handles_[1], &metadata, nullptr);
-      const FileMetaData& file = metadata[0][0];
-      ValueType vtype;
-      SequenceNumber seq;
-      UnPackSequenceAndType(ExtractInternalKeyFooter(file.largest.Encode()),
-                            &seq, &vtype);
-      ASSERT_GE(seq, 0);
-    }
-    std::vector<LiveFileMetaData> live_meta;
-    db_->GetLiveFilesMetaData(&live_meta);
-    ASSERT_EQ(live_meta.size(), 1);
-    std::vector<std::string> to_ingest_files;
-    to_ingest_files.emplace_back(live_meta[0].directory + "/" +
-                                 live_meta[0].relative_filename);
-    // Ingesting a file whose boundary key has non-zero seqno.
-    Status s = db_->IngestExternalFile(to_ingest_files, ingest_opts);
-    // This error msg is from checking seqno of boundary keys.
-    ASSERT_TRUE(
-        s.ToString().find("External file has non zero sequence number") !=
-        std::string::npos);
-    ASSERT_NOK(s);
-
-    {
-      // Only non-boundary key with non-zero seqno.
-      const Snapshot* snapshot = db_->GetSnapshot();
-      ASSERT_OK(Put(1, Key(70), "cf1_" + Key(70)));
-      ASSERT_OK(Flush(1));
-      CompactRangeOptions cro;
-      cro.bottommost_level_compaction =
-          BottommostLevelCompaction::kForceOptimized;
-      ASSERT_OK(db_->CompactRange(cro, handles_[1], nullptr, nullptr));
-
-      // Verify that only the non-boundary key of the file has non-zero seqno.
-      std::vector<std::vector<FileMetaData>> metadata;
-      // File may be at different level for different options.
-      dbfull()->TEST_GetFilesMetaData(handles_[1], &metadata, nullptr);
-      bool found_file = false;
-      for (const auto& level : metadata) {
-        if (level.empty()) {
-          continue;
-        }
-        ASSERT_FALSE(found_file);
-        found_file = true;
-        ASSERT_EQ(1, level.size());
-        const FileMetaData& file = level[0];
-        ValueType vtype;
-        SequenceNumber seq;
-        UnPackSequenceAndType(ExtractInternalKeyFooter(file.largest.Encode()),
-                              &seq, &vtype);
-        ASSERT_EQ(seq, 0);
-        UnPackSequenceAndType(ExtractInternalKeyFooter(file.smallest.Encode()),
-                              &seq, &vtype);
-        ASSERT_EQ(seq, 0);
-        ASSERT_GT(file.fd.largest_seqno, 0);
-      }
-      ASSERT_TRUE(found_file);
-      live_meta.clear();
-      db_->GetLiveFilesMetaData(&live_meta);
-      ASSERT_EQ(live_meta.size(), 1);
-      to_ingest_files[0] =
-          live_meta[0].directory + "/" + live_meta[0].relative_filename;
-      s = db_->IngestExternalFile(to_ingest_files, ingest_opts);
-      ASSERT_NOK(s);
-      // This error msg is from checking largest seqno in table property.
-      ASSERT_TRUE(s.ToString().find("non zero largest sequence number") !=
-                  std::string::npos);
-      db_->ReleaseSnapshot(snapshot);
-    }
 
+    Status s;
     CompactRangeOptions cro;
     cro.bottommost_level_compaction =
         BottommostLevelCompaction::kForceOptimized;
     ASSERT_OK(db_->CompactRange(cro, handles_[1], nullptr, nullptr));
-    live_meta.clear();
+
+    std::vector<LiveFileMetaData> live_meta;
+    std::vector<std::string> to_ingest_files;
     db_->GetLiveFilesMetaData(&live_meta);
     ASSERT_EQ(live_meta.size(), 1);
+    ASSERT_EQ(live_meta[0].column_family_name, "toto");
     ASSERT_EQ(0, live_meta[0].largest_seqno);
-    to_ingest_files[0] =
-        live_meta[0].directory + "/" + live_meta[0].relative_filename;
+    to_ingest_files.emplace_back(live_meta[0].directory + "/" +
+                                 live_meta[0].relative_filename);
 
+    // Ingesting a DB generated file with allow_db_generated_files = false
     ingest_opts.allow_db_generated_files = false;
-    // Ingesting a DB genrate file with allow_db_generated_files = false;
     s = db_->IngestExternalFile(to_ingest_files, ingest_opts);
     ASSERT_TRUE(s.ToString().find("External file version not found") !=
                 std::string::npos);
     ASSERT_NOK(s);
 
     const std::string err =
-        "An ingested file is assigned to a non-zero sequence number, which is "
-        "incompatible with ingestion option allow_db_generated_files";
+        "An ingested file overlaps with existing data in the DB and has been "
+        "assigned a non-zero sequence number";
     ingest_opts.allow_db_generated_files = true;
     s = db_->IngestExternalFile(to_ingest_files, ingest_opts);
     ASSERT_TRUE(s.ToString().find(err) != std::string::npos);
@@ -4111,6 +4080,440 @@ TEST_P(IngestDBGeneratedFileTest2, NotOverlapWithDB) {
     }
   } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction));
 }
+
+TEST_P(IngestDBGeneratedFileTest2, NonZeroSeqno) {
+  // Test ingestion of DB-generated SST files that contain non-zero sequence
+  // numbers.
+  IngestExternalFileOptions ingest_opts;
+  ingest_opts.allow_db_generated_files = true;
+  // This only works since we are ingesting without snapshot
+  // Failure case will be tested below.
+  ingest_opts.snapshot_consistency = std::get<0>(GetParam());
+  ingest_opts.allow_global_seqno = std::get<1>(GetParam());
+  ingest_opts.allow_blocking_flush = std::get<2>(GetParam());
+  ingest_opts.fail_if_not_bottommost_level = std::get<3>(GetParam());
+  ingest_opts.link_files = std::get<4>(GetParam());
+  Random* rnd = Random::GetTLSInstance();
+
+  do {
+    SCOPED_TRACE("option_config_ = " + std::to_string(option_config_));
+
+    Options options = CurrentOptions();
+    options.allow_concurrent_memtable_write =
+        false;  // Required for VectorRepFactory
+    CreateAndReopenWithCF({"non_overlap", "overlap"}, options);
+
+    ColumnFamilyHandle* non_overlap_cf = handles_[1];
+    ColumnFamilyHandle* overlap_cf = handles_[2];
+
+    std::vector<std::string> expected_values;
+    expected_values.resize(100);
+    WriteOptions wo;
+    // Setup target CF with non-overlapping base data Key1 and Key99
+    // Will ingest keys [1, 98] below.
+    expected_values[0] = rnd->RandomString(100);
+    ASSERT_OK(db_->Put(wo, non_overlap_cf, Key(0), expected_values[0]));
+    ASSERT_OK(db_->Flush({}, non_overlap_cf));
+    expected_values[99] = rnd->RandomString(100);
+    ASSERT_OK(db_->Put(wo, non_overlap_cf, Key(99), expected_values[99]));
+
+    // Set up overlapping cf
+    ASSERT_OK(db_->Put(wo, overlap_cf, Key(50), rnd->RandomString(100)));
+
+    // Create temp CF/DB
+    Options temp_cf_opts;
+    ColumnFamilyHandle* temp_cfh = nullptr;
+    DB* from_db = nullptr;
+    std::string temp_db_name;
+    // Using a separate DB also validates that latest sequence number
+    // of target db is updated after ingestion (to the max sequence number
+    // in ingested files).
+    const bool use_temp_db = rnd->OneIn(2);
+    SCOPED_TRACE("use_temp_db: " + std::to_string(use_temp_db));
+
+    std::vector<std::string> sst_file_paths;
+    // optional L5: files in key range [70, 98]
+    // L6: files in key range [1, 79]
+    temp_cf_opts.target_file_size_base =
+        4 << 10;  // Small files to create multiple SSTs
+    temp_cf_opts.num_levels = 7;
+    temp_cf_opts.disable_auto_compactions = true;  // Manually set up LSM
+    temp_cf_opts.env = options.env;
+
+    if (use_temp_db) {
+      temp_cf_opts.create_if_missing = true;
+      temp_db_name = dbname_ + "/temp_db_" + std::to_string(rnd->Next());
+      ASSERT_OK(DB::Open(temp_cf_opts, temp_db_name, &from_db));
+      temp_cfh = from_db->DefaultColumnFamily();
+    } else {
+      from_db = db_;
+      ASSERT_OK(
+          from_db->CreateColumnFamily(temp_cf_opts, "temp_cf", &temp_cfh));
+    }
+
+    // Use snapshot to ensure non-zero sequence numbers after compaction
+    const Snapshot* snapshot = from_db->GetSnapshot();
+
+    for (int k = 1; k < 99; ++k) {
+      expected_values[k] = rnd->RandomString(500);
+      ASSERT_OK(from_db->Put(wo, temp_cfh, Key(k), expected_values[k]));
+    }
+    ASSERT_OK(from_db->Flush({}, temp_cfh));
+    CompactRangeOptions cro;
+    cro.bottommost_level_compaction =
+        BottommostLevelCompaction::kForceOptimized;
+    ASSERT_OK(from_db->CompactRange(cro, temp_cfh, nullptr, nullptr));
+
+    ASSERT_GT(NumTableFilesAtLevel(6, temp_cfh, from_db), 1);
+
+    const bool multi_level_ingestion = rnd->OneIn(2);
+    SCOPED_TRACE("Multi-level ingestion: " +
+                 std::to_string(multi_level_ingestion));
+    if (multi_level_ingestion) {
+      for (int k = 80; k < 99; ++k) {
+        expected_values[k] = rnd->RandomString(500);
+        ASSERT_OK(from_db->Put(wo, temp_cfh, Key(k), expected_values[k]));
+      }
+      ASSERT_OK(from_db->Flush({}, temp_cfh));
+
+      // Do some overwrites, and overlap with previous L0 to avoid trivial move
+      for (int k = 70; k < 82; ++k) {
+        expected_values[k] = rnd->RandomString(500);
+        ASSERT_OK(from_db->Put(wo, temp_cfh, Key(k), expected_values[k]));
+      }
+      ASSERT_OK(from_db->Flush({}, temp_cfh));
+
+      if (rnd->OneIn(2)) {
+        MoveFilesToLevel(5, temp_cfh, from_db);
+        ASSERT_GT(NumTableFilesAtLevel(5, temp_cfh, from_db), 0);
+      }
+      ASSERT_GT(NumTableFilesAtLevel(6, temp_cfh, from_db), 0);
+    }
+    SCOPED_TRACE("LSM of from_db " + FilesPerLevel(temp_cfh, from_db));
+
+    ColumnFamilyMetaData cf_meta;
+    from_db->GetColumnFamilyMetaData(temp_cfh, &cf_meta);
+
+    // Iterate in reverse since IngestExternalFiles expect files to be ordered
+    // from old to new
+    for (auto level_meta = cf_meta.levels.rbegin();
+         level_meta != cf_meta.levels.rend(); ++level_meta) {
+      // L0 files need to be added in reverse order.
+      for (auto file_meta = level_meta->files.rbegin();
+           file_meta != level_meta->files.rend(); ++file_meta) {
+        // Validate that files contain non-zero sequence numbers
+        ASSERT_GT(file_meta->smallest_seqno, 0);
+        ASSERT_GE(file_meta->largest_seqno, file_meta->smallest_seqno);
+        sst_file_paths.emplace_back(file_meta->directory + "/" +
+                                    file_meta->relative_filename);
+      }
+    }
+    from_db->ReleaseSnapshot(snapshot);
+
+    Status s;
+    // Perform ingestion and validate results
+    if (multi_level_ingestion && options.num_levels > 1) {
+      // fail_if_bottommost requres ingesting all files into the last level,
+      // so it fails if we are assiging files to multiple levels.
+      ingest_opts.fail_if_not_bottommost_level = true;
+      s = db_->IngestExternalFile(non_overlap_cf, sst_file_paths, ingest_opts);
+      ASSERT_NOK(s);
+      ASSERT_TRUE(s.ToString().find("Files cannot be ingested to Lmax") !=
+                  std::string::npos);
+      ingest_opts.fail_if_not_bottommost_level = false;
+    }
+    if (ingest_opts.snapshot_consistency) {
+      // snapshot_consisteny requires global sequence number assignment to
+      // ingested files if there is any live snapshot.
+      snapshot = db_->GetSnapshot();
+      s = db_->IngestExternalFile(non_overlap_cf, sst_file_paths, ingest_opts);
+      ASSERT_NOK(s);
+      ASSERT_TRUE(s.ToString().find(
+          "An ingested file overlaps with existing data in the DB and has been "
+          "assigned a non-zero sequence number"));
+      db_->ReleaseSnapshot(snapshot);
+    }
+    ASSERT_OK(
+        db_->IngestExternalFile(non_overlap_cf, sst_file_paths, ingest_opts));
+
+    // Validate ingested data.
+    ReadOptions ro;
+    std::string val;
+    for (int k = 0; k < 100; ++k) {
+      s = db_->Get(ro, handles_[1], Key(k), &val);
+      ASSERT_OK(s) << "Should find ingested key " << Key(k);
+      ASSERT_EQ(val, expected_values[k]) << "key: " << Key(k);
+    }
+
+    // Overlap with data in the CF
+    if (ingest_opts.allow_blocking_flush) {
+      s = db_->IngestExternalFile(overlap_cf, sst_file_paths, ingest_opts);
+
+      ASSERT_NOK(s);
+      ASSERT_TRUE(s.ToString().find("An ingested file overlaps with existing "
+                                    "data in the DB and has been "
+                                    "assigned a non-zero sequence number") !=
+                  std::string::npos)
+          << s.ToString();
+    }
+
+    // Cleanup
+    // FIXME: Without this, the test triggers some data race between dropping
+    // CF and background compaction.
+    ASSERT_OK(db_->WaitForCompact({}));
+    if (use_temp_db) {
+      ASSERT_OK(from_db->Close());
+      delete from_db;
+      ASSERT_OK(DestroyDB(temp_db_name, temp_cf_opts));
+    } else {
+      ASSERT_OK(db_->DropColumnFamily(temp_cfh));
+      ASSERT_OK(db_->DestroyColumnFamilyHandle(temp_cfh));
+    }
+  } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction));
+}
+
+std::string GenSecondaryKey(const std::string& pk, const std::string& val) {
+  return "index_" + val + "_" + pk;
+};
+
+TEST_P(IngestDBGeneratedFileTest2, ZeroAndNonZeroSeqno) {
+  // Test ingestion of SST files with zero and with non-zero sequence numbers.
+  // Generate data using a temp CF and a temp DB:
+  // 1. Temp CF with cf_allow_ingest_behind enabled to preserve non-zero seqno.
+  // 2. Temp DB with everything compacted to have zero seqno.
+  // Then ingest both types of files together into a target CF.
+  // This mimics a user case where temp DB contains data read from a
+  // snapshot while temp CF contains live writes after a snapshot is taken.
+  IngestExternalFileOptions ingest_opts;
+  ingest_opts.allow_db_generated_files = true;
+  ingest_opts.snapshot_consistency = std::get<0>(GetParam());
+  ingest_opts.allow_global_seqno = std::get<1>(GetParam());
+  ingest_opts.allow_blocking_flush = std::get<2>(GetParam());
+  ingest_opts.fail_if_not_bottommost_level = std::get<3>(GetParam());
+  ingest_opts.link_files = std::get<4>(GetParam());
+
+  Random* rnd = Random::GetTLSInstance();
+
+  do {
+    SCOPED_TRACE("option_config_ = " + std::to_string(option_config_));
+    Options options = CurrentOptions();
+    options.allow_concurrent_memtable_write = false;
+    // Force more flushes/compactions and more files to be generated
+    options.target_file_size_base = 1 << 10;     // 1KB
+    options.max_bytes_for_level_base = 2 << 10;  // 2KB
+    options.max_bytes_for_level_multiplier = 2;
+    options.level0_file_num_compaction_trigger = 2;
+    options.level_compaction_dynamic_level_bytes = true;
+    DestroyAndReopen(options);
+    CreateAndReopenWithCF({"target_cf"}, options);
+    auto* target_cfh = handles_[1];
+
+    Options live_write_cf_opts = options;
+    live_write_cf_opts.memtable_factory.reset(new VectorRepFactory());
+    live_write_cf_opts.compaction_style = kCompactionStyleUniversal;
+    live_write_cf_opts.cf_allow_ingest_behind = true;
+    live_write_cf_opts.num_levels = 50;
+    ColumnFamilyHandle* live_write_cfh;
+    ASSERT_OK(db_->CreateColumnFamily(live_write_cf_opts, "live_write_cf",
+                                      &live_write_cfh));
+
+    // Expected value and key
+    std::map<std::string, std::string> expected;
+    std::unordered_set<std::string> deleted;
+    std::stringstream debug_info;
+
+    // Setup base data in target CF, will ingest keys with different prefixes
+    // so they don't overlap with the base data.
+    WriteOptions wo;
+    for (int k = 0; k < 100; ++k) {
+      int random_val = rnd->Uniform(20);
+      expected[Key(k)] = std::to_string(random_val);
+      ASSERT_OK(db_->Put(wo, target_cfh, Key(k), expected[Key(k)]));
+
+      // Force flush every 20 keys to create multiple SST files
+      if (rnd->OneIn(20)) {
+        ASSERT_OK(db_->Flush({}, target_cfh));
+        debug_info << "Flush after " << k
+                   << ", LSM state: " << FilesPerLevel(target_cfh) << "\n";
+      }
+    }
+
+    // Temp DB for snapshot data
+    Options temp_db_opts;
+    temp_db_opts.create_if_missing = true;
+    temp_db_opts.target_file_size_base = 1 << 10;
+    temp_db_opts.write_buffer_size = 1 << 10;
+    temp_db_opts.memtable_factory.reset(new VectorRepFactory());
+    temp_db_opts.allow_concurrent_memtable_write = false;
+    temp_db_opts.compaction_style = kCompactionStyleUniversal;
+    temp_db_opts.env = env_;
+    temp_db_opts.num_levels = 7;
+
+    std::string temp_db_name =
+        dbname_ + "/temp_db_" + std::to_string(rnd->Next());
+    DB* temp_db = nullptr;
+    ASSERT_OK(DB::Open(temp_db_opts, temp_db_name, &temp_db));
+
+    const Snapshot* snapshot = db_->GetSnapshot();
+    ReadOptions ro;
+    ro.snapshot = snapshot;
+    ro.total_order_seek = true;
+    std::unique_ptr<Iterator> iter{db_->NewIterator(ro, target_cfh)};
+    // transform data read from snapshot and write to temp DB
+    // Varying the number of files in temp DB.
+    const int kValSize = rnd->Uniform(200);
+    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+      std::string key = iter->key().ToString();
+      std::string value = iter->value().ToString();
+      std::string sk = GenSecondaryKey(key, value);
+      // Usually value is empty, here we use a larger value to generate
+      // multiple SST files in temp_db.
+      std::string sk_val = rnd->RandomString(kValSize);
+      ASSERT_OK(temp_db->Put(wo, sk, sk_val));
+      expected[sk] = sk_val;
+      debug_info << "Snapshot data: " << sk << " -> \n";
+    }
+    ASSERT_OK(iter->status());
+
+    // Do some live writes into target CF and live write CF.
+    for (int i = 0; i < 10; ++i) {
+      WriteBatch wb;
+      for (int j = 0; j < 5; ++j) {
+        std::string key = Key(rnd->Uniform(100));
+        std::string old_val = expected[key];
+        // Value range is 0-19, allow some PK to have the same value.
+        int random_val = rnd->Uniform(20);
+        std::string new_val = std::to_string(random_val);
+        std::string old_index_key = GenSecondaryKey(key, old_val);
+        std::string new_index_key = GenSecondaryKey(key, new_val);
+        ASSERT_OK(wb.SingleDelete(live_write_cfh, old_index_key));
+        std::string sk_val = rnd->RandomString(kValSize);
+        ASSERT_OK(wb.Put(live_write_cfh, new_index_key, sk_val));
+        ASSERT_OK(wb.Put(target_cfh, key, new_val));
+        expected[key] = new_val;
+        expected.erase(old_index_key);
+        expected[new_index_key] = sk_val;
+        deleted.insert(old_index_key);
+        deleted.erase(new_index_key);
+
+        debug_info << "Live write: SD " << old_index_key << "\n";
+        debug_info << "Live write: " << key << " -> " << new_val << "\n";
+        debug_info << "Live write: " << new_index_key << " -> \n";
+      }
+      ASSERT_OK(db_->Write(wo, &wb));
+      if (rnd->OneIn(3)) {
+        debug_info << "Flush after " << i << " live writes\n";
+        ASSERT_OK(db_->Flush({}, live_write_cfh));
+      }
+    }
+    iter.reset();
+    db_->ReleaseSnapshot(snapshot);
+
+    // Compact temp_db to ensure zero sequence numbers
+    CompactRangeOptions cro;
+    cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+    ASSERT_OK(temp_db->CompactRange(cro, nullptr, nullptr));
+    SCOPED_TRACE("Temp DB LSM: " +
+                 FilesPerLevel(temp_db->DefaultColumnFamily(), temp_db));
+
+    // Base data from snapshot
+    std::vector<std::string> sst_file_paths_zero_seqno;
+
+    // Collect SST file paths with zero sequence numbers
+    ASSERT_OK(temp_db->DisableFileDeletions());
+    ColumnFamilyMetaData cf_meta_temp_db;
+    temp_db->GetColumnFamilyMetaData(&cf_meta_temp_db);
+    for (const auto& level_meta : cf_meta_temp_db.levels) {
+      if (level_meta.level == 6) {
+        for (const auto& file_meta : level_meta.files) {
+          // Verify files have zero sequence numbers
+          ASSERT_EQ(0, file_meta.largest_seqno)
+              << "File " << file_meta.relative_filename
+              << " should have zero sequence number\n"
+              << debug_info.str();
+          sst_file_paths_zero_seqno.emplace_back(file_meta.directory + "/" +
+                                                 file_meta.relative_filename);
+        }
+      } else {
+        // All files should be in L6
+        ASSERT_EQ(0, level_meta.files.size()) << debug_info.str();
+      }
+    }
+
+    // Flush remaining catch up writes in memtable
+    ASSERT_OK(db_->Flush({}, live_write_cfh));
+    SCOPED_TRACE("LSM of live write cfh " + FilesPerLevel(live_write_cfh));
+    // Collect SST file paths with non-zero sequence numbers
+    ColumnFamilyMetaData live_write_cf_meta;
+    ASSERT_OK(db_->DisableFileDeletions());
+    db_->GetColumnFamilyMetaData(live_write_cfh, &live_write_cf_meta);
+
+    // Live writes after snapshot
+    std::vector<std::string> sst_file_paths_nonzero_seqno;
+    for (auto level_meta = live_write_cf_meta.levels.rbegin();
+         level_meta != live_write_cf_meta.levels.rend(); ++level_meta) {
+      // Reverse order is important for L0, where recent updates are ordered
+      // first
+      for (auto file_meta = level_meta->files.rbegin();
+           file_meta != level_meta->files.rend(); ++file_meta) {
+        sst_file_paths_nonzero_seqno.emplace_back(file_meta->directory + "/" +
+                                                  file_meta->relative_filename);
+        ASSERT_GT(file_meta->smallest_seqno, 0) << debug_info.str();
+      }
+      if (level_meta->level == 49) {
+        // Ingest behind does not compact to the last level
+        ASSERT_EQ(level_meta->files.size(), 0) << debug_info.str();
+      }
+    }
+
+    ASSERT_GT(sst_file_paths_zero_seqno.size(), 0) << debug_info.str();
+    ASSERT_GT(sst_file_paths_nonzero_seqno.size(), 0) << debug_info.str();
+
+    // Combine all SST file paths.
+    // File ingestion takes files from old to new.
+    std::vector<std::string> all_sst_files;
+    all_sst_files.insert(all_sst_files.end(), sst_file_paths_zero_seqno.begin(),
+                         sst_file_paths_zero_seqno.end());
+    all_sst_files.insert(all_sst_files.end(),
+                         sst_file_paths_nonzero_seqno.begin(),
+                         sst_file_paths_nonzero_seqno.end());
+    if (ingest_opts.fail_if_not_bottommost_level && options.num_levels > 1) {
+      // overlapping files will be ingested into different levels, including non
+      // Lmax
+      Status s =
+          db_->IngestExternalFile(target_cfh, all_sst_files, ingest_opts);
+      ASSERT_NOK(s);
+      ASSERT_TRUE(s.ToString().find("Files cannot be ingested to Lmax") !=
+                  std::string::npos);
+    } else {
+      ASSERT_OK(
+          db_->IngestExternalFile(target_cfh, all_sst_files, ingest_opts));
+
+      debug_info << "Zero seqno files: " << sst_file_paths_zero_seqno.size()
+                 << "\nNon-zero seqno files: "
+                 << sst_file_paths_nonzero_seqno.size() << "\n";
+
+      SCOPED_TRACE("Debug info:\n" + debug_info.str());
+      VerifyDBFromMap(expected, nullptr, false, nullptr, target_cfh, &deleted);
+    }
+
+    // clean up
+    ASSERT_OK(db_->EnableFileDeletions());
+    ASSERT_OK(temp_db->EnableFileDeletions());
+
+    // FIXME: Without this, the test triggers some data race between dropping
+    // CF and background compaction.
+    ASSERT_OK(db_->WaitForCompact({}));
+
+    ASSERT_OK(db_->DropColumnFamily(live_write_cfh));
+    ASSERT_OK(db_->DestroyColumnFamilyHandle(live_write_cfh));
+
+    ASSERT_OK(temp_db->Close());
+    delete temp_db;
+    ASSERT_OK(DestroyDB(temp_db_name, temp_db_opts));
+  } while (ChangeOptions(kSkipPlainTable | kSkipFIFOCompaction));
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/flush_job.cc

@@ -502,8 +502,7 @@ Status FlushJob::MemPurge() {
         kMaxSequenceNumber, &job_context_->snapshot_seqs, earliest_snapshot_,
         job_context_->earliest_write_conflict_snapshot,
         job_context_->GetJobSnapshotSequence(), job_context_->snapshot_checker,
-        env, ShouldReportDetailedTime(env, ioptions.stats),
-        true /* internal key corruption is not ok */, range_del_agg.get(),
+        env, ShouldReportDetailedTime(env, ioptions.stats), range_del_agg.get(),
         nullptr, ioptions.allow_data_in_errors,
         ioptions.enforce_single_del_contracts,
         /*manual_compaction_canceled=*/kManualCompactionCanceledFalse,

db/log_reader.cc

@@ -380,8 +380,11 @@ void Reader::MaybeVerifyPredecessorWALInfo(
   } else {
     if (observed_predecessor_wal_info_.GetLogNumber() !=
         recorded_predecessor_log_number) {
-      std::string reason = "Missing WAL of log number " +
-                           std::to_string(recorded_predecessor_log_number);
+      std::string reason =
+          "Mismatched predecessor log number of WAL file " +
+          file_->file_name() + " Recorded " +
+          std::to_string(recorded_predecessor_log_number) + ". Observed " +
+          std::to_string(observed_predecessor_wal_info_.GetLogNumber());
       ReportCorruption(fragment.size(), reason.c_str(),
                        recorded_predecessor_log_number);
     } else if (observed_predecessor_wal_info_.GetLastSeqnoRecorded() !=

db/memtable.cc

@@ -134,6 +134,16 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
   auto new_cache = std::make_shared<FragmentedRangeTombstoneListCache>();
   size_t size = cached_range_tombstone_.Size();
   for (size_t i = 0; i < size; ++i) {
+#if defined(__cpp_lib_atomic_shared_ptr)
+    std::atomic<std::shared_ptr<FragmentedRangeTombstoneListCache>>*
+        local_cache_ref_ptr = cached_range_tombstone_.AccessAtCore(i);
+    auto new_local_cache_ref = std::make_shared<
+        const std::shared_ptr<FragmentedRangeTombstoneListCache>>(new_cache);
+    std::shared_ptr<FragmentedRangeTombstoneListCache> aliased_ptr(
+        new_local_cache_ref, new_cache.get());
+    local_cache_ref_ptr->store(std::move(aliased_ptr),
+                               std::memory_order_relaxed);
+#else
     std::shared_ptr<FragmentedRangeTombstoneListCache>* local_cache_ref_ptr =
         cached_range_tombstone_.AccessAtCore(i);
     auto new_local_cache_ref = std::make_shared<
@@ -143,6 +153,7 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
         std::shared_ptr<FragmentedRangeTombstoneListCache>(new_local_cache_ref,
                                                            new_cache.get()),
         std::memory_order_relaxed);
+#endif
   }
   const Comparator* ucmp = cmp.user_comparator();
   assert(ucmp);
@@ -168,7 +179,7 @@ size_t MemTable::ApproximateMemoryUsage() {
     }
     total_usage += usage;
   }
-  approximate_memory_usage_.store(total_usage, std::memory_order_relaxed);
+  approximate_memory_usage_.StoreRelaxed(total_usage);
   // otherwise, return the actual usage
   return total_usage;
 }
@@ -182,12 +193,12 @@ bool MemTable::ShouldFlushNow() {
   // This is set if memtable_max_range_deletions is > 0,
   // and that many range deletions are done
   if (memtable_max_range_deletions_ > 0 &&
-      num_range_deletes_.load(std::memory_order_relaxed) >=
+      num_range_deletes_.LoadRelaxed() >=
           static_cast<uint64_t>(memtable_max_range_deletions_)) {
     return true;
   }
 
-  size_t write_buffer_size = write_buffer_size_.load(std::memory_order_relaxed);
+  size_t write_buffer_size = write_buffer_size_.LoadRelaxed();
   // In a lot of times, we cannot allocate arena blocks that exactly matches the
   // buffer size. Thus we have to decide if we should over-allocate or
   // under-allocate.
@@ -200,10 +211,10 @@ bool MemTable::ShouldFlushNow() {
   assert(range_del_table_->ApproximateMemoryUsage() == 0);
   // If arena still have room for new block allocation, we can safely say it
   // shouldn't flush.
-  auto allocated_memory = table_->ApproximateMemoryUsage() +
-                          arena_.MemoryAllocatedBytes();
+  auto allocated_memory =
+      table_->ApproximateMemoryUsage() + arena_.MemoryAllocatedBytes();
 
-  approximate_memory_usage_.store(allocated_memory, std::memory_order_relaxed);
+  approximate_memory_usage_.StoreRelaxed(allocated_memory);
 
   // if we can still allocate one more block without exceeding the
   // over-allocation ratio, then we should not flush.
@@ -745,7 +756,7 @@ FragmentedRangeTombstoneIterator* MemTable::NewRangeTombstoneIterator(
     const ReadOptions& read_options, SequenceNumber read_seq,
     bool immutable_memtable) {
   if (read_options.ignore_range_deletions ||
-      is_range_del_table_empty_.load(std::memory_order_relaxed)) {
+      is_range_del_table_empty_.LoadRelaxed()) {
     return nullptr;
   }
   return NewRangeTombstoneIteratorInternal(read_options, read_seq,
@@ -756,7 +767,7 @@ FragmentedRangeTombstoneIterator*
 MemTable::NewTimestampStrippingRangeTombstoneIterator(
     const ReadOptions& read_options, SequenceNumber read_seq, size_t ts_sz) {
   if (read_options.ignore_range_deletions ||
-      is_range_del_table_empty_.load(std::memory_order_relaxed)) {
+      is_range_del_table_empty_.LoadRelaxed()) {
     return nullptr;
   }
   if (!timestamp_stripping_fragmented_range_tombstone_list_) {
@@ -790,8 +801,13 @@ FragmentedRangeTombstoneIterator* MemTable::NewRangeTombstoneIteratorInternal(
 
   // takes current cache
   std::shared_ptr<FragmentedRangeTombstoneListCache> cache =
+#if defined(__cpp_lib_atomic_shared_ptr)
+      cached_range_tombstone_.Access()->load(std::memory_order_relaxed)
+#else
       std::atomic_load_explicit(cached_range_tombstone_.Access(),
-                                std::memory_order_relaxed);
+                                std::memory_order_relaxed)
+#endif
+      ;
   // construct fragmented tombstone list if necessary
   if (!cache->initialized.load(std::memory_order_acquire)) {
     cache->reader_mutex.lock();
@@ -815,7 +831,7 @@ void MemTable::ConstructFragmentedRangeTombstones() {
   // There should be no concurrent Construction.
   // We could also check fragmented_range_tombstone_list_ to avoid repeate
   // constructions. We just construct them here again to be safe.
-  if (!is_range_del_table_empty_.load(std::memory_order_relaxed)) {
+  if (!is_range_del_table_empty_.LoadRelaxed()) {
     // TODO: plumb Env::IOActivity, Env::IOPriority
     auto* unfragmented_iter = new MemTableIterator(
         MemTableIterator::kRangeDelEntries, *this, ReadOptions());
@@ -838,7 +854,7 @@ ReadOnlyMemTable::MemTableStats MemTable::ApproximateStats(
   if (entry_count == 0) {
     return {0, 0};
   }
-  uint64_t n = num_entries_.load(std::memory_order_relaxed);
+  uint64_t n = num_entries_.LoadRelaxed();
   if (n == 0) {
     return {0, 0};
   }
@@ -848,7 +864,7 @@ ReadOnlyMemTable::MemTableStats MemTable::ApproximateStats(
     // the inaccuracy.
     entry_count = n;
   }
-  uint64_t data_size = data_size_.load(std::memory_order_relaxed);
+  uint64_t data_size = data_size_.LoadRelaxed();
   return {entry_count * (data_size / n), entry_count};
 }
 
@@ -978,17 +994,14 @@ Status MemTable::Add(SequenceNumber s, ValueType type,
 
     // this is a bit ugly, but is the way to avoid locked instructions
     // when incrementing an atomic
-    num_entries_.store(num_entries_.load(std::memory_order_relaxed) + 1,
-                       std::memory_order_relaxed);
-    data_size_.store(data_size_.load(std::memory_order_relaxed) + encoded_len,
-                     std::memory_order_relaxed);
+    num_entries_.StoreRelaxed(num_entries_.LoadRelaxed() + 1);
+    data_size_.StoreRelaxed(data_size_.LoadRelaxed() + encoded_len);
     if (type == kTypeDeletion || type == kTypeSingleDeletion ||
         type == kTypeDeletionWithTimestamp) {
-      num_deletes_.store(num_deletes_.load(std::memory_order_relaxed) + 1,
-                         std::memory_order_relaxed);
+      num_deletes_.StoreRelaxed(num_deletes_.LoadRelaxed() + 1);
     } else if (type == kTypeRangeDeletion) {
-      uint64_t val = num_range_deletes_.load(std::memory_order_relaxed) + 1;
-      num_range_deletes_.store(val, std::memory_order_relaxed);
+      uint64_t val = num_range_deletes_.LoadRelaxed() + 1;
+      num_range_deletes_.StoreRelaxed(val);
     }
 
     if (bloom_filter_ && prefix_extractor_ &&
@@ -1059,6 +1072,16 @@ Status MemTable::Add(SequenceNumber s, ValueType type,
       range_del_mutex_.lock();
     }
     for (size_t i = 0; i < size; ++i) {
+#if defined(__cpp_lib_atomic_shared_ptr)
+      std::atomic<std::shared_ptr<FragmentedRangeTombstoneListCache>>*
+          local_cache_ref_ptr = cached_range_tombstone_.AccessAtCore(i);
+      auto new_local_cache_ref = std::make_shared<
+          const std::shared_ptr<FragmentedRangeTombstoneListCache>>(new_cache);
+      std::shared_ptr<FragmentedRangeTombstoneListCache> aliased_ptr(
+          new_local_cache_ref, new_cache.get());
+      local_cache_ref_ptr->store(std::move(aliased_ptr),
+                                 std::memory_order_relaxed);
+#else
       std::shared_ptr<FragmentedRangeTombstoneListCache>* local_cache_ref_ptr =
           cached_range_tombstone_.AccessAtCore(i);
       auto new_local_cache_ref = std::make_shared<
@@ -1073,12 +1096,13 @@ Status MemTable::Add(SequenceNumber s, ValueType type,
           std::shared_ptr<FragmentedRangeTombstoneListCache>(
               new_local_cache_ref, new_cache.get()),
           std::memory_order_relaxed);
+#endif
     }
 
     if (allow_concurrent) {
       range_del_mutex_.unlock();
     }
-    is_range_del_table_empty_.store(false, std::memory_order_relaxed);
+    is_range_del_table_empty_.StoreRelaxed(false);
   }
   UpdateOldestKeyTime();
 
@@ -1497,7 +1521,7 @@ void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
   // range tombstones. This is the simplest way to ensure range tombstones are
   // handled. TODO: allow Bloom checks where max_covering_tombstone_seq==0
   bool no_range_del = read_options.ignore_range_deletions ||
-                      is_range_del_table_empty_.load(std::memory_order_relaxed);
+                      is_range_del_table_empty_.LoadRelaxed();
   MultiGetRange temp_range(*range, range->begin(), range->end());
   if (bloom_filter_ && no_range_del) {
     bool whole_key =

db/memtable.h

@@ -8,7 +8,6 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #pragma once
-#include <atomic>
 #include <deque>
 #include <functional>
 #include <memory>
@@ -568,7 +567,7 @@ class MemTable final : public ReadOnlyMemTable {
   // As a cheap version of `ApproximateMemoryUsage()`, this function doesn't
   // require external synchronization. The value may be less accurate though
   size_t ApproximateMemoryUsageFast() const {
-    return approximate_memory_usage_.load(std::memory_order_relaxed);
+    return approximate_memory_usage_.LoadRelaxed();
   }
 
   size_t MemoryAllocatedBytes() const override {
@@ -689,16 +688,13 @@ class MemTable final : public ReadOnlyMemTable {
   // Used in concurrent memtable inserts.
   void BatchPostProcess(const MemTablePostProcessInfo& update_counters) {
     table_->BatchPostProcess();
-    num_entries_.fetch_add(update_counters.num_entries,
-                           std::memory_order_relaxed);
-    data_size_.fetch_add(update_counters.data_size, std::memory_order_relaxed);
+    num_entries_.FetchAddRelaxed(update_counters.num_entries);
+    data_size_.FetchAddRelaxed(update_counters.data_size);
     if (update_counters.num_deletes != 0) {
-      num_deletes_.fetch_add(update_counters.num_deletes,
-                             std::memory_order_relaxed);
+      num_deletes_.FetchAddRelaxed(update_counters.num_deletes);
     }
     if (update_counters.num_range_deletes > 0) {
-      num_range_deletes_.fetch_add(update_counters.num_range_deletes,
-                                   std::memory_order_relaxed);
+      num_range_deletes_.FetchAddRelaxed(update_counters.num_range_deletes);
       // noop for skip-list memtable
       // Besides correctness test in stress test, memtable flush record count
       // check will catch this if it were not noop.
@@ -707,35 +703,26 @@ class MemTable final : public ReadOnlyMemTable {
     UpdateFlushState();
   }
 
-  uint64_t NumEntries() const override {
-    return num_entries_.load(std::memory_order_relaxed);
-  }
+  uint64_t NumEntries() const override { return num_entries_.LoadRelaxed(); }
 
-  uint64_t NumDeletion() const override {
-    return num_deletes_.load(std::memory_order_relaxed);
-  }
+  uint64_t NumDeletion() const override { return num_deletes_.LoadRelaxed(); }
 
   uint64_t NumRangeDeletion() const override {
-    return num_range_deletes_.load(std::memory_order_relaxed);
+    return num_range_deletes_.LoadRelaxed();
   }
 
-  uint64_t GetDataSize() const override {
-    return data_size_.load(std::memory_order_relaxed);
-  }
+  uint64_t GetDataSize() const override { return data_size_.LoadRelaxed(); }
 
-  size_t write_buffer_size() const {
-    return write_buffer_size_.load(std::memory_order_relaxed);
-  }
+  size_t write_buffer_size() const { return write_buffer_size_.LoadRelaxed(); }
 
   // Dynamically change the memtable's capacity. If set below the current usage,
   // the next key added will trigger a flush. Can only increase size when
   // memtable prefix bloom is disabled, since we can't easily allocate more
-  // space.
+  // space. Non-atomic update ok because this is only called with DB mutex held.
   void UpdateWriteBufferSize(size_t new_write_buffer_size) {
     if (bloom_filter_ == nullptr ||
-        new_write_buffer_size < write_buffer_size_) {
-      write_buffer_size_.store(new_write_buffer_size,
-                               std::memory_order_relaxed);
+        new_write_buffer_size < write_buffer_size_.LoadRelaxed()) {
+      write_buffer_size_.StoreRelaxed(new_write_buffer_size);
     }
   }
 
@@ -827,7 +814,7 @@ class MemTable final : public ReadOnlyMemTable {
 
   bool IsFragmentedRangeTombstonesConstructed() const override {
     return fragmented_range_tombstone_list_.get() != nullptr ||
-           is_range_del_table_empty_;
+           is_range_del_table_empty_.LoadRelaxed();
   }
 
   //  Gets the newest user defined timestamps in the memtable. This should only
@@ -853,16 +840,22 @@ class MemTable final : public ReadOnlyMemTable {
   ConcurrentArena arena_;
   std::unique_ptr<MemTableRep> table_;
   std::unique_ptr<MemTableRep> range_del_table_;
-  std::atomic_bool is_range_del_table_empty_;
+  // This is OK to be relaxed access because consistency between table_ and
+  // range_del_table_ is provided by explicit multi-versioning with sequence
+  // numbers. It's ok for stale memory to say the range_del_table_ is empty when
+  // it's actually not because if it was relevant to our read (based on sequence
+  // number), the relaxed memory read would get a sufficiently updated value
+  // because of the ordering provided by LastPublishedSequence().
+  RelaxedAtomic<bool> is_range_del_table_empty_;
 
   // Total data size of all data inserted
-  std::atomic<uint64_t> data_size_;
-  std::atomic<uint64_t> num_entries_;
-  std::atomic<uint64_t> num_deletes_;
-  std::atomic<uint64_t> num_range_deletes_;
+  RelaxedAtomic<uint64_t> data_size_;
+  RelaxedAtomic<uint64_t> num_entries_;
+  RelaxedAtomic<uint64_t> num_deletes_;
+  RelaxedAtomic<uint64_t> num_range_deletes_;
 
   // Dynamically changeable memtable option
-  std::atomic<size_t> write_buffer_size_;
+  RelaxedAtomic<size_t> write_buffer_size_;
 
   // The sequence number of the kv that was inserted first
   std::atomic<SequenceNumber> first_seqno_;
@@ -898,7 +891,7 @@ class MemTable final : public ReadOnlyMemTable {
 
   // keep track of memory usage in table_, arena_, and range_del_table_.
   // Gets refreshed inside `ApproximateMemoryUsage()` or `ShouldFlushNow`
-  std::atomic<uint64_t> approximate_memory_usage_;
+  RelaxedAtomic<uint64_t> approximate_memory_usage_;
 
   // max range deletions in a memtable,  before automatic flushing, 0 for
   // unlimited.
@@ -949,9 +942,15 @@ class MemTable final : public ReadOnlyMemTable {
 
   // makes sure there is a single range tombstone writer to invalidate cache
   std::mutex range_del_mutex_;
+#if defined(__cpp_lib_atomic_shared_ptr)
+  CoreLocalArray<
+      std::atomic<std::shared_ptr<FragmentedRangeTombstoneListCache>>>
+      cached_range_tombstone_;
+#else
   CoreLocalArray<std::shared_ptr<FragmentedRangeTombstoneListCache>>
       cached_range_tombstone_;
 
+#endif
   void UpdateEntryChecksum(const ProtectionInfoKVOS64* kv_prot_info,
                            const Slice& key, const Slice& value, ValueType type,
                            SequenceNumber s, char* checksum_ptr);

db/merge_helper.cc

@@ -497,6 +497,7 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
           ikey.sequence <= latest_snapshot_
               ? CompactionFilter::Decision::kKeep
               : FilterMerge(orig_ikey.user_key, value_slice);
+      // FIXME: should also check for kRemove here
       if (filter != CompactionFilter::Decision::kRemoveAndSkipUntil &&
           range_del_agg != nullptr &&
           range_del_agg->ShouldDelete(

db/multi_scan.cc

@@ -10,24 +10,25 @@ namespace ROCKSDB_NAMESPACE {
 using MultiScanIterator = MultiScan::MultiScanIterator;
 
 MultiScan::MultiScan(const ReadOptions& read_options,
-                     const std::vector<ScanOptions>& scan_opts, DB* db,
+                     const MultiScanArgs& scan_opts, DB* db,
                      ColumnFamilyHandle* cfh)
     : read_options_(read_options), scan_opts_(scan_opts), db_(db), cfh_(cfh) {
   bool slow_path = false;
   // Setup read_options with iterate_uuper_bound based on the first scan.
   // Subsequent scans will update and allocate a new DB iterator as necessary
-  if (scan_opts[0].range.limit) {
-    upper_bound_ = *scan_opts[0].range.limit;
+  if (scan_opts.GetScanRanges()[0].range.limit) {
+    upper_bound_ = *scan_opts.GetScanRanges()[0].range.limit;
     read_options_.iterate_upper_bound = &upper_bound_;
   } else {
     read_options_.iterate_upper_bound = nullptr;
   }
-  for (auto opts : scan_opts) {
+  for (const auto& opts : scan_opts.GetScanRanges()) {
     // Check that all the ScanOptions either specify an upper bound or not. If
     // its mixed we take the slow path which avoids calling Prepare: we have to
     // reallocate the Iterator with updated read_options everytime we switch
     // between upper bound or no upper bound, which complicates Prepare.
-    if (opts.range.limit.has_value() != scan_opts[0].range.limit.has_value()) {
+    if (opts.range.limit.has_value() !=
+        scan_opts.GetScanRanges()[0].range.limit.has_value()) {
       slow_path = true;
       break;
     }

db/version_edit.h

@@ -110,7 +110,7 @@ constexpr uint64_t kUnknownOldestAncesterTime = 0;
 constexpr uint64_t kUnknownNewestKeyTime = 0;
 constexpr uint64_t kUnknownFileCreationTime = 0;
 constexpr uint64_t kUnknownEpochNumber = 0;
-// If `Options::allow_ingest_behind` is true, this epoch number
+// If `Options::cf_allow_ingest_behind` is true, this epoch number
 // will be dedicated to files ingested behind.
 constexpr uint64_t kReservedEpochNumberForFileIngestedBehind = 1;
 

db/version_edit_handler.cc

@@ -117,21 +117,43 @@ Status ListColumnFamiliesHandler::ApplyVersionEdit(
   return s;
 }
 
-Status FileChecksumRetriever::ApplyVersionEdit(VersionEdit& edit,
-                                               ColumnFamilyData** /*unused*/) {
-  for (const auto& deleted_file : edit.GetDeletedFiles()) {
-    Status s = file_checksum_list_.RemoveOneFileChecksum(deleted_file.second);
-    if (!s.ok()) {
-      return s;
+Status FileChecksumRetriever::FetchFileChecksumList(
+    FileChecksumList& file_checksum_list) {
+  Status s = Status::OK();
+  for (const auto& [cf, file_checksums] : cf_file_checksums_) {
+    [[maybe_unused]] const auto& _ = cf;
+    for (const auto& [file_number, info] : file_checksums) {
+      if (!(s = file_checksum_list.InsertOneFileChecksum(
+                file_number, info.first, info.second))
+               .ok()) {
+        break;
+      }
     }
   }
-  for (const auto& new_file : edit.GetNewFiles()) {
-    Status s = file_checksum_list_.InsertOneFileChecksum(
-        new_file.second.fd.GetNumber(), new_file.second.file_checksum,
-        new_file.second.file_checksum_func_name);
-    if (!s.ok()) {
-      return s;
+  return s;
+}
+
+Status FileChecksumRetriever::ApplyVersionEdit(VersionEdit& edit,
+                                               ColumnFamilyData** /*unused*/) {
+  uint32_t column_family_id = edit.GetColumnFamily();
+  if (edit.IsColumnFamilyDrop()) {
+    cf_file_checksums_.erase(column_family_id);
+  }
+  for (const auto& deleted_file : edit.GetDeletedFiles()) {
+    if (cf_file_checksums_.find(column_family_id) == cf_file_checksums_.end()) {
+      return Status::NotFound();
     }
+    if (cf_file_checksums_[column_family_id].find(deleted_file.second) ==
+        cf_file_checksums_[column_family_id].end()) {
+      return Status::NotFound();
+    }
+    cf_file_checksums_[column_family_id].erase(deleted_file.second);
+  }
+  for (const auto& new_file : edit.GetNewFiles()) {
+    cf_file_checksums_[column_family_id].emplace(
+        new_file.second.fd.GetNumber(),
+        std::make_pair(new_file.second.file_checksum,
+                       new_file.second.file_checksum_func_name));
   }
   for (const auto& new_blob_file : edit.GetBlobFileAdditions()) {
     std::string checksum_value = new_blob_file.GetChecksumValue();
@@ -141,11 +163,9 @@ Status FileChecksumRetriever::ApplyVersionEdit(VersionEdit& edit,
       checksum_value = kUnknownFileChecksum;
       checksum_method = kUnknownFileChecksumFuncName;
     }
-    Status s = file_checksum_list_.InsertOneFileChecksum(
-        new_blob_file.GetBlobFileNumber(), checksum_value, checksum_method);
-    if (!s.ok()) {
-      return s;
-    }
+    cf_file_checksums_[column_family_id].emplace(
+        new_blob_file.GetBlobFileNumber(),
+        std::make_pair(checksum_value, checksum_method));
   }
   return Status::OK();
 }

db/version_edit_handler.h

@@ -80,19 +80,42 @@ class ListColumnFamiliesHandler : public VersionEditHandlerBase {
 
 class FileChecksumRetriever : public VersionEditHandlerBase {
  public:
-  FileChecksumRetriever(const ReadOptions& read_options, uint64_t max_read_size,
-                        FileChecksumList& file_checksum_list)
-      : VersionEditHandlerBase(read_options, max_read_size),
-        file_checksum_list_(file_checksum_list) {}
+  FileChecksumRetriever(const ReadOptions& read_options, uint64_t max_read_size)
+      : VersionEditHandlerBase(read_options, max_read_size) {}
 
   ~FileChecksumRetriever() override {}
 
+  Status FetchFileChecksumList(FileChecksumList& file_checksum_list);
+
  protected:
   Status ApplyVersionEdit(VersionEdit& edit,
                           ColumnFamilyData** /*unused*/) override;
 
  private:
-  FileChecksumList& file_checksum_list_;
+  // Map from CF to file # to string pair, where first portion of the value
+  // is checksum, and second portion of the value is checksum function name.
+  //
+  // [column family id A]
+  //      |
+  //      |-- [file #1] -> [checksum #1, checksum function name #1]
+  //      |-- [file #2] -> [checksum #2, checksum function name #2]
+  //      |
+  //     ...
+  //      |
+  //      |-- [file #N] -> [checksum #N, checksum function name #N]
+  // [column family id B]
+  //      |
+  //      |-- [file #1] -> [checksum #1, checksum function name #1]
+  //      |
+  //     ...
+  //      |
+  //      |-- [file #M] -> [checksum #M, checksum function name #M]
+  //      |
+  //     ...
+  std::unordered_map<
+      uint32_t,
+      std::unordered_map<uint64_t, std::pair<std::string, std::string>>>
+      cf_file_checksums_;
 };
 
 using VersionBuilderUPtr = std::unique_ptr<BaseReferencedVersionBuilder>;

db/version_set.cc

@@ -95,7 +95,7 @@ namespace ROCKSDB_NAMESPACE {
 
 namespace {
 
-using ScanOptionsMap = std::unordered_map<size_t, std::vector<ScanOptions>>;
+using ScanOptionsMap = std::unordered_map<size_t, MultiScanArgs>;
 
 // Find File in LevelFilesBrief data structure
 // Within an index range defined by left and right
@@ -1101,17 +1101,17 @@ class LevelIterator final : public InternalIterator {
     read_seq_ = read_seq;
   }
 
-  void Prepare(const std::vector<ScanOptions>* scan_opts) override {
+  void Prepare(const MultiScanArgs* so) override {
     // We assume here that scan_opts is sorted such that
     // scan_opts[0].range.start < scan_opts[1].range.start, and non overlapping
-    scan_opts_ = scan_opts;
-    if (scan_opts_ == nullptr) {
+    if (so == nullptr) {
       return;
     }
+    scan_opts_ = so;
 
     file_to_scan_opts_ = std::make_unique<ScanOptionsMap>();
     for (size_t k = 0; k < scan_opts_->size(); k++) {
-      const ScanOptions& opt = scan_opts_->at(k);
+      const ScanOptions& opt = scan_opts_->GetScanRanges().at(k);
       auto start = opt.range.start;
       auto end = opt.range.limit;
 
@@ -1139,11 +1139,15 @@ class LevelIterator final : public InternalIterator {
       // 3. [  S  ] ...... [  E  ]
       for (auto i = fstart; i <= fend; i++) {
         if (i < flevel_->num_files) {
-          (*file_to_scan_opts_)[i].emplace_back(start.value(), end.value());
-          (*file_to_scan_opts_)[i].back().property_bag = opt.property_bag;
+          (*file_to_scan_opts_)[i].insert(start.value(), end.value(),
+                                          opt.property_bag);
         }
       }
     }
+    // Propagate io colaescing threshold
+    for (auto& file_to_arg : *file_to_scan_opts_) {
+      file_to_arg.second.io_coalesce_threshold = so->io_coalesce_threshold;
+    }
   }
 
  private:
@@ -1271,7 +1275,7 @@ class LevelIterator final : public InternalIterator {
   bool prefix_exhausted_ = false;
   // Whether next/prev key is a sentinel key.
   bool to_return_sentinel_ = false;
-  const std::vector<ScanOptions>* scan_opts_;
+  const MultiScanArgs* scan_opts_ = nullptr;
 
   // Our stored scan_opts for each prefix
   std::unique_ptr<ScanOptionsMap> file_to_scan_opts_ = nullptr;
@@ -1540,7 +1544,8 @@ bool LevelIterator::SkipEmptyFileForward() {
       // If we are doing prepared scan opts then we should seek to the values
       // specified by the scan opts
       if (scan_opts_ && (*file_to_scan_opts_)[file_index_].size()) {
-        const ScanOptions& opts = file_to_scan_opts_->at(file_index_).front();
+        const ScanOptions& opts =
+            file_to_scan_opts_->at(file_index_).GetScanRanges().front();
         if (opts.range.start.has_value()) {
           InternalKey target(*opts.range.start.AsPtr(), kMaxSequenceNumber,
                              kValueTypeForSeek);
@@ -1599,9 +1604,8 @@ void LevelIterator::SetFileIterator(InternalIterator* iter) {
   if (iter && scan_opts_) {
     if (file_to_scan_opts_.get() &&
         file_to_scan_opts_->find(file_index_) != file_to_scan_opts_->end()) {
-      const std::vector<ScanOptions>& opts =
-          file_to_scan_opts_->at(file_index_);
-      file_iter_.Prepare(&opts);
+      const MultiScanArgs& new_opts = file_to_scan_opts_->at(file_index_);
+      file_iter_.Prepare(&new_opts);
     } else {
       file_iter_.Prepare(scan_opts_);
     }
@@ -3528,7 +3532,9 @@ void VersionStorageInfo::ComputeCompactionScore(
   // maintaining it to be over 1.0, we scale the original score by 10x
   // if it is larger than 1.0.
   const double kScoreScale = 10.0;
-  int max_output_level = MaxOutputLevel(immutable_options.allow_ingest_behind);
+  int max_output_level =
+      MaxOutputLevel(immutable_options.cf_allow_ingest_behind ||
+                     immutable_options.allow_ingest_behind);
   for (int level = 0; level <= MaxInputLevel(); level++) {
     double score;
     if (level == 0) {
@@ -3713,6 +3719,7 @@ void VersionStorageInfo::ComputeCompactionScore(
   }
   ComputeFilesMarkedForCompaction(max_output_level);
   ComputeBottommostFilesMarkedForCompaction(
+      immutable_options.cf_allow_ingest_behind ||
       immutable_options.allow_ingest_behind);
   ComputeExpiredTtlFiles(immutable_options, mutable_cf_options.ttl);
   ComputeFilesMarkedForPeriodicCompaction(
@@ -4710,8 +4717,7 @@ void VersionStorageInfo::RecoverEpochNumbers(ColumnFamilyData* cfd,
   if (restart_epoch) {
     cfd->ResetNextEpochNumber();
 
-    bool reserve_epoch_num_for_file_ingested_behind =
-        cfd->ioptions().allow_ingest_behind;
+    bool reserve_epoch_num_for_file_ingested_behind = cfd->AllowIngestBehind();
     if (reserve_epoch_num_for_file_ingested_behind) {
       uint64_t reserved_epoch_number = cfd->NewEpochNumber();
       assert(reserved_epoch_number ==
@@ -4719,7 +4725,8 @@ void VersionStorageInfo::RecoverEpochNumbers(ColumnFamilyData* cfd,
       ROCKS_LOG_INFO(cfd->ioptions().info_log.get(),
                      "[%s]CF has reserved epoch number %" PRIu64
                      " for files ingested "
-                     "behind since `Options::allow_ingest_behind` is true",
+                     "behind since `Options::allow_ingest_behind` or "
+                     "`Options::cf_allow_ingest_behind` is true",
                      cfd->GetName().c_str(), reserved_epoch_number);
     }
   }
@@ -6488,7 +6495,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
                       nullptr /*BlockCacheTracer*/, nullptr /*IOTracer*/,
                       /*db_id*/ "",
                       /*db_session_id*/ "", options->daily_offpeak_time_utc,
-                      /*error_handler_*/ nullptr, /*read_only=*/false);
+                      /*error_handler_*/ nullptr, /*unchanging=*/false);
   Status status;
 
   std::vector<ColumnFamilyDescriptor> dummy;

db/version_set.h

@@ -669,6 +669,8 @@ class VersionStorageInfo {
 
   // List of files per level, files in each level are arranged
   // in increasing order of keys
+  // In L0, files are ordered in decreasing epoch number, meaning
+  // more recent updates are ordered first.
   std::vector<FileMetaData*>* files_;
 
   // Map of all table files in version. Maps file number to (level, position on

db_stress_tool/CMakeLists.txt

@@ -2,6 +2,7 @@ add_executable(db_stress${ARTIFACT_SUFFIX}
   batched_ops_stress.cc
   cf_consistency_stress.cc
   db_stress.cc
+  db_stress_compression_manager.cc
   db_stress_common.cc
   db_stress_driver.cc
   db_stress_filters.cc

db_stress_tool/db_stress_common.cc

@@ -228,6 +228,66 @@ void CompressedCacheSetCapacityThread(void* v) {
   }
 }
 
+void RemoteCompactionWorkerThread(void* v) {
+  assert(FLAGS_remote_compaction_worker_threads > 0);
+  assert(FLAGS_remote_compaction_worker_interval > 0);
+  auto* thread = static_cast<ThreadState*>(v);
+  SharedState* shared = thread->shared;
+  StressTest* stress_test = shared->GetStressTest();
+  assert(stress_test != nullptr);
+  while (true) {
+    {
+      MutexLock l(shared->GetMutex());
+      if (shared->ShouldStopBgThread()) {
+        shared->IncBgThreadsFinished();
+        if (shared->BgThreadsFinished()) {
+          shared->GetCondVar()->SignalAll();
+        }
+        return;
+      }
+    }
+    std::string job_id;
+    CompactionServiceJobInfo job_info;
+    std::string serialized_input;
+    if (shared->DequeueRemoteCompaction(&job_id, &job_info,
+                                        &serialized_input)) {
+      auto options = stress_test->GetOptions(job_info.cf_id);
+      CompactionServiceOptionsOverride override_options{
+          .file_checksum_gen_factory = options.file_checksum_gen_factory,
+          .merge_operator = options.merge_operator,
+          .compaction_filter = options.compaction_filter,
+          .compaction_filter_factory = options.compaction_filter_factory,
+          .prefix_extractor = options.prefix_extractor,
+          .table_factory = options.table_factory,
+          .sst_partitioner_factory = options.sst_partitioner_factory,
+          .listeners = {},
+          .statistics = options.statistics,
+          .table_properties_collector_factories =
+              options.table_properties_collector_factories};
+      std::string tmp_output_dir = job_info.db_name + "/" + "tmp_output_" +
+                                   db_stress_env->GenerateUniqueId();
+      std::string serialized_output;
+      Status s = DB::OpenAndCompact(OpenAndCompactOptions{}, job_info.db_name,
+                                    tmp_output_dir, serialized_input,
+                                    &serialized_output, override_options);
+      if (!s.ok()) {
+        // Print in stdout instead of stderr to avoid stress test failure,
+        // because OpenAndCompact() failure doesn't necessarily mean
+        // primary db instance failure.
+        fprintf(stdout, "Failed to run OpenAndCompact(%s): %s\n",
+                job_info.db_name.c_str(), s.ToString().c_str());
+      }
+      // Add the output regardless of status, so that primary DB doesn't rely on
+      // the timeout to finish waiting. The actual failure from the
+      // deserialization can fail the compaction properly
+      shared->AddRemoteCompactionResult(job_id, serialized_output);
+    }
+    db_stress_env->SleepForMicroseconds(
+        thread->rand.Next() % FLAGS_remote_compaction_worker_interval * 1000 +
+        1);
+  }
+}
+
 void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz) {
   if (!FLAGS_verbose) {
     return;

db_stress_tool/db_stress_common.h

@@ -422,12 +422,14 @@ DECLARE_string(file_temperature_age_thresholds);
 DECLARE_bool(allow_trivial_copy_when_change_temperature);
 DECLARE_uint32(commit_bypass_memtable_one_in);
 DECLARE_bool(track_and_verify_wals);
-DECLARE_bool(enable_remote_compaction);
+DECLARE_int32(remote_compaction_worker_threads);
+DECLARE_int32(remote_compaction_worker_interval);
 DECLARE_bool(auto_refresh_iterator_with_snapshot);
 DECLARE_uint32(memtable_op_scan_flush_trigger);
 DECLARE_uint32(memtable_avg_op_scan_flush_trigger);
 DECLARE_uint32(ingest_wbwi_one_in);
 DECLARE_bool(universal_reduce_file_locking);
+DECLARE_bool(use_multiscan);
 
 constexpr long KB = 1024;
 constexpr int kRandomValueMaxFactor = 3;
@@ -755,6 +757,8 @@ void PoolSizeChangeThread(void* v);
 
 void DbVerificationThread(void* v);
 
+void RemoteCompactionWorkerThread(void* v);
+
 void CompressedCacheSetCapacityThread(void* v);
 
 void TimestampedSnapshotsThread(void* v);

db_stress_tool/db_stress_compaction_service.h

@@ -5,6 +5,8 @@
 
 #pragma once
 
+#include "db_stress_shared_state.h"
+#include "db_stress_tool/db_stress_common.h"
 #include "rocksdb/options.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -12,28 +14,82 @@ namespace ROCKSDB_NAMESPACE {
 // Service to simulate Remote Compaction in Stress Test
 class DbStressCompactionService : public CompactionService {
  public:
-  explicit DbStressCompactionService() {}
+  explicit DbStressCompactionService(SharedState* shared)
+      : shared_(shared), aborted_(false) {}
 
   static const char* kClassName() { return "DbStressCompactionService"; }
 
   const char* Name() const override { return kClassName(); }
 
+  static constexpr uint64_t kWaitIntervalInMicros = 10 * 1000;  // 10ms
+  static constexpr uint64_t kWaitTimeoutInMicros =
+      30 * 1000 * 1000;  // 30 seconds
+
   CompactionServiceScheduleResponse Schedule(
-      const CompactionServiceJobInfo& /*info*/,
-      const std::string& /*compaction_service_input*/) override {
+      const CompactionServiceJobInfo& info,
+      const std::string& compaction_service_input) override {
+    std::string job_id = info.db_id + "_" + info.db_session_id + "_" +
+                         std::to_string(info.job_id);
+    if (aborted_.load()) {
+      return CompactionServiceScheduleResponse(
+          job_id, CompactionServiceJobStatus::kUseLocal);
+    }
+    shared_->EnqueueRemoteCompaction(job_id, info, compaction_service_input);
     CompactionServiceScheduleResponse response(
-        "Implement Me", CompactionServiceJobStatus::kUseLocal);
+        job_id, CompactionServiceJobStatus::kSuccess);
     return response;
   }
 
-  CompactionServiceJobStatus Wait(const std::string& /*scheduled_job_id*/,
-                                  std::string* /*result*/) override {
-    // TODO - Implement
-    return CompactionServiceJobStatus::kUseLocal;
+  CompactionServiceJobStatus Wait(const std::string& scheduled_job_id,
+                                  std::string* result) override {
+    auto start = Env::Default()->NowMicros();
+    while (Env::Default()->NowMicros() - start < kWaitTimeoutInMicros) {
+      if (aborted_.load()) {
+        return CompactionServiceJobStatus::kUseLocal;
+      }
+      if (shared_->GetRemoteCompactionResult(scheduled_job_id, result).ok()) {
+        if (result && result->empty()) {
+          // Race: Remote worker aborted before client sets aborted_ = true
+          return CompactionServiceJobStatus::kUseLocal;
+        }
+        return CompactionServiceJobStatus::kSuccess;
+      }
+      Env::Default()->SleepForMicroseconds(kWaitIntervalInMicros);
+    }
+    return CompactionServiceJobStatus::kFailure;
   }
 
-  // TODO - Implement
-  void CancelAwaitingJobs() override {}
+  void OnInstallation(const std::string& scheduled_job_id,
+                      CompactionServiceJobStatus /*status*/) override {
+    // Clean up tmp directory
+    std::string serialized;
+    CompactionServiceResult result;
+    if (shared_->GetRemoteCompactionResult(scheduled_job_id, &serialized)
+            .ok()) {
+      if (CompactionServiceResult::Read(serialized, &result).ok()) {
+        std::vector<std::string> filenames;
+        Status s = Env::Default()->GetChildren(result.output_path, &filenames);
+        for (size_t i = 0; s.ok() && i < filenames.size(); ++i) {
+          s = Env::Default()->DeleteFile(result.output_path + "/" +
+                                         filenames[i]);
+          if (!s.ok()) {
+            // TODO - Handle clean up failure?
+            break;
+          }
+        }
+        if (s.ok()) {
+          Env::Default()->DeleteDir(result.output_path).PermitUncheckedError();
+        }
+      }
+      shared_->RemoveRemoteCompactionResult(scheduled_job_id);
+    }
+  }
+
+  void CancelAwaitingJobs() override { aborted_.store(true); }
+
+ private:
+  SharedState* shared_;
+  std::atomic_bool aborted_{false};
 };
 
 }  // namespace ROCKSDB_NAMESPACE

db_stress_tool/db_stress_compression_manager.cc

@@ -0,0 +1,28 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+#include "db_stress_compression_manager.h"
+
+#include "rocksdb/utilities/object_registry.h"
+
+namespace ROCKSDB_NAMESPACE {
+void DbStressCustomCompressionManager::Register() {
+  // We must register any compression managers with a custom
+  // CompatibilityName() so that if it was used in a past invocation but not
+  // the current invocation, we can still read the SST files requiring it.
+  static std::once_flag loaded;
+  std::call_once(loaded, [&]() {
+    TEST_AllowUnsupportedFormatVersion() = true;
+    auto& library = *ObjectLibrary::Default();
+    library.AddFactory<CompressionManager>(
+        DbStressCustomCompressionManager().CompatibilityName(),
+        [](const std::string& /*uri*/,
+           std::unique_ptr<CompressionManager>* guard,
+           std::string* /*errmsg*/) {
+          *guard = std::make_unique<DbStressCustomCompressionManager>();
+          return guard->get();
+        });
+  });
+}
+}  // namespace ROCKSDB_NAMESPACE

db_stress_tool/db_stress_compression_manager.h

@@ -57,6 +57,8 @@ class DbStressCustomCompressionManager : public CompressionManager {
     return decomp;
   }
 
+  static void Register();
+
  protected:
   std::shared_ptr<CompressionManager> default_ =
       GetBuiltinV2CompressionManager();

db_stress_tool/db_stress_driver.cc

@@ -102,6 +102,14 @@ bool RunStressTestImpl(SharedState* shared) {
     shared->IncBgThreads();
   }
 
+  uint32_t remote_compaction_worker_thread_count =
+      FLAGS_remote_compaction_worker_threads;
+  if (remote_compaction_worker_thread_count > 0) {
+    for (uint32_t i = 0; i < remote_compaction_worker_thread_count; i++) {
+      shared->IncBgThreads();
+    }
+  }
+
   std::vector<ThreadState*> threads(n);
   for (uint32_t i = 0; i < n; i++) {
     threads[i] = new ThreadState(i, shared);
@@ -126,6 +134,17 @@ bool RunStressTestImpl(SharedState* shared) {
                                &compressed_cache_set_capacity_thread);
   }
 
+  std::vector<ThreadState*> remote_compaction_worker_threads;
+  if (remote_compaction_worker_thread_count > 0) {
+    remote_compaction_worker_threads.reserve(
+        remote_compaction_worker_thread_count);
+    for (uint32_t i = 0; i < remote_compaction_worker_thread_count; i++) {
+      ThreadState* ts = new ThreadState(i, shared);
+      remote_compaction_worker_threads.push_back(ts);
+      db_stress_env->StartThread(RemoteCompactionWorkerThread, ts);
+    }
+  }
+
   // Each thread goes through the following states:
   // initializing -> wait for others to init -> read/populate/depopulate
   // wait for others to operate -> verify -> done
@@ -218,6 +237,7 @@ bool RunStressTestImpl(SharedState* shared) {
     delete threads[i];
     threads[i] = nullptr;
   }
+
   now = clock->NowMicros();
   if (!FLAGS_skip_verifydb && !FLAGS_test_batches_snapshots &&
       !shared->HasVerificationFailedYet()) {
@@ -232,7 +252,8 @@ bool RunStressTestImpl(SharedState* shared) {
   if (FLAGS_compaction_thread_pool_adjust_interval > 0 ||
       FLAGS_continuous_verification_interval > 0 ||
       FLAGS_compressed_secondary_cache_size > 0 ||
-      FLAGS_compressed_secondary_cache_ratio > 0.0) {
+      FLAGS_compressed_secondary_cache_ratio > 0.0 ||
+      remote_compaction_worker_thread_count > 0) {
     MutexLock l(shared->GetMutex());
     shared->SetShouldStopBgThread();
     while (!shared->BgThreadsFinished()) {
@@ -240,6 +261,15 @@ bool RunStressTestImpl(SharedState* shared) {
     }
   }
 
+  assert(remote_compaction_worker_threads.size() ==
+         remote_compaction_worker_thread_count);
+  if (remote_compaction_worker_thread_count > 0) {
+    for (ThreadState* thread_state : remote_compaction_worker_threads) {
+      delete thread_state;
+    }
+    remote_compaction_worker_threads.clear();
+  }
+
   if (shared->HasVerificationFailedYet()) {
     fprintf(stderr, "Verification failed :(\n");
     return false;

db_stress_tool/db_stress_gflags.cc

@@ -844,8 +844,14 @@ DEFINE_bool(track_and_verify_wals,
             ROCKSDB_NAMESPACE::Options().track_and_verify_wals,
             "See Options::track_and_verify_wals");
 
-DEFINE_bool(enable_remote_compaction, false,
-            "Enable (simulated) Remote Compaction");
+DEFINE_int32(
+    remote_compaction_worker_threads, 2,
+    "Remote Compaction Worker Thread count. If 0, remote compaction is "
+    "disabled");
+
+DEFINE_int32(remote_compaction_worker_interval, 10,
+             "Remote Compaction Worker Thread dequeue tasks every N "
+             "milliseconds. (Default: 10ms)");
 
 DEFINE_uint32(ingest_wbwi_one_in, 0,
               "If set, will call"
@@ -1495,4 +1501,8 @@ DEFINE_bool(
         .compaction_options_universal.reduce_file_locking,
     "Sets "
     "ColumnFamilyOptions().compaciton_options_universal.reduce_file_locking.");
+
+DEFINE_bool(use_multiscan, false,
+            "If set, use the batched MultiScan API for scans.");
+
 #endif  // GFLAGS

db_stress_tool/db_stress_shared_state.h

@@ -276,6 +276,53 @@ class SharedState {
     return expected_state_manager_->GetPersistedSeqno();
   }
 
+  void EnqueueRemoteCompaction(const std::string& job_id,
+                               const CompactionServiceJobInfo& job_info,
+                               const std::string& serialized_input) {
+    MutexLock l(&remote_compaction_queue_mu_);
+    remote_compaction_queue_.emplace(job_id, job_info, serialized_input);
+  }
+
+  bool DequeueRemoteCompaction(std::string* job_id,
+                               CompactionServiceJobInfo* job_info,
+                               std::string* serialized_input) {
+    assert(job_id);
+    assert(job_info);
+    assert(serialized_input);
+    MutexLock l(&remote_compaction_queue_mu_);
+    if (!remote_compaction_queue_.empty()) {
+      const auto [id, info, input] = remote_compaction_queue_.front();
+      *job_id = id;
+      *job_info = info;
+      *serialized_input = input;
+      remote_compaction_queue_.pop();
+      return true;
+    }
+    return false;
+  }
+
+  void AddRemoteCompactionResult(const std::string& job_id,
+                                 const std::string& result) {
+    MutexLock l(&remote_compaction_result_map_mu_);
+    remote_compaction_result_map_.emplace(job_id, result);
+  }
+
+  Status GetRemoteCompactionResult(const std::string& job_id,
+                                   std::string* result) {
+    MutexLock l(&remote_compaction_result_map_mu_);
+    if (remote_compaction_result_map_.find(job_id) !=
+        remote_compaction_result_map_.end()) {
+      *result = remote_compaction_result_map_.at(job_id);
+      return Status::OK();
+    }
+    return Status::NotFound();
+  }
+
+  void RemoveRemoteCompactionResult(const std::string& job_id) {
+    MutexLock l(&remote_compaction_result_map_mu_);
+    remote_compaction_result_map_.erase(job_id);
+  }
+
   // Prepare a Put that will be started but not finish yet
   // This is useful for crash-recovery testing when the process may crash
   // before updating the corresponding expected value
@@ -430,6 +477,16 @@ class SharedState {
   std::atomic<bool> verification_failure_;
   std::atomic<bool> should_stop_test_;
 
+  // Queue for the remote compaction. Tuple of job id, job info and serialized
+  // compaction_service_input
+  port::Mutex remote_compaction_queue_mu_;
+  std::queue<std::tuple<std::string, CompactionServiceJobInfo, std::string>>
+      remote_compaction_queue_;
+  // Result Map for the remote compaciton. Key is the scheduled_job_id and value
+  // is serialized compaction_service_result
+  port::Mutex remote_compaction_result_map_mu_;
+  std::unordered_map<std::string, std::string> remote_compaction_result_map_;
+
   // Keys that should not be overwritten
   const std::unordered_set<int64_t> no_overwrite_ids_;
 

db_stress_tool/db_stress_test_base.cc

@@ -793,6 +793,12 @@ Status StressTest::SetOptions(ThreadState* thread) {
   return db_->SetOptions(cfh, opts);
 }
 
+Options StressTest::GetOptions(int cf_id) {
+  auto cfh = column_families_[cf_id];
+  assert(cfh);
+  return db_->GetOptions(cfh);
+}
+
 void StressTest::ProcessRecoveredPreparedTxns(SharedState* shared) {
   assert(txn_db_);
   std::vector<Transaction*> recovered_prepared_trans;
@@ -1458,9 +1464,23 @@ void StressTest::OperateDb(ThreadState* thread) {
       } else if (prob_op < iterate_bound) {
         assert(delrange_bound <= prob_op);
         // OPERATION iterate
-        if (!FLAGS_skip_verifydb &&
-            thread->rand.OneInOpt(
-                FLAGS_verify_iterator_with_expected_state_one_in)) {
+        if (FLAGS_use_multiscan) {
+          int num_seeks = static_cast<int>(
+              std::min(static_cast<uint64_t>(thread->rand.Uniform(64)),
+                       static_cast<uint64_t>(FLAGS_ops_per_thread - i - 1)));
+          // Generate 2x num_seeks random keys, as each scan has a start key
+          // and an upper bound
+          rand_keys = GenerateNKeys(thread, num_seeks * 2, i);
+          i += num_seeks - 1;
+          ThreadStatusUtil::SetEnableTracking(FLAGS_enable_thread_tracking);
+          ThreadStatusUtil::SetThreadOperation(
+              ThreadStatus::OperationType::OP_DBITERATOR);
+          Status s;
+          s = TestMultiScan(thread, read_opts, rand_column_families, rand_keys);
+          ThreadStatusUtil::ResetThreadStatus();
+        } else if (!FLAGS_skip_verifydb &&
+                   thread->rand.OneInOpt(
+                       FLAGS_verify_iterator_with_expected_state_one_in)) {
           ThreadStatusUtil::SetEnableTracking(FLAGS_enable_thread_tracking);
           ThreadStatusUtil::SetThreadOperation(
               ThreadStatus::OperationType::OP_DBITERATOR);
@@ -1638,6 +1658,172 @@ Status StressTest::TestIterateAttributeGroups(
       verify_func);
 }
 
+Status StressTest::TestMultiScan(ThreadState* thread,
+                                 const ReadOptions& read_opts,
+                                 const std::vector<int>& rand_column_families,
+                                 const std::vector<int64_t>& rand_keys) {
+  size_t num_scans = rand_keys.size() / 2;
+  assert(!rand_column_families.empty());
+  assert(!rand_keys.empty());
+
+  ManagedSnapshot snapshot_guard(db_);
+
+  ReadOptions ro = read_opts;
+  ro.snapshot = snapshot_guard.snapshot();
+
+  std::string read_ts_str;
+  Slice read_ts_slice;
+  MaybeUseOlderTimestampForRangeScan(thread, read_ts_str, read_ts_slice, ro);
+
+  std::vector<std::string> start_key_strs;
+  std::vector<std::string> end_key_strs;
+  MultiScanArgs scan_opts;
+  start_key_strs.reserve(num_scans);
+  end_key_strs.reserve(num_scans);
+
+  // Will be initialized before Seek() below.
+  Slice ub;
+  ro.iterate_upper_bound = &ub;
+  for (size_t i = 0; i < num_scans * 2; i += 2) {
+    assert(rand_keys[i] <= rand_keys[i + 1]);
+    start_key_strs.emplace_back(Key(rand_keys[i]));
+    end_key_strs.emplace_back(Key(rand_keys[i + 1]));
+    scan_opts.insert(Slice(start_key_strs.back()), Slice(end_key_strs.back()));
+  }
+
+  std::string op_logs;
+  ro.pin_data = thread->rand.OneIn(2);
+  ro.background_purge_on_iterator_cleanup = thread->rand.OneIn(2);
+
+  assert(options_.prefix_extractor.get() == nullptr);
+
+  std::unique_ptr<Iterator> iter;
+  iter.reset(db_->NewIterator(ro, column_families_[rand_column_families[0]]));
+  iter->Prepare(scan_opts);
+
+  constexpr size_t kOpLogsLimit = 50000;
+
+  auto verify_func = [](Iterator* iterator) {
+    if (!VerifyWideColumns(iterator->value(), iterator->columns())) {
+      fprintf(stderr,
+              "Value and columns inconsistent for iterator: value: %s, "
+              "columns: %s\n",
+              iterator->value().ToString(/* hex */ true).c_str(),
+              WideColumnsToHex(iterator->columns()).c_str());
+      return false;
+    }
+    return true;
+  };
+
+  for (const ScanOptions& scan_opt : scan_opts.GetScanRanges()) {
+    if (op_logs.size() > kOpLogsLimit) {
+      // Shouldn't take too much memory for the history log. Clear it.
+      op_logs = "(cleared...)\n";
+    }
+
+    // Set up an iterator, perform the same operations without bounds and with
+    // total order seek, and compare the results. This is to identify bugs
+    // related to bounds, prefix extractor, or reseeking. Sometimes we are
+    // comparing iterators with the same set-up, and it doesn't hurt to check
+    // them to be equal.
+    //
+    // This `ReadOptions` is for validation purposes. Ignore
+    // `FLAGS_rate_limit_user_ops` to avoid slowing any validation.
+    ReadOptions cmp_ro;
+    cmp_ro.timestamp = ro.timestamp;
+    cmp_ro.iter_start_ts = ro.iter_start_ts;
+    cmp_ro.snapshot = snapshot_guard.snapshot();
+    cmp_ro.auto_refresh_iterator_with_snapshot =
+        ro.auto_refresh_iterator_with_snapshot;
+    cmp_ro.total_order_seek = true;
+
+    ColumnFamilyHandle* const cmp_cfh =
+        GetControlCfh(thread, rand_column_families[0]);
+    assert(cmp_cfh);
+
+    std::unique_ptr<Iterator> cmp_iter(db_->NewIterator(cmp_ro, cmp_cfh));
+
+    bool diverged = false;
+
+    assert(scan_opt.range.start);
+    assert(scan_opt.range.limit);
+    Slice key = scan_opt.range.start.value();
+    ub = scan_opt.range.limit.value();
+
+    LastIterateOp last_op;
+    iter->Seek(key);
+    cmp_iter->Seek(key);
+    last_op = kLastOpSeek;
+    op_logs += "S " + key.ToString(true) + " ";
+
+    if (iter->Valid() && ro.allow_unprepared_value) {
+      op_logs += "*";
+
+      if (!iter->PrepareValue()) {
+        assert(!iter->Valid());
+        assert(!iter->status().ok());
+      }
+    }
+
+    if (!iter->status().ok() && IsErrorInjectedAndRetryable(iter->status())) {
+      return iter->status();
+    } else if (!cmp_iter->status().ok() &&
+               IsErrorInjectedAndRetryable(cmp_iter->status())) {
+      return cmp_iter->status();
+    }
+
+    VerifyIterator(thread, cmp_cfh, ro, iter.get(), cmp_iter.get(), last_op,
+                   key, op_logs, verify_func, &diverged);
+
+    while (iter->Valid()) {
+      iter->Next();
+      if (!diverged) {
+        assert(cmp_iter->Valid());
+        cmp_iter->Next();
+      }
+      op_logs += "N";
+
+      if (iter->Valid() && ro.allow_unprepared_value) {
+        op_logs += "*";
+
+        if (!iter->PrepareValue()) {
+          assert(!iter->Valid());
+          assert(!iter->status().ok());
+        }
+      }
+
+      if (!iter->status().ok() && IsErrorInjectedAndRetryable(iter->status())) {
+        return iter->status();
+      } else if (!cmp_iter->status().ok() &&
+                 IsErrorInjectedAndRetryable(cmp_iter->status())) {
+        return cmp_iter->status();
+      }
+
+      VerifyIterator(thread, cmp_cfh, ro, iter.get(), cmp_iter.get(), last_op,
+                     key, op_logs, verify_func, &diverged);
+
+      if (diverged) {
+        const std::vector<ScanOptions>& scanoptions = scan_opts.GetScanRanges();
+        for (const auto& t : scanoptions) {
+          fprintf(stdout, "Multiscan options: %s to %s \n",
+                  t.range.start.value().ToString(true).c_str(),
+                  t.range.limit.value().ToString(true).c_str());
+        }
+        break;
+      }
+    }
+
+    thread->stats.AddIterations(1);
+
+    op_logs += "; ";
+    if (diverged) {
+      break;
+    }
+  }
+
+  return Status::OK();
+}
+
 template <typename IterType, typename NewIterFunc, typename VerifyFunc>
 Status StressTest::TestIterateImpl(ThreadState* thread,
                                    const ReadOptions& read_opts,
@@ -3417,24 +3603,8 @@ void StressTest::Open(SharedState* shared, bool reopen) {
     InitializeOptionsFromFlags(cache_, filter_policy_, options_);
   }
   InitializeOptionsGeneral(cache_, filter_policy_, sqfc_factory_, options_);
-  {
-    // We must register any compression managers with a custom
-    // CompatibilityName() so that if it was used in a past invocation but not
-    // the current invocation, we can still read the SST files requiring it.
-    static std::once_flag loaded;
-    std::call_once(loaded, [&]() {
-      TEST_AllowUnsupportedFormatVersion() = true;
-      auto& library = *ObjectLibrary::Default();
-      library.AddFactory<CompressionManager>(
-          DbStressCustomCompressionManager().CompatibilityName(),
-          [](const std::string& /*uri*/,
-             std::unique_ptr<CompressionManager>* guard,
-             std::string* /*errmsg*/) {
-            *guard = std::make_unique<DbStressCustomCompressionManager>();
-            return guard->get();
-          });
-    });
-  }
+  DbStressCustomCompressionManager::Register();
+
   if (!strcasecmp(FLAGS_compression_manager.c_str(), "custom")) {
     options_.compression_manager =
         std::make_shared<DbStressCustomCompressionManager>();
@@ -3466,6 +3636,27 @@ void StressTest::Open(SharedState* shared, bool reopen) {
             "memtablerep != prefix_hash\n");
   }
 
+  // Remote Compaction
+  if (FLAGS_remote_compaction_worker_threads > 0) {
+    // TODO(jaykorean) Remove this after fix - remote worker shouldn't recover
+    // from WAL
+    if (!FLAGS_disable_wal) {
+      fprintf(stderr,
+              "WAL is not compatible with Remote Compaction in Stress Test\n");
+      exit(1);
+    }
+    if ((options_.enable_blob_files ||
+         options_.enable_blob_garbage_collection ||
+         FLAGS_allow_setting_blob_options_dynamically)) {
+      fprintf(stderr,
+              "Integrated BlobDB is currently incompatible with Remote "
+              "Compaction\n");
+      exit(1);
+    }
+    options_.compaction_service =
+        std::make_shared<DbStressCompactionService>(shared);
+  }
+
   if ((options_.enable_blob_files || options_.enable_blob_garbage_collection ||
        FLAGS_allow_setting_blob_options_dynamically) &&
       FLAGS_best_efforts_recovery) {
@@ -4374,11 +4565,6 @@ void InitializeOptionsFromFlags(
   options.inplace_update_support = FLAGS_inplace_update_support;
   options.uncache_aggressiveness = FLAGS_uncache_aggressiveness;
 
-  // Remote Compaction
-  if (FLAGS_enable_remote_compaction) {
-    options.compaction_service = std::make_shared<DbStressCompactionService>();
-  }
-
   options.memtable_op_scan_flush_trigger = FLAGS_memtable_op_scan_flush_trigger;
   options.compaction_options_universal.reduce_file_locking =
       FLAGS_universal_reduce_file_locking;

db_stress_tool/db_stress_test_base.h

@@ -53,6 +53,7 @@ class StressTest {
     Status s = db_->EnableAutoCompaction(column_families_);
     return s;
   }
+  Options GetOptions(int cf_id);
   void CleanUp();
 
  protected:
@@ -274,6 +275,10 @@ class StressTest {
     return Status::NotSupported();
   }
 
+  Status TestMultiScan(ThreadState* thread, const ReadOptions& read_opts,
+                       const std::vector<int>& rand_column_families,
+                       const std::vector<int64_t>& rand_keys);
+
   // Enum used by VerifyIterator() to identify the mode to validate.
   enum LastIterateOp {
     kLastOpSeek,

env/fs_posix.cc

@@ -1107,8 +1107,10 @@ class PosixFileSystem : public FileSystem {
         struct io_uring_cqe* cqe = nullptr;
         ssize_t ret = io_uring_wait_cqe(iu, &cqe);
         if (ret) {
-          // abort as it shouldn't be in indeterminate state and there is no
-          // good way currently to handle this error.
+          fprintf(stderr, "Poll: io_uring_wait_cqe failed: %ld", (long)ret);
+          if (ret == -EINTR || ret == -EAGAIN) {
+            continue;  // Retry
+          }
           abort();
         }
 
@@ -1210,8 +1212,10 @@ class PosixFileSystem : public FileSystem {
         struct io_uring_cqe* cqe = nullptr;
         ssize_t ret = io_uring_wait_cqe(iu, &cqe);
         if (ret) {
-          // abort as it shouldn't be in indeterminate state and there is no
-          // good way currently to handle this error.
+          fprintf(stderr, "AbortIO: io_uring_wait_cqe failed: %ld", (long)ret);
+          if (ret == -EINTR || ret == -EAGAIN) {
+            continue;  // Retry
+          }
           abort();
         }
         assert(cqe != nullptr);

include/rocksdb/advanced_compression.h

@@ -19,6 +19,7 @@ namespace ROCKSDB_NAMESPACE {
 
 // TODO: alias/adapt for compression
 struct FilterBuildingContext;
+class Decompressor;
 
 // A Compressor represents a very specific but potentially adapting strategy for
 // compressing blocks, including the relevant algorithm(s), options, dictionary,
@@ -156,9 +157,11 @@ class Compressor {
     return {};
   }
 
-  // Compress `uncompressed_data` to `compressed_output`, which should be
-  // passed in empty. Note that the compressed output will be decompressed
-  // by the sequence Decompressor::ExtractUncompressedSize() followed by
+  // Compress `uncompressed_data` to buffer `compressed_output` of size
+  // `*compressed_output_size`, storing the final compressed size in
+  // `*compressed_output_size` and compression type in `*out_compression_type`.
+  // Note that the compressed output will be decompressed by the sequence
+  // Decompressor::ExtractUncompressedSize() followed by
   // Decompressor::DecompressBlock(), which must also be provided the same
   // CompressionType saved in `out_compression_type`. (In many configurations,
   // `compressed_output` will have a prefix storing the uncompressed_data size
@@ -170,28 +173,34 @@ class Compressor {
   // If return status is not OK, then some fatal condition has arisen. On OK
   // status, setting `*out_compression_type = kNoCompression` means compression
   // is declined and the caller should use the original uncompressed_data and
-  // ignore any result in `compressed_output`. Otherwise, compression has
-  // happened with results in `compressed_output` and `out_compression_type`,
-  // which are allowed to vary from call to call.
+  // ignore any result in `compressed_output`. In this case, setting
+  // *compressed_output_size to 0 suggests that compression was quickly
+  // "bypassed" and *compressed_output_size > 0 suggests that compression was
+  // attempted but rejected (e.g. insufficient compression ratio).
+  //
+  // On OK status and `*out_compression_type != kNoCompression`, compression has
+  // happened with results in `compressed_output`, `compressed_output_size`, and
+  // `out_compression_type`. The output compression type is allowed to vary from
+  // call to call but does not for compressors from BuiltinV2CompressionManager.
   //
   // The working area is optional and used to optimize repeated compression by
   // a single thread. ManagedWorkingArea is provided rather than just
   // WorkingArea so that it can be used only if the `owner` matches expectation.
   // This could be useful for a Compressor wrapping more than one alternative
   // underlying Compressor.
-  //
-  // TODO: instead of string, consider a buffer only large enough for max
-  // tolerable compressed size. Does that work for all existing algorithms?
-  // * Looks like Snappy doesn't support that. :(
-  //   * Except perhaps using the Sink interface
-  // * But looks like everything else should. :)
-  // Could save CPU by eliminating extra zero-ing and giving up quicker when
-  // ratio is insufficient.
-  virtual Status CompressBlock(Slice uncompressed_data,
-                               std::string* compressed_output,
+  virtual Status CompressBlock(Slice uncompressed_data, char* compressed_output,
+                               size_t* compressed_output_size,
                                CompressionType* out_compression_type,
                                ManagedWorkingArea* working_area) = 0;
 
+  // OPTIONAL: Return a decompressor that is optimized for output from this
+  // compressor.
+  virtual std::shared_ptr<Decompressor> GetOptimizedDecompressor() const {
+    // Default implementation: no optimization. Get a Decompressor from the
+    // CompressionManager.
+    return nullptr;
+  }
+
   // TODO: something to populate table properties based on settings, after all
   // or as WorkingAreas released. Maybe also update stats, or that could be in
   // thread-specific WorkingArea.
@@ -441,14 +450,6 @@ class CompressionManager
     // Safe default implementation
     return GetDecompressor();
   }
-
-  // Get a decompressor that is allowed to have support only for the
-  // CompressionTypes used by the given Compressor.
-  virtual std::shared_ptr<Decompressor> GetDecompressorForCompressor(
-      const Compressor& compressor) {
-    // Reasonable default implementation
-    return GetDecompressorOptimizeFor(compressor.GetPreferredCompressionType());
-  }
 };
 
 // ************************* Utility wrappers etc. *********************** //
@@ -485,11 +486,17 @@ class CompressorWrapper : public Compressor {
   // ManagedWorkingArea takes care of calling it on the Compressor that created
   // the WorkingArea.
 
-  Status CompressBlock(Slice uncompressed_data, std::string* compressed_output,
+  Status CompressBlock(Slice uncompressed_data, char* compressed_output,
+                       size_t* compressed_output_size,
                        CompressionType* out_compression_type,
                        ManagedWorkingArea* working_area) override {
     return wrapped_->CompressBlock(uncompressed_data, compressed_output,
-                                   out_compression_type, working_area);
+                                   compressed_output_size, out_compression_type,
+                                   working_area);
+  }
+
+  std::shared_ptr<Decompressor> GetOptimizedDecompressor() const override {
+    return wrapped_->GetOptimizedDecompressor();
   }
 
  protected:
@@ -592,11 +599,6 @@ class CompressionManagerWrapper : public CompressionManager {
     return wrapped_->GetDecompressorForTypes(types_begin, types_end);
   }
 
-  std::shared_ptr<Decompressor> GetDecompressorForCompressor(
-      const Compressor& compressor) override {
-    return wrapped_->GetDecompressorForCompressor(compressor);
-  }
-
  protected:
   std::shared_ptr<CompressionManager> wrapped_;
 };

include/rocksdb/advanced_options.h

@@ -526,7 +526,7 @@ struct AdvancedColumnFamilyOptions {
   // By doing it, we give max_bytes_for_level_multiplier a priority against
   // max_bytes_for_level_base, for a more predictable LSM tree shape. It is
   // useful to limit worse case space amplification.
-  // If `allow_ingest_behind=true` or `preclude_last_level_data_seconds > 0`,
+  // If `cf_allow_ingest_behind=true` or `preclude_last_level_data_seconds > 0`,
   // then the last level is reserved, and we will start filling LSM from the
   // second last level.
   //
@@ -1146,6 +1146,33 @@ struct AdvancedColumnFamilyOptions {
   // Dynamically changeable through the SetOptions() API.
   uint32_t memtable_avg_op_scan_flush_trigger = 0;
 
+  // If either DBOptions::allow_ingest_behind or this option is set to true,
+  // this column family will prepare for ingesting files to the last level
+  // (IngestExternalFiles() with ingest_behind=true). Users should set only
+  // this option since DBOptions::allow_ingest_behind is deprecated.
+  //
+  // Specifically, preparing a column family for ingesting files to the last
+  // level has the following effects:
+  // 1) Disables some internal optimizations around SST file compression.
+  // 2) Reserves the last level for ingested files only.
+  // 3) Compaction will not include any file from the last level.
+  // 4) Compaction will preserve necessary tombstones that can apply on
+  // top of ingested files.
+  //
+  // Note that only Universal Compaction supports cf_allow_ingest_behind.
+  // `num_levels` should be >= 3 if this option is turned on.
+  //
+  // Note that this option needs to be set to true before any write to the CF.
+  // It's recommended to set the option to true since CF creation. Otherwise,
+  // ingestion with ingest_behind = true might fail. Once file ingestions are
+  // done, the option should be flipped to false. Flipping this option to false
+  // allows the CF to disable the behavior changes detailed above and resume
+  // more efficient operation.
+  //
+  // Default: false
+  // Immutable.
+  bool cf_allow_ingest_behind = false;
+
   // Create ColumnFamilyOptions with default values for all fields
   AdvancedColumnFamilyOptions();
   // Create ColumnFamilyOptions from Options

include/rocksdb/c.h

@@ -1154,6 +1154,8 @@ rocksdb_block_based_options_set_unpartitioned_pinning_tier(
     rocksdb_block_based_table_options_t*, int);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_write_buffer_manager(
     rocksdb_options_t* opt, rocksdb_write_buffer_manager_t* wbm);
+extern ROCKSDB_LIBRARY_API void rocksdb_options_set_sst_file_manager(
+    rocksdb_options_t* opt, rocksdb_sst_file_manager_t* sfm);
 
 /* Flush job info */
 
@@ -1173,6 +1175,8 @@ extern ROCKSDB_LIBRARY_API void rocksdb_reset_status(
     rocksdb_status_ptr_t* status_ptr);
 extern ROCKSDB_LIBRARY_API uint32_t
 rocksdb_flushjobinfo_flush_reason(const rocksdb_flushjobinfo_t* info);
+extern ROCKSDB_LIBRARY_API void rocksdb_status_ptr_get_error(
+    rocksdb_status_ptr_t* status, char** errptr);
 
 /* Compaction job info */
 extern ROCKSDB_LIBRARY_API void rocksdb_compactionjobinfo_status(
@@ -1923,6 +1927,10 @@ extern ROCKSDB_LIBRARY_API void
 rocksdb_options_add_compact_on_deletion_collector_factory_del_ratio(
     rocksdb_options_t*, size_t window_size, size_t num_dels_trigger,
     double deletion_ratio);
+extern ROCKSDB_LIBRARY_API void
+rocksdb_options_add_compact_on_deletion_collector_factory_min_file_size(
+    rocksdb_options_t*, size_t window_size, size_t num_dels_trigger,
+    double deletion_ratio, uint64_t min_file_size);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_manual_wal_flush(
     rocksdb_options_t* opt, unsigned char);
 extern ROCKSDB_LIBRARY_API unsigned char rocksdb_options_get_manual_wal_flush(

include/rocksdb/data_structure.h

@@ -10,8 +10,13 @@
 #include <array>
 #include <cstddef>
 #include <cstdint>
+#include <functional>
+#include <set>
+#include <variant>
 
+#include "rocksdb/comparator.h"
 #include "rocksdb/rocksdb_namespace.h"
+#include "rocksdb/slice.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -281,4 +286,486 @@ class ManagedPtr {
   Owner* owner_ = nullptr;
 };
 
+template <typename T, typename comp>
+class Interval;
+
+// The Interval Class is a generic class for holding a range, for example [2,
+// 4]. It can be used within the IntervalSet class, which is able to keep an
+// ordered, non-intersecting set of intervals within it.  Intervals can have
+// open-ended end points, (i.e., to infinity) for example [2,).
+template <typename T, typename comp = std::less<T>>
+class Interval {
+ public:
+  enum class End { INF };
+  struct CompareVariant {
+    comp comparator;
+    bool operator()(const std::variant<T, End>& a,
+                    const std::variant<T, End>& b) const {
+      if (std::holds_alternative<T>(a) && std::holds_alternative<T>(b)) {
+        return comparator(std::get<T>(a), std::get<T>(b));
+      }
+      if (std::holds_alternative<End>(a) && std::holds_alternative<End>(b)) {
+        return false;
+      }
+      if (std::holds_alternative<T>(a) && std::holds_alternative<End>(b)) {
+        return false;
+      }
+      return true;  // std::holds_alternative<End>(a) &&
+                    // std::holds_alternative<T>(b)
+    }
+  };
+
+  /* implicit */ Interval(const T& start, const T& end)
+      : start_(start), end_(end) {}
+  /* implicit */ Interval(const T& start) : start_(start), end_(End::INF) {}
+
+  // Add constructor that takes a pair
+  /* implicit */ Interval(const std::pair<T, T>& p)
+      : start_(p.first), end_(p.second) {}
+
+  T& start() { return start_; }
+
+  const T& start() const { return start_; }
+
+  bool has_end() const { return std::holds_alternative<T>(end_); }
+
+  T& end() { return std::get<T>(end_); }
+
+  const T& end() const { return std::get<T>(end_); }
+
+  // Support comparison with std::pair
+  bool operator==(const std::pair<T, T>& p) const {
+    return start_ == p.first && has_end() && end() == p.second;
+  }
+
+  // Support comparison with another Interval
+  bool operator==(const Interval& other) const {
+    if (start_ != other.start_) {
+      return false;
+    }
+
+    // Both have infinite end
+    if (!has_end() && !other.has_end()) {
+      return true;
+    }
+
+    // One has infinite end, the other doesn't
+    if (has_end() != other.has_end()) {
+      return false;
+    }
+
+    // Both have finite end
+    return end() == other.end();
+  }
+
+  // Support comparison with another Interval
+  bool operator<(const Interval& other) const {
+    return comparator(start_, other.start_);
+  }
+
+  bool Compare(const Interval& other) const {
+    return comparator(start_, other.start_);
+  }
+
+ private:
+  T start_;
+  std::variant<T, End> end_;
+  comp comparator;
+};
+
+// Specialized version of Interval for Slice
+template <>
+class Interval<Slice, Comparator> {
+ public:
+  enum class End { INF };
+
+  // Constructors that take a Comparator
+  /* implicit */ Interval(const Comparator* c, const Slice& start,
+                          const Slice& end)
+      : start_(start), end_(end), comparator_(c) {}
+
+  /* implicit */ Interval(const Comparator* c, const Slice& start)
+      : start_(start), end_(End::INF), comparator_(c) {}
+
+  // Constructor that takes a pair
+  /* implicit */ Interval(const Comparator* c, const std::pair<Slice, Slice>& p)
+      : start_(p.first), end_(p.second), comparator_(c) {}
+
+  Slice& start() { return start_; }
+
+  const Slice& start() const { return start_; }
+
+  bool has_end() const { return std::holds_alternative<Slice>(end_); }
+
+  Slice& end() { return std::get<Slice>(end_); }
+
+  const Slice& end() const { return std::get<Slice>(end_); }
+
+  // Support comparison with std::pair
+  bool operator==(const std::pair<Slice, Slice>& p) const {
+    return start_ == p.first && has_end() && end() == p.second;
+  }
+
+  // Support comparison with another Interval
+  bool operator==(const Interval& other) const {
+    if (comparator_->Compare(start_, other.start_) != 0) {
+      return false;
+    }
+
+    // Both have infinite end
+    if (!has_end() && !other.has_end()) {
+      return true;
+    }
+
+    // One has infinite end, the other doesn't
+    if (has_end() != other.has_end()) {
+      return false;
+    }
+
+    // Both have finite end
+    return comparator_->Compare(end(), other.end()) == 0;
+  }
+
+  // Support comparison with another Interval
+  bool operator<(const Interval& other) const {
+    return comparator_->Compare(start_, other.start_) < 0;
+  }
+
+  bool Compare(const Interval& other) const {
+    return comparator_->Compare(start_, other.start_) < 0;
+  }
+
+  const Comparator* GetComparator() const { return comparator_; }
+
+ private:
+  Slice start_;
+  std::variant<Slice, End> end_;
+  const Comparator* comparator_;
+
+  std::unordered_map<std::string, std::string> property_bag;
+};
+
+template <typename T, typename Compare = std::less<T>>
+struct CompareInterval {
+  bool operator()(const Interval<T, Compare>& a,
+                  const Interval<T, Compare>& b) const {
+    return a.Compare(b);
+  }
+};
+
+// IntervalSet will be used to represent a set of intervals (including unbounded
+// ones). The intervals are unique and disjoint. Intervals that are inserted
+// will merge with any range they intersect with.
+template <typename T, typename Compare = typename Interval<T>::CompareVariant>
+class IntervalSet {
+ public:
+  IntervalSet(Compare c = Compare()) : comp_(c) {}
+
+  void insert(Interval<T>&& i) { insertImpl(i); }
+
+  void insert(const T& start, const T& end) {
+    insertImpl(Interval<T>(start, end));
+  }
+
+  void insert(const T& start) { insertImpl(Interval<T>(start)); }
+
+  bool empty() const { return intervals_.empty(); }
+  void clear() { intervals_.clear(); }
+
+  auto begin() { return intervals_.begin(); }
+  auto end() { return intervals_.end(); }
+
+  auto cbegin() const { return intervals_.cbegin(); }
+  auto cend() const { return intervals_.cend(); }
+
+  size_t size() const { return intervals_.size(); }
+
+ private:
+  void insertImpl(const Interval<T>& i) {
+    // Skip empty intervals
+    if (i.has_end() && !comp_(i.start(), i.end()) &&
+        !comp_(i.end(), i.start())) {
+      return;
+    }
+
+    // First, check if there's any infinite interval that would contain this one
+    for (auto it = intervals_.begin(); it != intervals_.end(); ++it) {
+      if (!it->has_end() && !comp_(i.start(), it->start())) {
+        // This interval starts at or after an infinite interval
+        return;
+      }
+    }
+
+    // Find the position where the interval should be inserted
+    auto it = intervals_.begin();
+    while (it != intervals_.end() && comp_(it->start(), i.start())) {
+      ++it;
+    }
+
+    // Check if we need to consider the previous interval
+    if (it != intervals_.begin()) {
+      --it;
+      if (it->has_end() && comp_(it->end(), i.start())) {
+        ++it;
+      }
+    }
+
+    T new_start = i.start();
+    T new_end;
+    bool inf_end = false;
+    if (i.has_end()) {
+      new_end = i.end();
+    } else {
+      // For infinite end intervals, we need to merge all intervals that start
+      // after new_start
+      std::vector<decltype(it)> to_erase;
+      while (it != intervals_.end()) {
+        new_start = comp_(it->start(), new_start) ? it->start() : new_start;
+        to_erase.push_back(it++);
+      }
+
+      for (auto& eit : to_erase) {
+        intervals_.erase(eit);
+      }
+
+      // Insert the new interval with infinite end
+      intervals_.insert(Interval<T>(new_start));
+      return;
+    }
+
+    // For finite end intervals, proceed as before
+    std::vector<decltype(it)> to_erase;
+    while (it != intervals_.end() && !comp_(new_end, it->start())) {
+      if (it->has_end() && comp_(it->end(), new_start)) {
+        ++it;
+        continue;
+      }
+      new_start = comp_(it->start(), new_start) ? it->start() : new_start;
+      if (it->has_end()) {
+        new_end = comp_(new_end, it->end()) ? it->end() : new_end;
+      } else {
+        // If we encounter an interval with infinite end, our new interval also
+        // becomes infinite
+        inf_end = true;
+        break;
+      }
+      to_erase.push_back(it++);
+    }
+
+    // Check for any infinite intervals that start after this one
+    auto check_it = it;
+    while (check_it != intervals_.end()) {
+      if (!check_it->has_end()) {
+        inf_end = true;
+        to_erase.push_back(check_it);
+      }
+      ++check_it;
+    }
+
+    for (auto& eit : to_erase) {
+      intervals_.erase(eit);
+    }
+
+    if (inf_end) {
+      intervals_.insert(Interval<T>(new_start));
+    } else {
+      intervals_.insert(Interval<T>(new_start, new_end));
+    }
+  }
+
+  std::set<Interval<T>, CompareInterval<T>> intervals_;
+  Compare comp_;
+};
+
+// Specialization of IntervalSet for Slices.
+// Slice based intervals can have properties attached to them. This is used to
+// push down properties in the MultiScan API.  We accept two modes with
+// IntervalSet, fail_on_intersect, which imposes a restriction that inserted
+// ranges will be disjoint, this is needed when using properties. Insert will
+// fail if a range is found to not be disjoint. When fail_on_instersect is
+// false, the ranges will be merged.
+template <>
+class IntervalSet<Slice, Comparator> {
+ public:
+  explicit IntervalSet(const Comparator* c, bool fail_on_intersect = false)
+      : comp_(c), prop_(fail_on_intersect) {}
+
+  // Insert returns true if the interval was inserted. False indicates that the
+  // interval was not inserted, this could be do to an empty range OR that the
+  // IntervalSet is in with_properties mode and the interval overlaps with an
+  // existing interval.
+  bool insert(const Slice& start, const Slice& end) {
+    return insertImpl(Interval<Slice, Comparator>(comp_, start, end));
+  }
+
+  // Insert returns true if the interval was inserted. False indicates that the
+  // interval was not inserted, this could be do to an empty range OR that the
+  // IntervalSet is in with_properties mode and the interval overlaps with an
+  // existing interval.
+  bool insert(const Slice& start) {
+    // Create an interval with infinite end
+    Interval<Slice, Comparator> interval(comp_, start);
+    return insertImpl(interval);
+  }
+
+  bool insert(Interval<Slice, Comparator>&& i) { return insertImpl(i); }
+
+  bool empty() const { return intervals_.empty(); }
+  void clear() { intervals_.clear(); }
+
+  auto begin() { return intervals_.begin(); }
+  auto end() { return intervals_.end(); }
+
+  auto cbegin() const { return intervals_.cbegin(); }
+  auto cend() const { return intervals_.cend(); }
+
+  size_t size() const { return intervals_.size(); }
+
+ private:
+  // Custom comparator for finding intervals in the vector
+  struct IntervalComparator {
+    explicit IntervalComparator(const Comparator* comp) : comp_(comp) {}
+
+    bool operator()(const Interval<Slice, Comparator>& a,
+                    const Interval<Slice, Comparator>& b) const {
+      return comp_->Compare(a.start(), b.start()) < 0;
+    }
+
+    const Comparator* comp_;
+  };
+
+  typename std::vector<Interval<Slice, Comparator>>::iterator findPosition(
+      const Interval<Slice, Comparator>& interval) {
+    // Find the position where the new interval should be inserted
+    for (auto it = intervals_.begin(); it != intervals_.end(); ++it) {
+      if (comp_->Compare(it->start(), interval.start()) >= 0) {
+        return it;
+      }
+    }
+    return intervals_.end();
+  }
+
+  bool insertImpl(const Interval<Slice, Comparator>& i) {
+    // Skip empty intervals
+    if (i.has_end() && comp_->Compare(i.start(), i.end()) >= 0) {
+      return false;
+    }
+
+    // Find the position where this interval would be inserted
+    // This also checks if the interval is completely contained within an
+    // existing one
+    auto it = findPosition(i);
+
+    // Check if we need to merge with previous interval
+    if (it != intervals_.begin()) {
+      auto prev = it - 1;
+      if (prev->has_end() && comp_->Compare(prev->end(), i.start()) < 0) {
+        // No overlap with previous interval
+      } else {
+        // There is overlap, adjust iterator to include previous interval
+        if (prop_) {
+          return false;
+        }
+        it = prev;
+      }
+    }
+
+    Slice new_start = i.start();
+    Slice new_end;
+    bool inf_end = false;
+
+    if (i.has_end()) {
+      new_end = i.end();
+    } else {
+      // For infinite end intervals, we need to merge all intervals that start
+      // after new_start
+      auto erase_start = it;
+      while (it != intervals_.end()) {
+        if (comp_->Compare(it->start(), new_start) < 0) {
+          if (prop_) {
+            return false;
+          }
+          new_start = it->start();
+        }
+        ++it;
+      }
+
+      // Erase all intervals from erase_start to end
+      if (erase_start != intervals_.end()) {
+        if (prop_) {
+          return false;
+        }
+        intervals_.erase(erase_start, intervals_.end());
+      }
+
+      // Insert the new interval with infinite end
+      Interval<Slice, Comparator> new_interval(comp_, new_start);
+      auto pos = findPosition(new_interval);
+      intervals_.insert(pos, new_interval);
+      return true;
+    }
+
+    // For finite end intervals, find all overlapping intervals
+    auto erase_start = it;
+    auto erase_end = it;
+
+    while (it != intervals_.end() &&
+           comp_->Compare(new_end, it->start()) >= 0) {
+      if (it->has_end() && comp_->Compare(it->end(), new_start) < 0) {
+        // No overlap
+        ++it;
+        erase_end = it;
+        continue;
+      }
+
+      if (comp_->Compare(it->start(), new_start) < 0) {
+        new_start = it->start();
+      }
+
+      if (it->has_end()) {
+        if (comp_->Compare(new_end, it->end()) < 0) {
+          new_end = it->end();
+        }
+      } else {
+        // If we encounter an interval with infinite end, our new interval also
+        // becomes infinite
+        inf_end = true;
+        erase_end = intervals_.end();
+        break;
+      }
+
+      ++it;
+      erase_end = it;
+    }
+
+    // Check for any infinite intervals that start after this one
+    while (it != intervals_.end()) {
+      if (!it->has_end()) {
+        inf_end = true;
+        erase_end = intervals_.end();
+        break;
+      }
+      ++it;
+    }
+
+    // Erase all merged intervals
+    if (erase_start != erase_end) {
+      intervals_.erase(erase_start, erase_end);
+    }
+
+    // Insert the new merged interval
+    Interval<Slice, Comparator> new_interval =
+        inf_end ? Interval<Slice, Comparator>(comp_, new_start)
+                : Interval<Slice, Comparator>(comp_, new_start, new_end);
+
+    auto pos = findPosition(new_interval);
+    intervals_.insert(pos, new_interval);
+    return true;
+  }
+
+  const Comparator* comp_;
+  std::vector<Interval<Slice, Comparator>> intervals_;
+  bool prop_;
+};
+
 }  // namespace ROCKSDB_NAMESPACE

include/rocksdb/db.h

@@ -1119,7 +1119,7 @@ class DB {
   //  }
   virtual std::unique_ptr<MultiScan> NewMultiScan(
       const ReadOptions& /*options*/, ColumnFamilyHandle* /*column_family*/,
-      const std::vector<ScanOptions>& /*scan_opts*/) {
+      const MultiScanArgs& /*scan_opts*/) {
     std::unique_ptr<Iterator> iter(NewErrorIterator(Status::NotSupported()));
     std::unique_ptr<MultiScan> ms_iter =
         std::make_unique<MultiScan>(std::move(iter));
@@ -1966,7 +1966,9 @@ class DB {
   // In the first mode we will try to find the lowest possible level that
   // the file can fit in, and ingest the file into this level (2). A file that
   // have a key range that overlap with the memtable key range will require us
-  // to Flush the memtable first before ingesting the file.
+  // to Flush the memtable first before ingesting the file. If ingested files
+  // have any overlap with each other, level and sequence number assignment
+  // ensure later files overwrite earlier files.
   // In the second mode we will always ingest in the bottom most level (see
   // docs to IngestExternalFileOptions::ingest_behind).
   // For a column family that enables user-defined timestamps, ingesting
@@ -1984,7 +1986,7 @@ class DB {
   //     even if the file compression doesn't match the level compression
   // (3) If IngestExternalFileOptions->ingest_behind is set to true,
   //     we always ingest at the bottommost level, which should be reserved
-  //     for this purpose (see DBOPtions::allow_ingest_behind flag).
+  //     for this purpose (see ColumnFamilyOptions::cf_allow_ingest_behind).
   // (4) If IngestExternalFileOptions->fail_if_not_bottommost_level is set to
   //     true, then this method can return Status:TryAgain() indicating that
   //     the files cannot be ingested to the bottommost level, and it is the

include/rocksdb/iterator.h

@@ -109,7 +109,7 @@ class Iterator : public IteratorBase {
   //
   // If Prepare() is called, it overrides the iterate_upper_bound in
   // ReadOptions
-  virtual void Prepare(const std::vector<ScanOptions>& /*scan_opts*/) {}
+  virtual void Prepare(const MultiScanArgs& /*scan_opts*/) {}
 };
 
 // Return an empty iterator (yields nothing).

include/rocksdb/metadata.h

@@ -239,6 +239,9 @@ struct ColumnFamilyMetaData {
   // The name of the column family.
   std::string name;
   // The metadata of all levels in this column family.
+  // levels[i] contains files in level i.
+  // For level 0, files with recent updates are ordered first.
+  // For level 1+, files are ordered by increasing key range.
   std::vector<LevelMetaData> levels;
 
   // The total size of all blob files

include/rocksdb/multi_scan.h

@@ -152,9 +152,8 @@ class Scan {
 // A Status exception is thrown if there is an error.
 class MultiScan {
  public:
-  MultiScan(const ReadOptions& read_options,
-            const std::vector<ScanOptions>& scan_opts, DB* db,
-            ColumnFamilyHandle* cfh);
+  MultiScan(const ReadOptions& read_options, const MultiScanArgs& scan_opts,
+            DB* db, ColumnFamilyHandle* cfh);
 
   explicit MultiScan(std::unique_ptr<Iterator>&& db_iter)
       : db_iter_(std::move(db_iter)) {}
@@ -220,15 +219,15 @@ class MultiScan {
   };
 
   MultiScanIterator begin() {
-    return MultiScanIterator(scan_opts_, db_, cfh_, read_options_,
-                             &upper_bound_, db_iter_);
+    return MultiScanIterator(scan_opts_.GetScanRanges(), db_, cfh_,
+                             read_options_, &upper_bound_, db_iter_);
   }
 
   std::nullptr_t end() { return nullptr; }
 
  private:
   ReadOptions read_options_;
-  const std::vector<ScanOptions> scan_opts_;
+  const MultiScanArgs scan_opts_;
   DB* db_;
   ColumnFamilyHandle* cfh_;
   Slice upper_bound_;

include/rocksdb/options.h

@@ -498,6 +498,7 @@ struct CompactionServiceJobInfo {
   // the output level of the compaction.
   int output_level;
 
+  CompactionServiceJobInfo() {}
   CompactionServiceJobInfo(std::string db_name_, std::string db_id_,
                            std::string db_session_id_, uint32_t cf_id_,
                            std::string cf_name_, uint64_t job_id_,
@@ -1360,16 +1361,11 @@ struct DBOptions {
   // Dynamically changeable through SetDBOptions() API.
   bool avoid_flush_during_shutdown = false;
 
-  // Set this option to true during creation of database if you want
-  // to be able to ingest behind (call IngestExternalFile() skipping keys
-  // that already exist, rather than overwriting matching keys).
-  // Setting this option to true has the following effects:
-  // 1) Disable some internal optimizations around SST file compression.
-  // 2) Reserve the last level for ingested files only.
-  // 3) Compaction will not include any file from the last level.
-  // Note that only Universal Compaction supports allow_ingest_behind.
-  // `num_levels` should be >= 3 if this option is turned on.
+  // DEPRECATED: use ColumnFamilyOptions::cf_allow_ingest_behind instead.
+  // This option might be removed in a future release.
   //
+  // See comment for `ColumnFamilyOptions::cf_allow_ingest_behind` for
+  // detail about the option's functionality and use cases.
   //
   // DEFAULT: false
   // Immutable.
@@ -1780,6 +1776,101 @@ struct ScanOptions {
       : range(_start, _upper_bound) {}
 };
 
+// Container for multiple scan ranges that can be used with MultiScan.
+// This replaces std::vector<ScanOptions> with a more efficient implementation
+// that can merge overlapping ranges.
+class MultiScanArgs {
+ public:
+  // Constructor that takes a comparator
+  explicit MultiScanArgs(const Comparator* comparator = BytewiseComparator())
+      : comp_(comparator) {}
+
+  // Copy Constructor
+  MultiScanArgs(const MultiScanArgs& other) {
+    comp_ = other.comp_;
+    original_ranges_ = other.original_ranges_;
+    io_coalesce_threshold = other.io_coalesce_threshold;
+    max_prefetch_size = other.max_prefetch_size;
+  }
+  MultiScanArgs(MultiScanArgs&& other) noexcept
+      : io_coalesce_threshold(other.io_coalesce_threshold),
+        max_prefetch_size(other.max_prefetch_size),
+        comp_(other.comp_),
+        original_ranges_(std::move(other.original_ranges_)) {}
+
+  MultiScanArgs& operator=(const MultiScanArgs& other) {
+    comp_ = other.comp_;
+    original_ranges_ = other.original_ranges_;
+    io_coalesce_threshold = other.io_coalesce_threshold;
+    max_prefetch_size = other.max_prefetch_size;
+    return *this;
+  }
+
+  MultiScanArgs& operator=(MultiScanArgs&& other) noexcept {
+    if (this != &other) {
+      comp_ = other.comp_;
+      original_ranges_ = std::move(other.original_ranges_);
+      io_coalesce_threshold = other.io_coalesce_threshold;
+      max_prefetch_size = other.max_prefetch_size;
+    }
+    return *this;
+  }
+
+  void insert(const Slice& s, const Slice& b) {
+    original_ranges_.emplace_back(s, b);
+  }
+
+  void insert(const Slice& s, const Slice& b,
+              const std::optional<std::unordered_map<std::string, std::string>>&
+                  property_bag) {
+    original_ranges_.emplace_back(s, b);
+    original_ranges_.back().property_bag = property_bag;
+  }
+
+  void insert(const Slice& s) { original_ranges_.emplace_back(s); }
+
+  void insert(const Slice& s,
+              const std::optional<std::unordered_map<std::string, std::string>>&
+                  property_bag) {
+    original_ranges_.emplace_back(s);
+    original_ranges_.back().property_bag = property_bag;
+  }
+
+  size_t size() const { return original_ranges_.size(); }
+  bool empty() const { return original_ranges_.empty(); }
+
+  void reserve(size_t size) { original_ranges_.reserve(size); }
+
+  operator std::vector<ScanOptions>*() { return &original_ranges_; }
+
+  operator const std::vector<ScanOptions>*() const { return &original_ranges_; }
+  // Destructor
+  ~MultiScanArgs() {}
+
+  const std::vector<ScanOptions>& GetScanRanges() const {
+    return original_ranges_;
+  }
+
+  uint64_t io_coalesce_threshold = 16 << 10;  // 16KB by default
+
+  // Maximum size (in bytes) for the data blocks loaded by a MultiScan.
+  // This limits the amount of I/O and memory usage by pinned data blocks.
+  //
+  // When set to 0 (the default), there is no limit. When the limit is reached,
+  // the iterator will start returning Status::PrefetchLimitReached().
+  //
+  // Note that prefetching happens only once in Prepare(), which is different
+  // from ReadOptions::readahead_size, which applies any time the iterator does
+  // I/O.
+  // Note that this limit is per file and applies to compressed block size.
+  uint64_t max_prefetch_size = 0;
+
+ private:
+  // The comparator used for ordering ranges
+  const Comparator* comp_;
+  std::vector<ScanOptions> original_ranges_;
+};
+
 // Options that control read operations
 struct ReadOptions {
   // *** BEGIN options relevant to point lookups as well as scans ***
@@ -2344,7 +2435,47 @@ struct CompactRangeOptions {
   double blob_garbage_collection_age_cutoff = -1;
 };
 
-// IngestExternalFileOptions is used by IngestExternalFile()
+// IngestExternalFileOptions setting guide:
+//
+// The options in IngestExternalFileOptions interact in complex ways depending
+// on the source and overlap of SST files. Below is a summary of recommended
+// non-default settings for common use cases:
+//
+// 1. Ingesting only SST writer generated non-overlapping SSTs that are not
+// expected to overlap with existing data:
+//    - Optionally set fail_if_not_bottommost_level = true to enforce placement
+//    in the last level. This is better paird with SST partitioner to guarantee
+//    that there are no existing file with keys across the ingesting key range.
+//    - Set allow_blocking_flush to false: Not expecting to overlap with
+//    memtable and cause a flush.
+//    - If snapshot consistency is not expected, set snapshot_consistency to
+//    false and allow_global_seqno to false. allow_global_seqno = false will
+//    fail ingestion if any input file overlap with each other.
+//
+// 2. Ingesting SST writer generated overlapping SSTs:
+//    - order files with older updates first, newer overwrites later.
+//    - Set allow_global_seqno = true since newer files need to be assigned
+//    larger sequence numbers.
+//
+// 3. Ingesting DB generated SSTs: overlapping with target CF data is not
+// allowed. Input files are allowed to contain both DB generated files and SST
+// file writer generated files. They will all be treated as DB generated.
+//    - Set allow_db_generated_files = true.
+//    - Set snapshot_consistency = false: snapshot consistency requires
+// assigning higher sequence number to ingested files. DB generated files
+// don't support global seqno assignment yet.
+//    - Set allow_blocking_flush to false: Not expecting to overlap with
+//    memtable and cause a flush.
+//    - If the source live DB is running, set link_files = true instead of
+//    move_files.
+// 3a) SST files are non-overlapping and all keys have seqno 0: e.g., a
+// temporary RocksDB instance used to sort some data, and compacts all
+// data into the last level before ingestion.
+//    - Optionally set fail_if_not_bottommost_level = true to enforce placement
+//    in the last level.
+// 3b) SST files are overlapping, e.g. ingesting files from one CF to another.
+//    - Ensure older updates are ordered first and newer updates are ordered
+//    later. See more in option comment for allow_db_generated_files.
 struct IngestExternalFileOptions {
   // Can be set to true to move the files instead of copying them.
   // The input files will be unlinked after successful ingestion.
@@ -2361,10 +2492,20 @@ struct IngestExternalFileOptions {
   // If set to false, an ingested file keys could appear in existing snapshots
   // that where created before the file was ingested.
   bool snapshot_consistency = true;
-  // If set to false, IngestExternalFile() will fail if the file key range
+  // Enables assiging a global sequence number to each ingested file, i.e.,
+  // all keys in the ingested file will be treated as having this seqno.
+  // If set to false, we will use the sequence numbers in the ingested file
+  // as is, and IngestExternalFile() will fail if the ingested key range
   // overlaps with existing keys or tombstones or output of ongoing compaction
-  // during file ingestion in the DB (the conditions under which a global_seqno
-  // must be assigned to the ingested file).
+  // in the CF (the conditions under which a global seqno must be assigned to
+  // the ingested file).
+  // If the ingested files overlap with each other, we need to assign global
+  // sequence to the ingested files and this option needs to be enabled. One
+  // exception to this is when ingesting DB generated SST files (see option
+  // allow_db_generated_files below). DB generated files do not support
+  // global seqno assignment and can be ingested even if they overlap with
+  // each other. This option has no effect when allow_db_generated_files is
+  // enabled.
   bool allow_global_seqno = true;
   // Normally (true), IngestExternalFile() will trigger and block for flushing
   // memtable(s) if there is overlap between ingested files and memtable(s). If
@@ -2376,8 +2517,8 @@ struct IngestExternalFileOptions {
   // to be skipped rather than overwriting existing data under that key.
   // Use case: back-fill of some historical data in the database without
   // over-writing existing newer version of data.
-  // This option could only be used if the DB has been running
-  // with allow_ingest_behind=true since the dawn of time.
+  // This option could only be used if the CF has been running
+  // with cf_allow_ingest_behind=true since CF creation (or before any write).
   // All files will be ingested at the bottommost level with seqno=0.
   bool ingest_behind = false;
   // DEPRECATED - Set to true if you would like to write global_seqno to
@@ -2430,18 +2571,53 @@ struct IngestExternalFileOptions {
   //
   // XXX: "bottommost" is obsolete/confusing terminology to refer to last level
   bool fail_if_not_bottommost_level = false;
-  // EXPERIMENTAL
-  // Enables ingestion of files not generated by SstFileWriter. When true:
+  // EXPERIMENTAL, SUBJECT TO CHANGE
+  //
+  // Enables special mode of ingestion that allows files generated by a live DB,
+  // instead of SstFileWriter. When true:
   // - Allows files to be ingested when their cf_id doesn't match the CF they
   //   are being ingested into.
+  // - Allows files with any sequence numbers to be ingested.
+  // - Original sequence numbers are preserved (no reassignment).
+  //
   // REQUIREMENTS:
-  // - Ingested files must not overlap with existing keys.
-  // - `write_global_seqno` must be false.
-  // - All keys in ingested files should have sequence number 0. We fail
-  // ingestion if any sequence numbers is non-zero.
-  // WARNING: If a DB contains ingested files generated by another DB/CF,
-  // RepairDB() may not recover these files correctly, potentially leading to
-  // data loss.
+  // - Ingested files must NOT overlap with any existing data in the DB. Since
+  //   no sequence number reassignment is performed on db generated files.
+  //   Ingestion will fail if any overlap is detected. However, input files
+  //   are allowed to overlap with each other when this option is enabled. This
+  //   is useful when ingesting multiple levels of files from a CF, where
+  //   levels naturally overlap with each other.
+  // - CAUTION: If input files overlap with each other, then for any given user
+  //   key appearing in multiple files, earlier files MUST have smaller sequence
+  //   numbers than later files. Later files will be placed at a higher level
+  //   (smaller level number). This is to ensure the LSM invariant where for
+  //   the same key, recent updates are in higher levels. This means that
+  //   if you are ingesting files from multiple levels of a CF, you should
+  //   put files from lower levels first, and files from higher levels later.
+  //   Example for getting files from a CF for ingestion:
+  //
+  // ColumnFamilyMetaData cf_meta;
+  // from_db->GetColumnFamilyMetaData(from_cf, &cf_meta);
+  // // iterate in reverse to start from lowest level
+  // for (auto level_meta = cf_meta.levels.rbegin();
+  //      level_meta != cf_meta.levels.rend(); ++level_meta) {
+  //   // L0 files need to be added in reverse order so we iterate in reverse
+  //   // within a level too
+  //   for (auto file_meta = level_meta->files.rbegin();
+  //        file_meta != level_meta->files.rend(); ++file_meta) {
+  //     // Add file for ingestion
+  //   }
+  // }
+  //
+  //   WARNING: Violating the sequence number ordering requirement will cause
+  //   LSM invariant violations and may lead to incorrect reads or data
+  //   corruption.
+  // - If you would like to enforce that the ingested files do not overlap
+  //   with each other, you can set `fail_if_not_bottommost_level` to true.
+  //   If ingested files overlap with each other, some file will be placed
+  //   above Lmax, failing the ingestion if the option is set.
+  // - `write_global_seqno` must be false (sequence numbers cannot be
+  //    reassigned).
   bool allow_db_generated_files = false;
 
   // Controls whether data and metadata blocks (e.g. index, filter) read during

include/rocksdb/secondary_cache.h

@@ -33,8 +33,8 @@ namespace ROCKSDB_NAMESPACE {
 // Wait() or SecondaryCache::WaitAll() may be skipped if IsReady() happens to
 // return true, but (depending on the implementation) IsReady() might never
 // return true without Wait() or SecondaryCache::WaitAll(). After the handle
-// is known ready, calling Value() is required to avoid a memory leak in case
-// of a cache hit.
+// is known ready, calling Value() and taking ownership is required to avoid
+// a memory leak in case of a cache hit.
 class SecondaryCacheResultHandle {
  public:
   virtual ~SecondaryCacheResultHandle() = default;

include/rocksdb/statistics.h

@@ -542,6 +542,9 @@ enum Tickers : uint32_t {
   // TransactionOptions::large_txn_commit_optimize_threshold.
   NUMBER_WBWI_INGEST,
 
+  // Failure to load the UDI during SST table open
+  SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT,
+
   TICKER_ENUM_MAX
 };
 

include/rocksdb/status.h

@@ -115,6 +115,7 @@ class Status {
     kIOFenced = 14,
     kMergeOperatorFailed = 15,
     kMergeOperandThresholdExceeded = 16,
+    kPrefetchLimitReached = 17,
     kMaxSubCode
   };
 
@@ -318,6 +319,10 @@ class Status {
 
   static Status LockLimit() { return Status(kAborted, kLockLimit); }
 
+  static Status PrefetchLimitReached() {
+    return Status(kIncomplete, kPrefetchLimitReached);
+  }
+
   // Returns true iff the status indicates success.
   bool ok() const {
     MarkChecked();
@@ -486,6 +491,13 @@ class Status {
     return (code() == kIOError) && (subcode() == kIOFenced);
   }
 
+  // Returns true iff the status indicates prefetch limit reached during
+  // MultiScan.
+  bool IsPrefetchLimitReached() const {
+    MarkChecked();
+    return (code() == kIncomplete) && (subcode() == kPrefetchLimitReached);
+  }
+
   // Return a string representation of this status suitable for printing.
   // Returns the string "OK" for success.
   std::string ToString() const;

include/rocksdb/table.h

@@ -440,10 +440,9 @@ struct BlockBasedTableOptions {
   // versions of RocksDB able to read partitioned filters are able to read
   // decoupled partitioned filters.)
   //
-  // decouple_partitioned_filters = false is the original behavior, because of
-  // limitations in the initial implementation, and the new behavior
-  // decouple_partitioned_filters = true is expected to become the new default.
-  bool decouple_partitioned_filters = false;
+  // decouple_partitioned_filters = true is the new default. This option is now
+  // DEPRECATED and might be ignored and/or removed in a future release.
+  bool decouple_partitioned_filters = true;
 
   // Option to generate Bloom/Ribbon filters that minimize memory
   // internal fragmentation.
@@ -503,6 +502,12 @@ struct BlockBasedTableOptions {
   // during table building.
   std::shared_ptr<UserDefinedIndexFactory> user_defined_index_factory = nullptr;
 
+  // EXPERIMENTAL
+  //
+  // Return an error Status if a user_defined_index_factory is configured,
+  // but there's no corresponding UDI block in the SST file being opened.
+  bool fail_if_no_udi_on_open = false;
+
   // If true, place whole keys in the filter (not just prefixes).
   // This must generally be true for gets to be efficient.
   bool whole_key_filtering = true;

include/rocksdb/table_properties.h

@@ -307,6 +307,8 @@ struct TableProperties {
   // table is empty).
   uint64_t key_largest_seqno = UINT64_MAX;
 
+  bool HasKeyLargestSeqno() const { return key_largest_seqno != UINT64_MAX; }
+
   // DB identity
   // db_id is an identifier generated the first time the DB is created
   // If DB identity is unset or unassigned, `db_id` will be an empty string.

include/rocksdb/user_defined_index.h

@@ -27,6 +27,11 @@ inline const std::string kUserDefinedIndexPrefix =
 // It allows users to define their own index format and build custom
 // indexes during table building. Currently, only a monolithic index
 // block is supported (no partitioned index).
+//
+// This is currently supported only for a restricted set of use cases. The
+// CF must be ingest only, and only files containing Puts generated by
+// SstFileWriter are supported. The user_comparator used for the CF must
+// be BytewiseComparator.
 
 // The interface for building user-defined index.
 class UserDefinedIndexBuilder {
@@ -51,6 +56,10 @@ class UserDefinedIndexBuilder {
   // The previous index entry key and the new index entry key cover
   // all the keys in the data block associated with the new index entry.
   //
+  // The last_key_in_current_block and first_key_in_next_block will be user
+  // keys, i.e the user key string, and optionally the user timestamp if one
+  // is configured, without a sequence number suffix.
+  //
   // Called before the OnKeyAdded() call for first_key_in_next_block.
   // @last_key_in_current_block: The last key in the current data block
   // @first_key_in_next_block: it will be nullptr if the entry being added is
@@ -72,6 +81,9 @@ class UserDefinedIndexBuilder {
   // override OnKeyAdded() if they need to collect additional information.
   // The type argument indicates whether the value is a full value or partial.
   // At the moment, only full values are supported.
+  //
+  // The key will be a user key. RocksDB guarantees that there will only be
+  // one entry for each key in the file/index.
   virtual void OnKeyAdded(const Slice& /*key*/, ValueType /*type*/,
                           const Slice& /*value*/) {}
 
@@ -100,6 +112,14 @@ class UserDefinedIndexIterator {
   // termination criteria, kInbound if the data block is definitely fully
   // within bounds, or kUnknown if the data block could be partially
   // within bounds.
+  // The UDI implementation needs to be careful about returning kOutOfBound.
+  // If a limit key is specified in ScanOptions, an implementation that
+  // does not store the first key in the block for the corresponding index
+  // entry cannot reliably determine if the block is out of bounds. It must
+  // compare against the previous index key to determine if the current block
+  // is out of bounds w.r.t the limit. Other termination criteria (specified
+  // in property_bag) may cause the scan to terminate earlier, in which case
+  // kOutOfBound can be returned earlier.
   virtual Status SeekAndGetResult(const Slice& target,
                                   IterateResult* result) = 0;
 
@@ -130,6 +150,12 @@ class UserDefinedIndexFactory : public Customizable {
  public:
   virtual ~UserDefinedIndexFactory() = default;
 
+  static const char* Type() { return "UserDefinedIndexFactory"; }
+
+  static Status CreateFromString(
+      const ConfigOptions& config_options, const std::string& value,
+      std::shared_ptr<UserDefinedIndexFactory>* factory);
+
   // Create a new builder for user-defined index.
   virtual UserDefinedIndexBuilder* NewBuilder() const = 0;
 

include/rocksdb/utilities/stackable_db.h

@@ -292,7 +292,7 @@ class StackableDB : public DB {
   using DB::NewMultiScan;
   std::unique_ptr<MultiScan> NewMultiScan(
       const ReadOptions& opts, ColumnFamilyHandle* column_family,
-      const std::vector<ScanOptions>& scan_opts) override {
+      const MultiScanArgs& scan_opts) override {
     return db_->NewMultiScan(opts, column_family, scan_opts);
   }
 

include/rocksdb/version.h

@@ -13,7 +13,7 @@
 // minor or major version number planned for release.
 #define ROCKSDB_MAJOR 10
 #define ROCKSDB_MINOR 6
-#define ROCKSDB_PATCH 0
+#define ROCKSDB_PATCH 2
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these

memory/memory_allocator_impl.h

@@ -12,8 +12,8 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-struct CustomDeleter {
-  CustomDeleter(MemoryAllocator* a = nullptr) : allocator(a) {}
+struct CacheAllocationDeleter {
+  CacheAllocationDeleter(MemoryAllocator* a = nullptr) : allocator(a) {}
 
   void operator()(char* ptr) const {
     if (allocator) {
@@ -26,7 +26,7 @@ struct CustomDeleter {
   MemoryAllocator* allocator;
 };
 
-using CacheAllocationPtr = std::unique_ptr<char[], CustomDeleter>;
+using CacheAllocationPtr = std::unique_ptr<char[], CacheAllocationDeleter>;
 
 inline CacheAllocationPtr AllocateBlock(size_t size,
                                         MemoryAllocator* allocator) {

memtable/inlineskiplist.h

@@ -44,8 +44,6 @@
 #include <assert.h>
 #include <stdlib.h>
 
-#include <algorithm>
-#include <atomic>
 #include <type_traits>
 
 #include "memory/allocator.h"
@@ -53,7 +51,7 @@
 #include "port/port.h"
 #include "rocksdb/slice.h"
 #include "test_util/sync_point.h"
-#include "util/coding.h"
+#include "util/atomic.h"
 #include "util/random.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -215,18 +213,17 @@ class InlineSkipList {
   Comparator const compare_;
   Node* const head_;
 
-  // Modified only by Insert().  Read racily by readers, but stale
-  // values are ok.
-  std::atomic<int> max_height_;  // Height of the entire list
+  // Maximum height of any node in the list (or in the process of being added).
+  //  Modified only by Insert().  Relaxed reads are always OK because starting
+  // from higher levels only helps efficiency, not correctness.
+  RelaxedAtomic<int> max_height_;
 
   // seq_splice_ is a Splice used for insertions in the non-concurrent
   // case.  It caches the prev and next found during the most recent
   // non-concurrent insertion.
   Splice* seq_splice_;
 
-  inline int GetMaxHeight() const {
-    return max_height_.load(std::memory_order_relaxed);
-  }
+  inline int GetMaxHeight() const { return max_height_.LoadRelaxed(); }
 
   int RandomHeight();
 
@@ -311,7 +308,7 @@ struct InlineSkipList<Comparator>::Node {
   // Stores the height of the node in the memory location normally used for
   // next_[0].  This is used for passing data from AllocateKey to Insert.
   void StashHeight(const int height) {
-    assert(sizeof(int) <= sizeof(next_[0]));
+    static_assert(sizeof(int) <= sizeof(next_[0]));
     memcpy(static_cast<void*>(&next_[0]), &height, sizeof(int));
   }
 
@@ -332,30 +329,30 @@ struct InlineSkipList<Comparator>::Node {
     assert(n >= 0);
     // Use an 'acquire load' so that we observe a fully initialized
     // version of the returned Node.
-    return ((&next_[0] - n)->load(std::memory_order_acquire));
+    return ((&next_[0] - n)->Load());
   }
 
   void SetNext(int n, Node* x) {
     assert(n >= 0);
     // Use a 'release store' so that anybody who reads through this
     // pointer observes a fully initialized version of the inserted node.
-    (&next_[0] - n)->store(x, std::memory_order_release);
+    (&next_[0] - n)->Store(x);
   }
 
   bool CASNext(int n, Node* expected, Node* x) {
     assert(n >= 0);
-    return (&next_[0] - n)->compare_exchange_strong(expected, x);
+    return (&next_[0] - n)->CasStrong(expected, x);
   }
 
   // No-barrier variants that can be safely used in a few locations.
   Node* NoBarrier_Next(int n) {
     assert(n >= 0);
-    return (&next_[0] - n)->load(std::memory_order_relaxed);
+    return (&next_[0] - n)->LoadRelaxed();
   }
 
   void NoBarrier_SetNext(int n, Node* x) {
     assert(n >= 0);
-    (&next_[0] - n)->store(x, std::memory_order_relaxed);
+    (&next_[0] - n)->StoreRelaxed(x);
   }
 
   // Insert node after prev on specific level.
@@ -369,7 +366,7 @@ struct InlineSkipList<Comparator>::Node {
  private:
   // next_[0] is the lowest level link (level 0).  Higher levels are
   // stored _earlier_, so level 1 is at next_[-1].
-  std::atomic<Node*> next_[1];
+  AcqRelAtomic<Node*> next_[1];
 };
 
 template <class Comparator>
@@ -789,7 +786,7 @@ char* InlineSkipList<Comparator>::AllocateKey(size_t key_size) {
 template <class Comparator>
 typename InlineSkipList<Comparator>::Node*
 InlineSkipList<Comparator>::AllocateNode(size_t key_size, int height) {
-  auto prefix = sizeof(std::atomic<Node*>) * (height - 1);
+  auto prefix = sizeof(AcqRelAtomic<Node*>) * (height - 1);
 
   // prefix is space for the height - 1 pointers that we store before
   // the Node instance (next_[-(height - 1) .. -1]).  Node starts at
@@ -923,9 +920,9 @@ bool InlineSkipList<Comparator>::Insert(const char* key, Splice* splice,
   int height = x->UnstashHeight();
   assert(height >= 1 && height <= kMaxHeight_);
 
-  int max_height = max_height_.load(std::memory_order_relaxed);
+  int max_height = max_height_.LoadRelaxed();
   while (height > max_height) {
-    if (max_height_.compare_exchange_weak(max_height, height)) {
+    if (max_height_.CasWeakRelaxed(max_height, height)) {
       // successfully updated it
       max_height = height;
       break;

memtable/skiplist.h

@@ -34,10 +34,9 @@
 #include <assert.h>
 #include <stdlib.h>
 
-#include <atomic>
-
 #include "memory/allocator.h"
 #include "port/port.h"
+#include "util/atomic.h"
 #include "util/random.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -128,7 +127,7 @@ class SkipList {
 
   // Modified only by Insert().  Read racily by readers, but stale
   // values are ok.
-  std::atomic<int> max_height_;  // Height of the entire list
+  RelaxedAtomic<int> max_height_;  // Height of the entire list
 
   // Used for optimizing sequential insert patterns.  Tricky.  prev_[i] for
   // i up to max_height_ is the predecessor of prev_[0] and prev_height_
@@ -137,9 +136,7 @@ class SkipList {
   int32_t prev_height_;
   Node** prev_;
 
-  inline int GetMaxHeight() const {
-    return max_height_.load(std::memory_order_relaxed);
-  }
+  inline int GetMaxHeight() const { return max_height_.LoadRelaxed(); }
 
   Node* NewNode(const Key& key, int height);
   int RandomHeight();
@@ -179,35 +176,35 @@ struct SkipList<Key, Comparator>::Node {
     assert(n >= 0);
     // Use an 'acquire load' so that we observe a fully initialized
     // version of the returned Node.
-    return (next_[n].load(std::memory_order_acquire));
+    return (next_[n].Load());
   }
   void SetNext(int n, Node* x) {
     assert(n >= 0);
     // Use a 'release store' so that anybody who reads through this
     // pointer observes a fully initialized version of the inserted node.
-    next_[n].store(x, std::memory_order_release);
+    next_[n].Store(x);
   }
 
   // No-barrier variants that can be safely used in a few locations.
   Node* NoBarrier_Next(int n) {
     assert(n >= 0);
-    return next_[n].load(std::memory_order_relaxed);
+    return next_[n].LoadRelaxed();
   }
   void NoBarrier_SetNext(int n, Node* x) {
     assert(n >= 0);
-    next_[n].store(x, std::memory_order_relaxed);
+    next_[n].StoreRelaxed(x);
   }
 
  private:
   // Array of length equal to the node height.  next_[0] is lowest level link.
-  std::atomic<Node*> next_[1];
+  AcqRelAtomic<Node*> next_[1];
 };
 
 template <typename Key, class Comparator>
 typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::NewNode(
     const Key& key, int height) {
   char* mem = allocator_->AllocateAligned(
-      sizeof(Node) + sizeof(std::atomic<Node*>) * (height - 1));
+      sizeof(Node) + sizeof(AcqRelAtomic<Node*>) * (height - 1));
   return new (mem) Node(key);
 }
 
@@ -494,7 +491,7 @@ void SkipList<Key, Comparator>::Insert(const Key& key) {
     // the loop below.  In the former case the reader will
     // immediately drop to the next level since nullptr sorts after all
     // keys.  In the latter case the reader will use the new node.
-    max_height_.store(height, std::memory_order_relaxed);
+    max_height_.StoreRelaxed(height);
   }
 
   Node* x = NewNode(key, height);

monitoring/statistics.cc

@@ -273,6 +273,8 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
     {FILE_READ_CORRUPTION_RETRY_SUCCESS_COUNT,
      "rocksdb.file.read.corruption.retry.success.count"},
     {NUMBER_WBWI_INGEST, "rocksdb.number.wbwi.ingest"},
+    {SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT,
+     "rocksdb.sst.user.defined.index.load.fail.count"},
 };
 
 const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {

options/cf_options.cc

@@ -893,6 +893,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
          {offsetof(struct ImmutableCFOptions, persist_user_defined_timestamps),
           OptionType::kBoolean, OptionVerificationType::kNormal,
           OptionTypeFlags::kCompareLoose}},
+        {"cf_allow_ingest_behind",
+         {offsetof(struct ImmutableCFOptions, cf_allow_ingest_behind),
+          OptionType::kBoolean, OptionVerificationType::kNormal,
+          OptionTypeFlags::kNone}},
 };
 
 const std::string OptionsHelper::kCFOptionsName = "ColumnFamilyOptions";
@@ -1032,7 +1036,8 @@ ImmutableCFOptions::ImmutableCFOptions(const ColumnFamilyOptions& cf_options)
       sst_partitioner_factory(cf_options.sst_partitioner_factory),
       blob_cache(cf_options.blob_cache),
       persist_user_defined_timestamps(
-          cf_options.persist_user_defined_timestamps) {}
+          cf_options.persist_user_defined_timestamps),
+      cf_allow_ingest_behind(cf_options.cf_allow_ingest_behind) {}
 
 ImmutableOptions::ImmutableOptions() : ImmutableOptions(Options()) {}
 

options/cf_options.h

@@ -82,6 +82,8 @@ struct ImmutableCFOptions {
   std::shared_ptr<Cache> blob_cache;
 
   bool persist_user_defined_timestamps;
+
+  bool cf_allow_ingest_behind;
 };
 
 struct ImmutableOptions : public ImmutableDBOptions, public ImmutableCFOptions {

options/options.cc

@@ -462,6 +462,8 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
                    experimental_mempurge_threshold);
   ROCKS_LOG_HEADER(log, "           Options.memtable_max_range_deletions: %d",
                    memtable_max_range_deletions);
+  ROCKS_LOG_HEADER(log, "                 Options.cf_allow_ingest_behind: %s",
+                   cf_allow_ingest_behind ? "true" : "false");
 }  // ColumnFamilyOptions::Dump
 
 void Options::Dump(Logger* log) const {

options/options_helper.cc

@@ -339,6 +339,7 @@ void UpdateColumnFamilyOptions(const ImmutableCFOptions& ioptions,
   cf_opts->persist_user_defined_timestamps =
       ioptions.persist_user_defined_timestamps;
   cf_opts->default_temperature = ioptions.default_temperature;
+  cf_opts->cf_allow_ingest_behind = ioptions.cf_allow_ingest_behind;
 
   // TODO(yhchiang): find some way to handle the following derived options
   // * max_file_size

options/options_settable_test.cc

@@ -203,7 +203,8 @@ TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) {
       "max_auto_readahead_size=0;"
       "prepopulate_block_cache=kDisable;"
       "initial_auto_readahead_size=0;"
-      "num_file_reads_for_auto_readahead=0",
+      "num_file_reads_for_auto_readahead=0;"
+      "fail_if_no_udi_on_open=true",
       new_bbto));
 
   ASSERT_EQ(unset_bytes_base,
@@ -681,7 +682,8 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
       "uncache_aggressiveness=1234;"
       "paranoid_memory_checks=1;"
       "memtable_op_scan_flush_trigger=123;"
-      "memtable_avg_op_scan_flush_trigger=12;",
+      "memtable_avg_op_scan_flush_trigger=12;"
+      "cf_allow_ingest_behind=1;",
       new_options));
 
   ASSERT_NE(new_options->blob_cache.get(), nullptr);

port/mmap.cc

@@ -43,7 +43,7 @@ MemMapping& MemMapping::operator=(MemMapping&& other) noexcept {
     return *this;
   }
   this->~MemMapping();
-  std::memcpy(this, &other, sizeof(*this));
+  std::memcpy(static_cast<void*>(this), &other, sizeof(*this));
   new (&other) MemMapping();
   return *this;
 }

port/win/xpress_win.cc

@@ -125,6 +125,57 @@ bool Compress(const char* input, size_t length, std::string* output) {
   return true;
 }
 
+size_t CompressWithMaxSize(const char* input, size_t length, char* output,
+                           size_t max_output_size) {
+  assert(input != nullptr);
+  if (max_output_size == 0) {
+    return 0;
+  }
+  assert(output != nullptr);
+
+  COMPRESS_ALLOCATION_ROUTINES* allocRoutinesPtr = nullptr;
+
+  COMPRESSOR_HANDLE compressor = NULL;
+
+  BOOL success =
+      CreateCompressor(COMPRESS_ALGORITHM_XPRESS,  //  Compression Algorithm
+                       allocRoutinesPtr,  //  Optional allocation routine
+                       &compressor);      //  Handle
+
+  if (!success) {
+#ifdef _DEBUG
+    std::cerr << "XPRESS: Failed to create Compressor LastError: "
+              << GetLastError() << std::endl;
+#endif
+    return 0;
+  }
+
+  std::unique_ptr<void, decltype(CloseCompressorFun)> compressorGuard(
+      compressor, CloseCompressorFun);
+
+  SIZE_T compressed_size = 0;
+  //  Compress
+  success = ::Compress(compressor,                //  Compressor Handle
+                       const_cast<char*>(input),  //  Input buffer
+                       length,                    //  Uncompressed data size
+                       output,                    //  Compressed Buffer
+                       max_output_size,           //  Compressed Buffer size
+                       &compressed_size);         //  Compressed Data size
+
+  if (!success) {
+#ifdef _DEBUG
+    auto error = GetLastError();
+    if (error != ERROR_INSUFFICIENT_BUFFER) {
+      std::cerr << "XPRESS: Failed to compress LastError " << error
+                << std::endl;
+    }
+#endif
+    return 0;
+  } else {
+    return compressed_size;
+  }
+}
+
 char* Decompress(const char* input_data, size_t input_length,
                  size_t* uncompressed_size) {
   assert(input_data != nullptr);

port/win/xpress_win.h

@@ -19,6 +19,10 @@ namespace xpress {
 
 bool Compress(const char* input, size_t length, std::string* output);
 
+// Returns written size or 0 on failure including if buffer is too small.
+size_t CompressWithMaxSize(const char* input, size_t length, char* output,
+                           size_t max_output_size);
+
 char* Decompress(const char* input_data, size_t input_length,
                  size_t* uncompressed_size);