Update version.h to 7.10.2

Summary:
Fix a bug in the calculation of the input buffer address/offset in log_reader.cc. The bug is when consecutive fragments of a compressed record are located at the same offset in the log reader buffer, the second fragment input buffer is treated as a leftover from the previous input buffer. As a result, the offset in the `ZSTD_inBuffer` is not reset.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/11198

Test Plan: Add a unit test in log_test.cc that fails without the fix and passes with it.

Reviewed By: ajkr, cbi42

Differential Revision: D43102692

Pulled By: anand1976

fbshipit-source-id: aa2648f4802c33991b76a3233c5a58d4cc9e77fd

.circleci/config.yml

@@ -76,7 +76,7 @@ commands:
           name: "Test RocksDB"
           shell: powershell.exe
           command: |
-            build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
+            build_tools\run_ci_db_test.ps1 -SuiteRun arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
   pre-steps-macos:
       steps:
         - pre-steps

CMakeLists.txt

@@ -245,6 +245,14 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
   endif(HAS_S390X_MARCH_NATIVE)
 endif(CMAKE_SYSTEM_PROCESSOR MATCHES "s390x")
 
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")
+  CHECK_C_COMPILER_FLAG("-march=loongarch64" HAS_LOONGARCH64)
+  if(HAS_LOONGARCH64)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=loongarch64 -mtune=loongarch64")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=loongarch64 -mtune=loongarch64")
+  endif(HAS_LOONGARCH64)
+endif(CMAKE_SYSTEM_PROCESSOR MATCHES "loongarch64")
+
 option(PORTABLE "build a portable binary" OFF)
 option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
 option(FORCE_AVX "force building with AVX, even when PORTABLE=ON" OFF)
@@ -275,6 +283,9 @@ if(PORTABLE)
     if(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390x")
       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=z196")
     endif()
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "^loongarch64")
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=loongarch64")
+    endif()
   endif()
 else()
   if(MSVC)
@@ -649,6 +660,7 @@ set(SOURCES
         cache/cache.cc
         cache/cache_entry_roles.cc
         cache/cache_key.cc
+        cache/cache_helpers.cc
         cache/cache_reservation_manager.cc
         cache/charged_cache.cc
         cache/clock_cache.cc
@@ -796,6 +808,7 @@ set(SOURCES
         options/options.cc
         options/options_helper.cc
         options/options_parser.cc
+        port/mmap.cc
         port/stack_trace.cc
         table/adaptive/adaptive_table_factory.cc
         table/block_based/binary_search_index_reader.cc
@@ -805,6 +818,7 @@ set(SOURCES
         table/block_based/block_based_table_iterator.cc
         table/block_based/block_based_table_reader.cc
         table/block_based/block_builder.cc
+        table/block_based/block_cache.cc
         table/block_based/block_prefetcher.cc
         table/block_based/block_prefix_index.cc
         table/block_based/data_block_hash_index.cc
@@ -978,6 +992,12 @@ if ( ROCKSDB_PLUGINS )
         plugin/${plugin}/${src}
         PROPERTIES COMPILE_FLAGS "${${plugin}_COMPILE_FLAGS}")
     endforeach()
+    foreach (test ${${plugin}_TESTS})
+      list(APPEND PLUGIN_TESTS plugin/${plugin}/${test})
+      set_source_files_properties(
+        plugin/${plugin}/${test}
+        PROPERTIES COMPILE_FLAGS "${${plugin}_COMPILE_FLAGS}")
+    endforeach()
     foreach (path ${${plugin}_INCLUDE_PATHS})
       include_directories(${path})
     endforeach()
@@ -1469,6 +1489,7 @@ if(WITH_TESTS)
         utilities/ttl/ttl_test.cc
         utilities/util_merge_operators_test.cc
         utilities/write_batch_with_index/write_batch_with_index_test.cc
+	${PLUGIN_TESTS}
     )
   endif()
 

HISTORY.md

@@ -1,15 +1,44 @@
 # Rocksdb Change Log
-## 7.9.2 (12/21/2022)
+## 7.10.2 (02/10/2023)
 ### Bug Fixes
-* Fixed a heap use after free bug in async scan prefetching when the scan thread and another thread try to read and load the same seek block into cache.
+* Fixed a bug in DB open/recovery from a compressed WAL that was caused due to incorrect handling of certain record fragments with the same offset within a WAL block.
+
+## 7.10.1 (02/01/2023)
+### Bug Fixes
+* Fixed a data race on `ColumnFamilyData::flush_reason` caused by concurrent flushes.
+* Fixed `DisableManualCompaction()` and `CompactRangeOptions::canceled` to cancel compactions even when they are waiting on conflicting compactions to finish
+* Fixed a bug in which a successful `GetMergeOperands()` could transiently return `Status::MergeInProgress()`
+* Return the correct error (Status::NotSupported()) to MultiGet caller when ReadOptions::async_io flag is true and IO uring is not enabled. Previously, Status::Corruption() was being returned when the actual failure was lack of async IO support.
+
+## 7.10.0 (01/23/2023)
+### Behavior changes
+* Make best-efforts recovery verify SST unique ID before Version construction (#10962)
+* Introduce `epoch_number` and sort L0 files by `epoch_number` instead of `largest_seqno`. `epoch_number` represents the order of a file being flushed or ingested/imported. Compaction output file will be assigned with the minimum `epoch_number` among input files'. For L0, larger `epoch_number` indicates newer L0 file.
 
-## 7.9.1 (12/8/2022)
 ### Bug Fixes
 * Fixed a regression in iterator where range tombstones after `iterate_upper_bound` is processed.
 * Fixed a memory leak in MultiGet with async_io read option, caused by IO errors during table file open
+* Fixed a bug that multi-level FIFO compaction deletes one file in non-L0 even when `CompactionOptionsFIFO::max_table_files_size` is no exceeded since #10348 or 7.8.0.
+* Fixed a bug caused by `DB::SyncWAL()` affecting `track_and_verify_wals_in_manifest`. Without the fix, application may see "open error: Corruption: Missing WAL with log number" while trying to open the db. The corruption is a false alarm but prevents DB open (#10892).
+* Fixed a BackupEngine bug in which RestoreDBFromLatestBackup would fail if the latest backup was deleted and there is another valid backup available.
+* Fix L0 file misorder corruption caused by ingesting files of overlapping seqnos with memtable entries' through introducing `epoch_number`. Before the fix, `force_consistency_checks=true` may catch the corruption before it's exposed to readers, in which case writes returning `Status::Corruption` would be expected. Also replace the previous incomplete fix (#5958) to the same corruption with this new and more complete fix.
+* Fixed a bug in LockWAL() leading to re-locking mutex (#11020).
+* Fixed a heap use after free bug in async scan prefetching when the scan thread and another thread try to read and load the same seek block into cache.
+* Fixed a heap use after free in async scan prefetching if dictionary compression is enabled, in which case sync read of the compression dictionary gets mixed with async prefetching
+* Fixed a data race bug of `CompactRange()` under `change_level=true` acts on overlapping range with an ongoing file ingestion for level compaction. This will either result in overlapping file ranges corruption at a certain level caught by `force_consistency_checks=true` or protentially two same keys both with seqno 0 in two different levels (i.e, new data ends up in lower/older level). The latter will be caught by assertion in debug build but go silently and result in read returning wrong result in release build. This fix is general so it also replaced previous fixes to a similar problem for `CompactFiles()` (#4665), general `CompactRange()` and auto compaction (commit 5c64fb6 and 87dfc1d).
+* Fixed a bug in compaction output cutting where small output files were produced due to TTL file cutting states were not being updated (#11075).
 
-### Behavior changes
-* Make best-efforts recovery verify SST unique ID before Version construction (#10962)
+### New Features
+* When an SstPartitionerFactory is configured, CompactRange() now automatically selects for compaction any files overlapping a partition boundary that is in the compaction range, even if no actual entries are in the requested compaction range. With this feature, manual compaction can be used to (re-)establish SST partition points when SstPartitioner changes, without a full compaction.
+* Add BackupEngine feature to exclude files from backup that are known to be backed up elsewhere, using `CreateBackupOptions::exclude_files_callback`. To restore the DB, the excluded files must be provided in alternative backup directories using `RestoreOptions::alternate_dirs`.
+
+### Public API Changes
+* Substantial changes have been made to the Cache class to support internal development goals. Direct use of Cache class members is discouraged and further breaking modifications are expected in the future. SecondaryCache has some related changes and implementations will need to be updated. (Unlike Cache, SecondaryCache is still intended to support user implementations, and disruptive changes will be avoided.) (#10975)
+* Add `MergeOperationOutput::op_failure_scope` for merge operator users to control the blast radius of merge operator failures. Existing merge operator users do not need to make any change to preserve the old behavior
+
+### Performance Improvements
+* Updated xxHash source code, which should improve kXXH3 checksum speed, at least on ARM (#11098).
+* Improved CPU efficiency of DB reads, from block cache access improvements (#10975).
 
 ## 7.9.0 (11/21/2022)
 ### Performance Improvements

Makefile

@@ -266,6 +266,7 @@ ROCKSDB_PLUGIN_EXTERNS = $(foreach p, $(ROCKSDB_PLUGIN_W_FUNCS), int $($(p)_FUNC
 ROCKSDB_PLUGIN_BUILTINS = $(foreach p, $(ROCKSDB_PLUGIN_W_FUNCS), {\"$(p)\"\, $($(p)_FUNC)}\,)
 ROCKSDB_PLUGIN_LDFLAGS = $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_LDFLAGS))
 ROCKSDB_PLUGIN_PKGCONFIG_REQUIRES = $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_PKGCONFIG_REQUIRES))
+ROCKSDB_PLUGIN_TESTS = $(foreach p, $(ROCKSDB_PLUGINS), $(foreach test, $($(p)_TESTS), plugin/$(p)/$(test)))
 
 CXXFLAGS += $(foreach plugin, $(ROCKSDB_PLUGINS), $($(plugin)_CXXFLAGS))
 PLATFORM_LDFLAGS += $(ROCKSDB_PLUGIN_LDFLAGS)
@@ -647,10 +648,12 @@ STRESS_OBJECTS =  $(patsubst %.cc, $(OBJ_DIR)/%.o, $(STRESS_LIB_SOURCES))
 ALL_SOURCES  = $(filter-out util/build_version.cc, $(LIB_SOURCES)) $(TEST_LIB_SOURCES) $(MOCK_LIB_SOURCES) $(GTEST_DIR)/gtest/gtest-all.cc
 ALL_SOURCES += $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(CACHE_BENCH_LIB_SOURCES) $(ANALYZER_LIB_SOURCES) $(STRESS_LIB_SOURCES)
 ALL_SOURCES += $(TEST_MAIN_SOURCES) $(TOOL_MAIN_SOURCES) $(BENCH_MAIN_SOURCES)
-ALL_SOURCES += $(ROCKSDB_PLUGIN_SOURCES)
+ALL_SOURCES += $(ROCKSDB_PLUGIN_SOURCES) $(ROCKSDB_PLUGIN_TESTS)
 
+PLUGIN_TESTS = $(patsubst %.cc, %, $(notdir $(ROCKSDB_PLUGIN_TESTS)))
 TESTS = $(patsubst %.cc, %, $(notdir $(TEST_MAIN_SOURCES)))
 TESTS += $(patsubst %.c, %, $(notdir $(TEST_MAIN_SOURCES_C)))
+TESTS += $(PLUGIN_TESTS)
 
 # `make check-headers` to very that each header file includes its own
 # dependencies
@@ -702,6 +705,7 @@ NON_PARALLEL_TEST = \
 	env_test \
 	deletefile_test \
 	db_bloom_filter_test \
+	$(PLUGIN_TESTS) \
 
 PARALLEL_TEST = $(filter-out $(NON_PARALLEL_TEST), $(TESTS))
 
@@ -1355,6 +1359,14 @@ db_sanity_test: $(OBJ_DIR)/tools/db_sanity_test.o $(LIBRARY)
 db_repl_stress: $(OBJ_DIR)/tools/db_repl_stress.o $(LIBRARY)
 	$(AM_LINK)
 
+define MakeTestRule
+$(notdir $(1:%.cc=%)): $(1:%.cc=$$(OBJ_DIR)/%.o) $$(TEST_LIBRARY) $$(LIBRARY)
+	$$(AM_LINK)
+endef
+
+# For each PLUGIN test, create a rule to generate the test executable
+$(foreach test, $(ROCKSDB_PLUGIN_TESTS), $(eval $(call MakeTestRule, $(test))))
+
 arena_test: $(OBJ_DIR)/memory/arena_test.o $(TEST_LIBRARY) $(LIBRARY)
 	$(AM_LINK)
 
@@ -2058,7 +2070,7 @@ JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux
 ifeq ($(PLATFORM), OS_SOLARIS)
 	ARCH := $(shell isainfo -b)
 else ifeq ($(PLATFORM), OS_OPENBSD)
-	ifneq (,$(filter amd64 ppc64 ppc64le s390x arm64 aarch64 sparc64, $(MACHINE)))
+	ifneq (,$(filter amd64 ppc64 ppc64le s390x arm64 aarch64 sparc64 loongarch64, $(MACHINE)))
 		ARCH := 64
 	else
 		ARCH := 32
@@ -2079,7 +2091,7 @@ ifneq ($(origin JNI_LIBC), undefined)
 endif
 
 ifeq (,$(ROCKSDBJNILIB))
-ifneq (,$(filter ppc% s390x arm64 aarch64 sparc64, $(MACHINE)))
+ifneq (,$(filter ppc% s390x arm64 aarch64 sparc64 loongarch64, $(MACHINE)))
 	ROCKSDBJNILIB = librocksdbjni-linux-$(MACHINE)$(JNI_LIBC_POSTFIX).so
 else
 	ROCKSDBJNILIB = librocksdbjni-linux$(ARCH)$(JNI_LIBC_POSTFIX).so

TARGETS

@@ -11,6 +11,7 @@ load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrap
 cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "cache/cache.cc",
         "cache/cache_entry_roles.cc",
+        "cache/cache_helpers.cc",
         "cache/cache_key.cc",
         "cache/cache_reservation_manager.cc",
         "cache/charged_cache.cc",
@@ -163,6 +164,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "options/options.cc",
         "options/options_helper.cc",
         "options/options_parser.cc",
+        "port/mmap.cc",
         "port/port_posix.cc",
         "port/stack_trace.cc",
         "port/win/env_default.cc",
@@ -179,6 +181,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
         "table/block_based/block_based_table_iterator.cc",
         "table/block_based/block_based_table_reader.cc",
         "table/block_based/block_builder.cc",
+        "table/block_based/block_cache.cc",
         "table/block_based/block_prefetcher.cc",
         "table/block_based/block_prefix_index.cc",
         "table/block_based/data_block_footer.cc",
@@ -350,6 +353,7 @@ cpp_library_wrapper(name="rocksdb_lib", srcs=[
 cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "cache/cache.cc",
         "cache/cache_entry_roles.cc",
+        "cache/cache_helpers.cc",
         "cache/cache_key.cc",
         "cache/cache_reservation_manager.cc",
         "cache/charged_cache.cc",
@@ -502,6 +506,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "options/options.cc",
         "options/options_helper.cc",
         "options/options_parser.cc",
+        "port/mmap.cc",
         "port/port_posix.cc",
         "port/stack_trace.cc",
         "port/win/env_default.cc",
@@ -518,6 +523,7 @@ cpp_library_wrapper(name="rocksdb_whole_archive_lib", srcs=[
         "table/block_based/block_based_table_iterator.cc",
         "table/block_based/block_based_table_reader.cc",
         "table/block_based/block_builder.cc",
+        "table/block_based/block_cache.cc",
         "table/block_based/block_prefetcher.cc",
         "table/block_based/block_prefix_index.cc",
         "table/block_based/data_block_footer.cc",

USERS.md

@@ -79,6 +79,9 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.
 ## TiKV
 [TiKV](https://github.com/pingcap/tikv) is a GEO-replicated, high-performance, distributed, transactional key-value database. TiKV is powered by Rust and Raft. TiKV uses RocksDB as its persistence layer.
 
+## Apache Spark
+[Spark Structured Streaming](https://docs.databricks.com/structured-streaming/rocksdb-state-store.html) uses RocksDB as the local state store.
+
 ## Apache Flink
 [Apache Flink](https://flink.apache.org/news/2016/03/08/release-1.0.0.html) uses RocksDB to store state locally on a machine.
 

cache/cache_bench_tool.cc

@@ -226,7 +226,7 @@ struct KeyGen {
   }
 };
 
-char* createValue(Random64& rnd) {
+Cache::ObjectPtr createValue(Random64& rnd) {
   char* rv = new char[FLAGS_value_bytes];
   // Fill with some filler data, and take some CPU time
   for (uint32_t i = 0; i < FLAGS_value_bytes; i += 8) {
@@ -236,28 +236,33 @@ char* createValue(Random64& rnd) {
 }
 
 // Callbacks for secondary cache
-size_t SizeFn(void* /*obj*/) { return FLAGS_value_bytes; }
+size_t SizeFn(Cache::ObjectPtr /*obj*/) { return FLAGS_value_bytes; }
 
-Status SaveToFn(void* obj, size_t /*offset*/, size_t size, void* out) {
-  memcpy(out, obj, size);
+Status SaveToFn(Cache::ObjectPtr from_obj, size_t /*from_offset*/,
+                size_t length, char* out) {
+  memcpy(out, from_obj, length);
   return Status::OK();
 }
 
-// Different deleters to simulate using deleter to gather
-// stats on the code origin and kind of cache entries.
-void deleter1(const Slice& /*key*/, void* value) {
-  delete[] static_cast<char*>(value);
-}
-void deleter2(const Slice& /*key*/, void* value) {
-  delete[] static_cast<char*>(value);
-}
-void deleter3(const Slice& /*key*/, void* value) {
+Status CreateFn(const Slice& data, Cache::CreateContext* /*context*/,
+                MemoryAllocator* /*allocator*/, Cache::ObjectPtr* out_obj,
+                size_t* out_charge) {
+  *out_obj = new char[data.size()];
+  memcpy(*out_obj, data.data(), data.size());
+  *out_charge = data.size();
+  return Status::OK();
+};
+
+void DeleteFn(Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) {
   delete[] static_cast<char*>(value);
 }
 
-Cache::CacheItemHelper helper1(SizeFn, SaveToFn, deleter1);
-Cache::CacheItemHelper helper2(SizeFn, SaveToFn, deleter2);
-Cache::CacheItemHelper helper3(SizeFn, SaveToFn, deleter3);
+Cache::CacheItemHelper helper1(CacheEntryRole::kDataBlock, DeleteFn, SizeFn,
+                               SaveToFn, CreateFn);
+Cache::CacheItemHelper helper2(CacheEntryRole::kIndexBlock, DeleteFn, SizeFn,
+                               SaveToFn, CreateFn);
+Cache::CacheItemHelper helper3(CacheEntryRole::kFilterBlock, DeleteFn, SizeFn,
+                               SaveToFn, CreateFn);
 }  // namespace
 
 class CacheBench {
@@ -436,7 +441,7 @@ class CacheBench {
     uint64_t total_entry_count = 0;
     uint64_t table_occupancy = 0;
     uint64_t table_size = 0;
-    std::set<Cache::DeleterFn> deleters;
+    std::set<const Cache::CacheItemHelper*> helpers;
     StopWatchNano timer(clock);
 
     for (;;) {
@@ -461,7 +466,7 @@ class CacheBench {
                  << BytesToHumanString(static_cast<uint64_t>(
                         1.0 * total_charge / total_entry_count))
                  << "\n"
-                 << "Unique deleters: " << deleters.size() << "\n";
+                 << "Unique helpers: " << helpers.size() << "\n";
             *stats_report = ostr.str();
             return;
           }
@@ -477,14 +482,14 @@ class CacheBench {
       total_key_size = 0;
       total_charge = 0;
       total_entry_count = 0;
-      deleters.clear();
-      auto fn = [&](const Slice& key, void* /*value*/, size_t charge,
-                    Cache::DeleterFn deleter) {
+      helpers.clear();
+      auto fn = [&](const Slice& key, Cache::ObjectPtr /*value*/, size_t charge,
+                    const Cache::CacheItemHelper* helper) {
         total_key_size += key.size();
         total_charge += charge;
         ++total_entry_count;
-        // Something slightly more expensive as in (future) stats by category
-        deleters.insert(deleter);
+        // Something slightly more expensive as in stats by category
+        helpers.insert(helper);
       };
       timer.Start();
       Cache::ApplyToAllEntriesOptions opts;
@@ -533,14 +538,6 @@ class CacheBench {
     for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
       Slice key = gen.GetRand(thread->rnd, max_key_, max_log_);
       uint64_t random_op = thread->rnd.Next();
-      Cache::CreateCallback create_cb = [](const void* buf, size_t size,
-                                           void** out_obj,
-                                           size_t* charge) -> Status {
-        *out_obj = reinterpret_cast<void*>(new char[size]);
-        memcpy(*out_obj, buf, size);
-        *charge = size;
-        return Status::OK();
-      };
 
       timer.Start();
 
@@ -550,8 +547,8 @@ class CacheBench {
           handle = nullptr;
         }
         // do lookup
-        handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
-                                true);
+        handle = cache_->Lookup(key, &helper2, /*context*/ nullptr,
+                                Cache::Priority::LOW, true);
         if (handle) {
           if (!FLAGS_lean) {
             // do something with the data
@@ -579,8 +576,8 @@ class CacheBench {
           handle = nullptr;
         }
         // do lookup
-        handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
-                                true);
+        handle = cache_->Lookup(key, &helper2, /*context*/ nullptr,
+                                Cache::Priority::LOW, true);
         if (handle) {
           if (!FLAGS_lean) {
             // do something with the data

cache/cache_entry_roles.cc

@@ -101,34 +101,4 @@ std::string BlockCacheEntryStatsMapKeys::UsedPercent(CacheEntryRole role) {
   return GetPrefixedCacheEntryRoleName(kPrefix, role);
 }
 
-namespace {
-
-struct Registry {
-  std::mutex mutex;
-  UnorderedMap<Cache::DeleterFn, CacheEntryRole> role_map;
-  void Register(Cache::DeleterFn fn, CacheEntryRole role) {
-    std::lock_guard<std::mutex> lock(mutex);
-    role_map[fn] = role;
-  }
-  UnorderedMap<Cache::DeleterFn, CacheEntryRole> Copy() {
-    std::lock_guard<std::mutex> lock(mutex);
-    return role_map;
-  }
-};
-
-Registry& GetRegistry() {
-  STATIC_AVOID_DESTRUCTION(Registry, registry);
-  return registry;
-}
-
-}  // namespace
-
-void RegisterCacheDeleterRole(Cache::DeleterFn fn, CacheEntryRole role) {
-  GetRegistry().Register(fn, role);
-}
-
-UnorderedMap<Cache::DeleterFn, CacheEntryRole> CopyCacheDeleterRoleMap() {
-  return GetRegistry().Copy();
-}
-
 }  // namespace ROCKSDB_NAMESPACE

cache/cache_entry_roles.h

@@ -7,11 +7,8 @@
 
 #include <array>
 #include <cstdint>
-#include <memory>
-#include <type_traits>
 
 #include "rocksdb/cache.h"
-#include "util/hash_containers.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -20,84 +17,4 @@ extern std::array<std::string, kNumCacheEntryRoles>
 extern std::array<std::string, kNumCacheEntryRoles>
     kCacheEntryRoleToHyphenString;
 
-// To associate cache entries with their role, we use a hack on the
-// existing Cache interface. Because the deleter of an entry can authenticate
-// the code origin of an entry, we can elaborate the choice of deleter to
-// also encode role information, without inferring false role information
-// from entries not choosing to encode a role.
-//
-// The rest of this file is for handling mappings between deleters and
-// roles.
-
-// To infer a role from a deleter, the deleter must be registered. This
-// can be done "manually" with this function. This function is thread-safe,
-// and the registration mappings go into private but static storage. (Note
-// that DeleterFn is a function pointer, not std::function. Registrations
-// should not be too many.)
-void RegisterCacheDeleterRole(Cache::DeleterFn fn, CacheEntryRole role);
-
-// Gets a copy of the registered deleter -> role mappings. This is the only
-// function for reading the mappings made with RegisterCacheDeleterRole.
-// Why only this interface for reading?
-// * This function has to be thread safe, which could incur substantial
-// overhead. We should not pay this overhead for every deleter look-up.
-// * This is suitable for preparing for batch operations, like with
-// CacheEntryStatsCollector.
-// * The number of mappings should be sufficiently small (dozens).
-UnorderedMap<Cache::DeleterFn, CacheEntryRole> CopyCacheDeleterRoleMap();
-
-// ************************************************************** //
-// An automatic registration infrastructure. This enables code
-// to simply ask for a deleter associated with a particular type
-// and role, and registration is automatic. In a sense, this is
-// a small dependency injection infrastructure, because linking
-// in new deleter instantiations is essentially sufficient for
-// making stats collection (using CopyCacheDeleterRoleMap) aware
-// of them.
-
-namespace cache_entry_roles_detail {
-
-template <typename T, CacheEntryRole R>
-struct RegisteredDeleter {
-  RegisteredDeleter() { RegisterCacheDeleterRole(Delete, R); }
-
-  // These have global linkage to help ensure compiler optimizations do not
-  // break uniqueness for each <T,R>
-  static void Delete(const Slice& /* key */, void* value) {
-    // Supports T == Something[], unlike delete operator
-    std::default_delete<T>()(
-        static_cast<typename std::remove_extent<T>::type*>(value));
-  }
-};
-
-template <CacheEntryRole R>
-struct RegisteredNoopDeleter {
-  RegisteredNoopDeleter() { RegisterCacheDeleterRole(Delete, R); }
-
-  static void Delete(const Slice& /* key */, void* /* value */) {
-    // Here was `assert(value == nullptr);` but we can also put pointers
-    // to static data in Cache, for testing at least.
-  }
-};
-
-}  // namespace cache_entry_roles_detail
-
-// Get an automatically registered deleter for value type T and role R.
-// Based on C++ semantics, registration is invoked exactly once in a
-// thread-safe way on first call to this function, for each <T, R>.
-template <typename T, CacheEntryRole R>
-Cache::DeleterFn GetCacheEntryDeleterForRole() {
-  static cache_entry_roles_detail::RegisteredDeleter<T, R> reg;
-  return reg.Delete;
-}
-
-// Get an automatically registered no-op deleter (value should be nullptr)
-// and associated with role R. This is used for Cache "reservation" entries
-// such as for WriteBufferManager.
-template <CacheEntryRole R>
-Cache::DeleterFn GetNoopDeleterForRole() {
-  static cache_entry_roles_detail::RegisteredNoopDeleter<R> reg;
-  return reg.Delete;
-}
-
 }  // namespace ROCKSDB_NAMESPACE

cache/cache_entry_stats.h

@@ -10,8 +10,8 @@
 #include <memory>
 #include <mutex>
 
-#include "cache/cache_helpers.h"
 #include "cache/cache_key.h"
+#include "cache/typed_cache.h"
 #include "port/lang.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/status.h"
@@ -111,11 +111,14 @@ class CacheEntryStatsCollector {
   // Gets or creates a shared instance of CacheEntryStatsCollector in the
   // cache itself, and saves into `ptr`. This shared_ptr will hold the
   // entry in cache until all refs are destroyed.
-  static Status GetShared(Cache *cache, SystemClock *clock,
+  static Status GetShared(Cache *raw_cache, SystemClock *clock,
                           std::shared_ptr<CacheEntryStatsCollector> *ptr) {
-    const Slice &cache_key = GetCacheKey();
+    assert(raw_cache);
+    BasicTypedCacheInterface<CacheEntryStatsCollector, CacheEntryRole::kMisc>
+        cache{raw_cache};
 
-    Cache::Handle *h = cache->Lookup(cache_key);
+    const Slice &cache_key = GetCacheKey();
+    auto h = cache.Lookup(cache_key);
     if (h == nullptr) {
       // Not yet in cache, but Cache doesn't provide a built-in way to
       // avoid racing insert. So we double-check under a shared mutex,
@@ -123,15 +126,15 @@ class CacheEntryStatsCollector {
       STATIC_AVOID_DESTRUCTION(std::mutex, static_mutex);
       std::lock_guard<std::mutex> lock(static_mutex);
 
-      h = cache->Lookup(cache_key);
+      h = cache.Lookup(cache_key);
       if (h == nullptr) {
-        auto new_ptr = new CacheEntryStatsCollector(cache, clock);
+        auto new_ptr = new CacheEntryStatsCollector(cache.get(), clock);
         // TODO: non-zero charge causes some tests that count block cache
         // usage to go flaky. Fix the problem somehow so we can use an
         // accurate charge.
         size_t charge = 0;
-        Status s = cache->Insert(cache_key, new_ptr, charge, Deleter, &h,
-                                 Cache::Priority::HIGH);
+        Status s =
+            cache.Insert(cache_key, new_ptr, charge, &h, Cache::Priority::HIGH);
         if (!s.ok()) {
           assert(h == nullptr);
           delete new_ptr;
@@ -140,11 +143,11 @@ class CacheEntryStatsCollector {
       }
     }
     // If we reach here, shared entry is in cache with handle `h`.
-    assert(cache->GetDeleter(h) == Deleter);
+    assert(cache.get()->GetCacheItemHelper(h) == &cache.kBasicHelper);
 
     // Build an aliasing shared_ptr that keeps `ptr` in cache while there
     // are references.
-    *ptr = MakeSharedCacheHandleGuard<CacheEntryStatsCollector>(cache, h);
+    *ptr = cache.SharedGuard(h);
     return Status::OK();
   }
 
@@ -157,10 +160,6 @@ class CacheEntryStatsCollector {
         cache_(cache),
         clock_(clock) {}
 
-  static void Deleter(const Slice &, void *value) {
-    delete static_cast<CacheEntryStatsCollector *>(value);
-  }
-
   static const Slice &GetCacheKey() {
     // For each template instantiation
     static CacheKey ckey = CacheKey::CreateUniqueForProcessLifetime();

cache/cache_helpers.cc

@@ -0,0 +1,40 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "cache/cache_helpers.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+void ReleaseCacheHandleCleanup(void* arg1, void* arg2) {
+  Cache* const cache = static_cast<Cache*>(arg1);
+  assert(cache);
+
+  Cache::Handle* const cache_handle = static_cast<Cache::Handle*>(arg2);
+  assert(cache_handle);
+
+  cache->Release(cache_handle);
+}
+
+Status WarmInCache(Cache* cache, const Slice& key, const Slice& saved,
+                   Cache::CreateContext* create_context,
+                   const Cache::CacheItemHelper* helper,
+                   Cache::Priority priority, size_t* out_charge) {
+  assert(helper);
+  assert(helper->create_cb);
+  Cache::ObjectPtr value;
+  size_t charge;
+  Status st = helper->create_cb(saved, create_context,
+                                cache->memory_allocator(), &value, &charge);
+  if (st.ok()) {
+    st =
+        cache->Insert(key, value, helper, charge, /*handle*/ nullptr, priority);
+    if (out_charge) {
+      *out_charge = charge;
+    }
+  }
+  return st;
+}
+
+}  // namespace ROCKSDB_NAMESPACE

cache/cache_helpers.h

@@ -17,22 +17,17 @@ template <typename T>
 T* GetFromCacheHandle(Cache* cache, Cache::Handle* handle) {
   assert(cache);
   assert(handle);
-
   return static_cast<T*>(cache->Value(handle));
 }
 
-// Simple generic deleter for Cache (to be used with Cache::Insert).
-template <typename T>
-void DeleteCacheEntry(const Slice& /* key */, void* value) {
-  delete static_cast<T*>(value);
-}
-
 // Turns a T* into a Slice so it can be used as a key with Cache.
 template <typename T>
-Slice GetSlice(const T* t) {
+Slice GetSliceForKey(const T* t) {
   return Slice(reinterpret_cast<const char*>(t), sizeof(T));
 }
 
+void ReleaseCacheHandleCleanup(void* arg1, void* arg2);
+
 // Generic resource management object for cache handles that releases the handle
 // when destroyed. Has unique ownership of the handle, so copying it is not
 // allowed, while moving it transfers ownership.
@@ -88,7 +83,7 @@ class CacheHandleGuard {
     if (cleanable) {
       if (handle_ != nullptr) {
         assert(cache_);
-        cleanable->RegisterCleanup(&ReleaseCacheHandle, cache_, handle_);
+        cleanable->RegisterCleanup(&ReleaseCacheHandleCleanup, cache_, handle_);
       }
     }
     ResetFields();
@@ -115,16 +110,6 @@ class CacheHandleGuard {
     value_ = nullptr;
   }
 
-  static void ReleaseCacheHandle(void* arg1, void* arg2) {
-    Cache* const cache = static_cast<Cache*>(arg1);
-    assert(cache);
-
-    Cache::Handle* const cache_handle = static_cast<Cache::Handle*>(arg2);
-    assert(cache_handle);
-
-    cache->Release(cache_handle);
-  }
-
  private:
   Cache* cache_ = nullptr;
   Cache::Handle* handle_ = nullptr;
@@ -139,7 +124,16 @@ template <typename T>
 std::shared_ptr<T> MakeSharedCacheHandleGuard(Cache* cache,
                                               Cache::Handle* handle) {
   auto wrapper = std::make_shared<CacheHandleGuard<T>>(cache, handle);
-  return std::shared_ptr<T>(wrapper, static_cast<T*>(cache->Value(handle)));
+  return std::shared_ptr<T>(wrapper, GetFromCacheHandle<T>(cache, handle));
 }
 
+// Given the persistable data (saved) for a block cache entry, parse that
+// into a cache entry object and insert it into the given cache. The charge
+// of the new entry can be returned to the caller through `out_charge`.
+Status WarmInCache(Cache* cache, const Slice& key, const Slice& saved,
+                   Cache::CreateContext* create_context,
+                   const Cache::CacheItemHelper* helper,
+                   Cache::Priority priority = Cache::Priority::LOW,
+                   size_t* out_charge = nullptr);
+
 }  // namespace ROCKSDB_NAMESPACE

cache/cache_reservation_manager.cc

@@ -13,7 +13,6 @@
 #include <cstring>
 #include <memory>
 
-#include "cache/cache_entry_roles.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"
@@ -41,17 +40,17 @@ CacheReservationManagerImpl<
 template <CacheEntryRole R>
 CacheReservationManagerImpl<R>::CacheReservationManagerImpl(
     std::shared_ptr<Cache> cache, bool delayed_decrease)
-    : delayed_decrease_(delayed_decrease),
+    : cache_(cache),
+      delayed_decrease_(delayed_decrease),
       cache_allocated_size_(0),
       memory_used_(0) {
   assert(cache != nullptr);
-  cache_ = cache;
 }
 
 template <CacheEntryRole R>
 CacheReservationManagerImpl<R>::~CacheReservationManagerImpl() {
   for (auto* handle : dummy_handles_) {
-    cache_->Release(handle, true);
+    cache_.ReleaseAndEraseIfLastRef(handle);
   }
 }
 
@@ -115,8 +114,7 @@ Status CacheReservationManagerImpl<R>::IncreaseCacheReservation(
   Status return_status = Status::OK();
   while (new_mem_used > cache_allocated_size_.load(std::memory_order_relaxed)) {
     Cache::Handle* handle = nullptr;
-    return_status = cache_->Insert(GetNextCacheKey(), nullptr, kSizeDummyEntry,
-                                   GetNoopDeleterForRole<R>(), &handle);
+    return_status = cache_.Insert(GetNextCacheKey(), kSizeDummyEntry, &handle);
 
     if (return_status != Status::OK()) {
       return return_status;
@@ -141,7 +139,7 @@ Status CacheReservationManagerImpl<R>::DecreaseCacheReservation(
          cache_allocated_size_.load(std::memory_order_relaxed)) {
     assert(!dummy_handles_.empty());
     auto* handle = dummy_handles_.back();
-    cache_->Release(handle, true);
+    cache_.ReleaseAndEraseIfLastRef(handle);
     dummy_handles_.pop_back();
     cache_allocated_size_ -= kSizeDummyEntry;
   }
@@ -169,8 +167,9 @@ Slice CacheReservationManagerImpl<R>::GetNextCacheKey() {
 }
 
 template <CacheEntryRole R>
-Cache::DeleterFn CacheReservationManagerImpl<R>::TEST_GetNoopDeleterForRole() {
-  return GetNoopDeleterForRole<R>();
+const Cache::CacheItemHelper*
+CacheReservationManagerImpl<R>::TEST_GetCacheItemHelperForRole() {
+  return &CacheInterface::kHelper;
 }
 
 template class CacheReservationManagerImpl<

cache/cache_reservation_manager.h

@@ -18,7 +18,7 @@
 
 #include "cache/cache_entry_roles.h"
 #include "cache/cache_key.h"
-#include "rocksdb/cache.h"
+#include "cache/typed_cache.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"
 #include "util/coding.h"
@@ -197,10 +197,10 @@ class CacheReservationManagerImpl
 
   static constexpr std::size_t GetDummyEntrySize() { return kSizeDummyEntry; }
 
-  // For testing only - it is to help ensure the NoopDeleterForRole<R>
+  // For testing only - it is to help ensure the CacheItemHelperForRole<R>
   // accessed from CacheReservationManagerImpl and the one accessed from the
   // test are from the same translation units
-  static Cache::DeleterFn TEST_GetNoopDeleterForRole();
+  static const Cache::CacheItemHelper *TEST_GetCacheItemHelperForRole();
 
  private:
   static constexpr std::size_t kSizeDummyEntry = 256 * 1024;
@@ -211,7 +211,8 @@ class CacheReservationManagerImpl
   Status IncreaseCacheReservation(std::size_t new_mem_used);
   Status DecreaseCacheReservation(std::size_t new_mem_used);
 
-  std::shared_ptr<Cache> cache_;
+  using CacheInterface = PlaceholderSharedCacheInterface<R>;
+  CacheInterface cache_;
   bool delayed_decrease_;
   std::atomic<std::size_t> cache_allocated_size_;
   std::size_t memory_used_;

cache/cache_test.cc

@@ -16,6 +16,7 @@
 #include <vector>
 
 #include "cache/lru_cache.h"
+#include "cache/typed_cache.h"
 #include "port/stack_trace.h"
 #include "test_util/testharness.h"
 #include "util/coding.h"
@@ -55,23 +56,31 @@ int DecodeKey32Bits(const Slice& k) {
   return DecodeFixed32(k.data());
 }
 
-void* EncodeValue(uintptr_t v) { return reinterpret_cast<void*>(v); }
+Cache::ObjectPtr EncodeValue(uintptr_t v) {
+  return reinterpret_cast<Cache::ObjectPtr>(v);
+}
 
 int DecodeValue(void* v) {
   return static_cast<int>(reinterpret_cast<uintptr_t>(v));
 }
 
-void DumbDeleter(const Slice& /*key*/, void* /*value*/) {}
+const Cache::CacheItemHelper kDumbHelper{
+    CacheEntryRole::kMisc,
+    [](Cache::ObjectPtr /*value*/, MemoryAllocator* /*alloc*/) {}};
 
-void EraseDeleter1(const Slice& /*key*/, void* value) {
-  Cache* cache = reinterpret_cast<Cache*>(value);
-  cache->Erase("foo");
-}
+const Cache::CacheItemHelper kEraseOnDeleteHelper1{
+    CacheEntryRole::kMisc,
+    [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) {
+      Cache* cache = static_cast<Cache*>(value);
+      cache->Erase("foo");
+    }};
 
-void EraseDeleter2(const Slice& /*key*/, void* value) {
-  Cache* cache = reinterpret_cast<Cache*>(value);
-  cache->Erase(EncodeKey16Bytes(1234));
-}
+const Cache::CacheItemHelper kEraseOnDeleteHelper2{
+    CacheEntryRole::kMisc,
+    [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) {
+      Cache* cache = static_cast<Cache*>(value);
+      cache->Erase(EncodeKey16Bytes(1234));
+    }};
 
 const std::string kLRU = "lru";
 const std::string kHyperClock = "hyper_clock";
@@ -83,14 +92,11 @@ class CacheTest : public testing::TestWithParam<std::string> {
   static CacheTest* current_;
   static std::string type_;
 
-  static void Deleter(const Slice& key, void* v) {
-    if (type_ == kHyperClock) {
-      current_->deleted_keys_.push_back(DecodeKey16Bytes(key));
-    } else {
-      current_->deleted_keys_.push_back(DecodeKey32Bits(key));
-    }
+  static void Deleter(Cache::ObjectPtr v, MemoryAllocator*) {
     current_->deleted_values_.push_back(DecodeValue(v));
   }
+  static constexpr Cache::CacheItemHelper kHelper{CacheEntryRole::kMisc,
+                                                  &Deleter};
 
   static const int kCacheSize = 1000;
   static const int kNumShardBits = 4;
@@ -98,7 +104,6 @@ class CacheTest : public testing::TestWithParam<std::string> {
   static const int kCacheSize2 = 100;
   static const int kNumShardBits2 = 2;
 
-  std::vector<int> deleted_keys_;
   std::vector<int> deleted_values_;
   std::shared_ptr<Cache> cache_;
   std::shared_ptr<Cache> cache2_;
@@ -182,8 +187,8 @@ class CacheTest : public testing::TestWithParam<std::string> {
 
   void Insert(std::shared_ptr<Cache> cache, int key, int value,
               int charge = 1) {
-    EXPECT_OK(cache->Insert(EncodeKey(key), EncodeValue(value), charge,
-                            &CacheTest::Deleter));
+    EXPECT_OK(
+        cache->Insert(EncodeKey(key), EncodeValue(value), &kHelper, charge));
   }
 
   void Erase(std::shared_ptr<Cache> cache, int key) {
@@ -236,10 +241,8 @@ TEST_P(CacheTest, UsageTest) {
       key = EncodeKey(i);
     }
     auto kv_size = key.size() + 5;
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
-                            DumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    kv_size, DumbDeleter));
+    ASSERT_OK(cache->Insert(key, value, &kDumbHelper, kv_size));
+    ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, kv_size));
     usage += kv_size;
     ASSERT_EQ(usage, cache->GetUsage());
     if (type == kHyperClock) {
@@ -262,10 +265,8 @@ TEST_P(CacheTest, UsageTest) {
     } else {
       key = EncodeKey(static_cast<int>(1000 + i));
     }
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
-                            DumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    key.size() + 5, DumbDeleter));
+    ASSERT_OK(cache->Insert(key, value, &kDumbHelper, key.size() + 5));
+    ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, key.size() + 5));
   }
 
   // the usage should be close to the capacity
@@ -320,11 +321,9 @@ TEST_P(CacheTest, PinnedUsageTest) {
     auto kv_size = key.size() + 5;
     Cache::Handle* handle;
     Cache::Handle* handle_in_precise_cache;
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
-                            DumbDeleter, &handle));
+    ASSERT_OK(cache->Insert(key, value, &kDumbHelper, kv_size, &handle));
     assert(handle);
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    kv_size, DumbDeleter,
+    ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, kv_size,
                                     &handle_in_precise_cache));
     assert(handle_in_precise_cache);
     pinned_usage += kv_size;
@@ -365,10 +364,8 @@ TEST_P(CacheTest, PinnedUsageTest) {
     } else {
       key = EncodeKey(static_cast<int>(1000 + i));
     }
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
-                            DumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    key.size() + 5, DumbDeleter));
+    ASSERT_OK(cache->Insert(key, value, &kDumbHelper, key.size() + 5));
+    ASSERT_OK(precise_cache->Insert(key, value, &kDumbHelper, key.size() + 5));
   }
   ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
   ASSERT_EQ(precise_cache_pinned_usage, precise_cache->GetPinnedUsage());
@@ -416,8 +413,7 @@ TEST_P(CacheTest, HitAndMiss) {
   ASSERT_EQ(201, Lookup(200));
   ASSERT_EQ(-1, Lookup(300));
 
-  ASSERT_EQ(1U, deleted_keys_.size());
-  ASSERT_EQ(100, deleted_keys_[0]);
+  ASSERT_EQ(1U, deleted_values_.size());
   if (GetParam() == kHyperClock) {
     ASSERT_EQ(102, deleted_values_[0]);
   } else {
@@ -438,21 +434,20 @@ TEST_P(CacheTest, InsertSameKey) {
 
 TEST_P(CacheTest, Erase) {
   Erase(200);
-  ASSERT_EQ(0U, deleted_keys_.size());
+  ASSERT_EQ(0U, deleted_values_.size());
 
   Insert(100, 101);
   Insert(200, 201);
   Erase(100);
   ASSERT_EQ(-1, Lookup(100));
   ASSERT_EQ(201, Lookup(200));
-  ASSERT_EQ(1U, deleted_keys_.size());
-  ASSERT_EQ(100, deleted_keys_[0]);
+  ASSERT_EQ(1U, deleted_values_.size());
   ASSERT_EQ(101, deleted_values_[0]);
 
   Erase(100);
   ASSERT_EQ(-1, Lookup(100));
   ASSERT_EQ(201, Lookup(200));
-  ASSERT_EQ(1U, deleted_keys_.size());
+  ASSERT_EQ(1U, deleted_values_.size());
 }
 
 TEST_P(CacheTest, EntriesArePinned) {
@@ -469,23 +464,21 @@ TEST_P(CacheTest, EntriesArePinned) {
   Insert(100, 102);
   Cache::Handle* h2 = cache_->Lookup(EncodeKey(100));
   ASSERT_EQ(102, DecodeValue(cache_->Value(h2)));
-  ASSERT_EQ(0U, deleted_keys_.size());
+  ASSERT_EQ(0U, deleted_values_.size());
   ASSERT_EQ(2U, cache_->GetUsage());
 
   cache_->Release(h1);
-  ASSERT_EQ(1U, deleted_keys_.size());
-  ASSERT_EQ(100, deleted_keys_[0]);
+  ASSERT_EQ(1U, deleted_values_.size());
   ASSERT_EQ(101, deleted_values_[0]);
   ASSERT_EQ(1U, cache_->GetUsage());
 
   Erase(100);
   ASSERT_EQ(-1, Lookup(100));
-  ASSERT_EQ(1U, deleted_keys_.size());
+  ASSERT_EQ(1U, deleted_values_.size());
   ASSERT_EQ(1U, cache_->GetUsage());
 
   cache_->Release(h2);
-  ASSERT_EQ(2U, deleted_keys_.size());
-  ASSERT_EQ(100, deleted_keys_[1]);
+  ASSERT_EQ(2U, deleted_values_.size());
   ASSERT_EQ(102, deleted_values_[1]);
   ASSERT_EQ(0U, cache_->GetUsage());
 }
@@ -588,9 +581,9 @@ TEST_P(CacheTest, EvictEmptyCache) {
   // Insert item large than capacity to trigger eviction on empty cache.
   auto cache = NewCache(1, 0, false);
   if (type == kLRU) {
-    ASSERT_OK(cache->Insert("foo", nullptr, 10, DumbDeleter));
+    ASSERT_OK(cache->Insert("foo", nullptr, &kDumbHelper, 10));
   } else {
-    ASSERT_OK(cache->Insert(EncodeKey(1000), nullptr, 10, DumbDeleter));
+    ASSERT_OK(cache->Insert(EncodeKey(1000), nullptr, &kDumbHelper, 10));
   }
 }
 
@@ -601,19 +594,19 @@ TEST_P(CacheTest, EraseFromDeleter) {
   // the cache at that point.
   std::shared_ptr<Cache> cache = NewCache(10, 0, false);
   std::string foo, bar;
-  Cache::DeleterFn erase_deleter;
+  const Cache::CacheItemHelper* erase_helper;
   if (type == kLRU) {
     foo = "foo";
     bar = "bar";
-    erase_deleter = EraseDeleter1;
+    erase_helper = &kEraseOnDeleteHelper1;
   } else {
     foo = EncodeKey(1234);
     bar = EncodeKey(5678);
-    erase_deleter = EraseDeleter2;
+    erase_helper = &kEraseOnDeleteHelper2;
   }
 
-  ASSERT_OK(cache->Insert(foo, nullptr, 1, DumbDeleter));
-  ASSERT_OK(cache->Insert(bar, cache.get(), 1, erase_deleter));
+  ASSERT_OK(cache->Insert(foo, nullptr, &kDumbHelper, 1));
+  ASSERT_OK(cache->Insert(bar, cache.get(), erase_helper, 1));
 
   cache->Erase(bar);
   ASSERT_EQ(nullptr, cache->Lookup(foo));
@@ -675,50 +668,51 @@ TEST_P(CacheTest, NewId) {
   ASSERT_NE(a, b);
 }
 
-class Value {
- public:
-  explicit Value(int v) : v_(v) {}
-
-  int v_;
-};
-
-namespace {
-void deleter(const Slice& /*key*/, void* value) {
-  delete static_cast<Value*>(value);
-}
-}  // namespace
-
 TEST_P(CacheTest, ReleaseAndErase) {
   std::shared_ptr<Cache> cache = NewCache(5, 0, false);
   Cache::Handle* handle;
-  Status s = cache->Insert(EncodeKey(100), EncodeValue(100), 1,
-                           &CacheTest::Deleter, &handle);
+  Status s =
+      cache->Insert(EncodeKey(100), EncodeValue(100), &kHelper, 1, &handle);
   ASSERT_TRUE(s.ok());
   ASSERT_EQ(5U, cache->GetCapacity());
   ASSERT_EQ(1U, cache->GetUsage());
-  ASSERT_EQ(0U, deleted_keys_.size());
+  ASSERT_EQ(0U, deleted_values_.size());
   auto erased = cache->Release(handle, true);
   ASSERT_TRUE(erased);
   // This tests that deleter has been called
-  ASSERT_EQ(1U, deleted_keys_.size());
+  ASSERT_EQ(1U, deleted_values_.size());
 }
 
 TEST_P(CacheTest, ReleaseWithoutErase) {
   std::shared_ptr<Cache> cache = NewCache(5, 0, false);
   Cache::Handle* handle;
-  Status s = cache->Insert(EncodeKey(100), EncodeValue(100), 1,
-                           &CacheTest::Deleter, &handle);
+  Status s =
+      cache->Insert(EncodeKey(100), EncodeValue(100), &kHelper, 1, &handle);
   ASSERT_TRUE(s.ok());
   ASSERT_EQ(5U, cache->GetCapacity());
   ASSERT_EQ(1U, cache->GetUsage());
-  ASSERT_EQ(0U, deleted_keys_.size());
+  ASSERT_EQ(0U, deleted_values_.size());
   auto erased = cache->Release(handle);
   ASSERT_FALSE(erased);
   // This tests that deleter is not called. When cache has free capacity it is
   // not expected to immediately erase the released items.
-  ASSERT_EQ(0U, deleted_keys_.size());
+  ASSERT_EQ(0U, deleted_values_.size());
 }
 
+namespace {
+class Value {
+ public:
+  explicit Value(int v) : v_(v) {}
+
+  int v_;
+
+  static constexpr auto kCacheEntryRole = CacheEntryRole::kMisc;
+};
+
+using SharedCache = BasicTypedSharedCacheInterface<Value>;
+using TypedHandle = SharedCache::TypedHandle;
+}  // namespace
+
 TEST_P(CacheTest, SetCapacity) {
   auto type = GetParam();
   if (type == kHyperClock) {
@@ -731,19 +725,19 @@ TEST_P(CacheTest, SetCapacity) {
   // lets create a cache with capacity 5,
   // then, insert 5 elements, then increase capacity
   // to 10, returned capacity should be 10, usage=5
-  std::shared_ptr<Cache> cache = NewCache(5, 0, false);
-  std::vector<Cache::Handle*> handles(10);
+  SharedCache cache{NewCache(5, 0, false)};
+  std::vector<TypedHandle*> handles(10);
   // Insert 5 entries, but not releasing.
   for (int i = 0; i < 5; i++) {
     std::string key = EncodeKey(i + 1);
-    Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
+    Status s = cache.Insert(key, new Value(i + 1), 1, &handles[i]);
     ASSERT_TRUE(s.ok());
   }
-  ASSERT_EQ(5U, cache->GetCapacity());
-  ASSERT_EQ(5U, cache->GetUsage());
-  cache->SetCapacity(10);
-  ASSERT_EQ(10U, cache->GetCapacity());
-  ASSERT_EQ(5U, cache->GetUsage());
+  ASSERT_EQ(5U, cache.get()->GetCapacity());
+  ASSERT_EQ(5U, cache.get()->GetUsage());
+  cache.get()->SetCapacity(10);
+  ASSERT_EQ(10U, cache.get()->GetCapacity());
+  ASSERT_EQ(5U, cache.get()->GetUsage());
 
   // test2: decrease capacity
   // insert 5 more elements to cache, then release 5,
@@ -751,77 +745,77 @@ TEST_P(CacheTest, SetCapacity) {
   // and usage should be 7
   for (int i = 5; i < 10; i++) {
     std::string key = EncodeKey(i + 1);
-    Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
+    Status s = cache.Insert(key, new Value(i + 1), 1, &handles[i]);
     ASSERT_TRUE(s.ok());
   }
-  ASSERT_EQ(10U, cache->GetCapacity());
-  ASSERT_EQ(10U, cache->GetUsage());
+  ASSERT_EQ(10U, cache.get()->GetCapacity());
+  ASSERT_EQ(10U, cache.get()->GetUsage());
   for (int i = 0; i < 5; i++) {
-    cache->Release(handles[i]);
+    cache.Release(handles[i]);
   }
-  ASSERT_EQ(10U, cache->GetCapacity());
-  ASSERT_EQ(10U, cache->GetUsage());
-  cache->SetCapacity(7);
-  ASSERT_EQ(7, cache->GetCapacity());
-  ASSERT_EQ(7, cache->GetUsage());
+  ASSERT_EQ(10U, cache.get()->GetCapacity());
+  ASSERT_EQ(10U, cache.get()->GetUsage());
+  cache.get()->SetCapacity(7);
+  ASSERT_EQ(7, cache.get()->GetCapacity());
+  ASSERT_EQ(7, cache.get()->GetUsage());
 
   // release remaining 5 to keep valgrind happy
   for (int i = 5; i < 10; i++) {
-    cache->Release(handles[i]);
+    cache.Release(handles[i]);
   }
 
   // Make sure this doesn't crash or upset ASAN/valgrind
-  cache->DisownData();
+  cache.get()->DisownData();
 }
 
 TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
   // test1: set the flag to false. Insert more keys than capacity. See if they
   // all go through.
-  std::shared_ptr<Cache> cache = NewCache(5, 0, false);
-  std::vector<Cache::Handle*> handles(10);
+  SharedCache cache{NewCache(5, 0, false)};
+  std::vector<TypedHandle*> handles(10);
   Status s;
   for (int i = 0; i < 10; i++) {
     std::string key = EncodeKey(i + 1);
-    s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
+    s = cache.Insert(key, new Value(i + 1), 1, &handles[i]);
     ASSERT_OK(s);
     ASSERT_NE(nullptr, handles[i]);
   }
-  ASSERT_EQ(10, cache->GetUsage());
+  ASSERT_EQ(10, cache.get()->GetUsage());
 
   // test2: set the flag to true. Insert and check if it fails.
   std::string extra_key = EncodeKey(100);
   Value* extra_value = new Value(0);
-  cache->SetStrictCapacityLimit(true);
-  Cache::Handle* handle;
-  s = cache->Insert(extra_key, extra_value, 1, &deleter, &handle);
+  cache.get()->SetStrictCapacityLimit(true);
+  TypedHandle* handle;
+  s = cache.Insert(extra_key, extra_value, 1, &handle);
   ASSERT_TRUE(s.IsMemoryLimit());
   ASSERT_EQ(nullptr, handle);
-  ASSERT_EQ(10, cache->GetUsage());
+  ASSERT_EQ(10, cache.get()->GetUsage());
 
   for (int i = 0; i < 10; i++) {
-    cache->Release(handles[i]);
+    cache.Release(handles[i]);
   }
 
   // test3: init with flag being true.
-  std::shared_ptr<Cache> cache2 = NewCache(5, 0, true);
+  SharedCache cache2{NewCache(5, 0, true)};
   for (int i = 0; i < 5; i++) {
     std::string key = EncodeKey(i + 1);
-    s = cache2->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
+    s = cache2.Insert(key, new Value(i + 1), 1, &handles[i]);
     ASSERT_OK(s);
     ASSERT_NE(nullptr, handles[i]);
   }
-  s = cache2->Insert(extra_key, extra_value, 1, &deleter, &handle);
+  s = cache2.Insert(extra_key, extra_value, 1, &handle);
   ASSERT_TRUE(s.IsMemoryLimit());
   ASSERT_EQ(nullptr, handle);
   // test insert without handle
-  s = cache2->Insert(extra_key, extra_value, 1, &deleter);
+  s = cache2.Insert(extra_key, extra_value, 1);
   // AS if the key have been inserted into cache but get evicted immediately.
   ASSERT_OK(s);
-  ASSERT_EQ(5, cache2->GetUsage());
-  ASSERT_EQ(nullptr, cache2->Lookup(extra_key));
+  ASSERT_EQ(5, cache2.get()->GetUsage());
+  ASSERT_EQ(nullptr, cache2.Lookup(extra_key));
 
   for (int i = 0; i < 5; i++) {
-    cache2->Release(handles[i]);
+    cache2.Release(handles[i]);
   }
 }
 
@@ -829,55 +823,54 @@ TEST_P(CacheTest, OverCapacity) {
   size_t n = 10;
 
   // a LRUCache with n entries and one shard only
-  std::shared_ptr<Cache> cache = NewCache(n, 0, false);
-
-  std::vector<Cache::Handle*> handles(n + 1);
+  SharedCache cache{NewCache(n, 0, false)};
+  std::vector<TypedHandle*> handles(n + 1);
 
   // Insert n+1 entries, but not releasing.
   for (int i = 0; i < static_cast<int>(n + 1); i++) {
     std::string key = EncodeKey(i + 1);
-    Status s = cache->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
+    Status s = cache.Insert(key, new Value(i + 1), 1, &handles[i]);
     ASSERT_TRUE(s.ok());
   }
 
   // Guess what's in the cache now?
   for (int i = 0; i < static_cast<int>(n + 1); i++) {
     std::string key = EncodeKey(i + 1);
-    auto h = cache->Lookup(key);
+    auto h = cache.Lookup(key);
     ASSERT_TRUE(h != nullptr);
-    if (h) cache->Release(h);
+    if (h) cache.Release(h);
   }
 
   // the cache is over capacity since nothing could be evicted
-  ASSERT_EQ(n + 1U, cache->GetUsage());
+  ASSERT_EQ(n + 1U, cache.get()->GetUsage());
   for (int i = 0; i < static_cast<int>(n + 1); i++) {
-    cache->Release(handles[i]);
+    cache.Release(handles[i]);
   }
 
   if (GetParam() == kHyperClock) {
     // Make sure eviction is triggered.
-    ASSERT_OK(cache->Insert(EncodeKey(-1), nullptr, 1, &deleter, &handles[0]));
+    ASSERT_OK(cache.Insert(EncodeKey(-1), nullptr, 1, &handles[0]));
 
     // cache is under capacity now since elements were released
-    ASSERT_GE(n, cache->GetUsage());
+    ASSERT_GE(n, cache.get()->GetUsage());
 
     // clean up
-    cache->Release(handles[0]);
+    cache.Release(handles[0]);
   } else {
     // LRUCache checks for over-capacity in Release.
 
     // cache is exactly at capacity now with minimal eviction
-    ASSERT_EQ(n, cache->GetUsage());
+    ASSERT_EQ(n, cache.get()->GetUsage());
 
     // element 0 is evicted and the rest is there
     // This is consistent with the LRU policy since the element 0
     // was released first
     for (int i = 0; i < static_cast<int>(n + 1); i++) {
       std::string key = EncodeKey(i + 1);
-      auto h = cache->Lookup(key);
+      auto h = cache.Lookup(key);
       if (h) {
         ASSERT_NE(static_cast<size_t>(i), 0U);
-        cache->Release(h);
+        cache.Release(h);
       } else {
         ASSERT_EQ(static_cast<size_t>(i), 0U);
       }
@@ -885,40 +878,15 @@ TEST_P(CacheTest, OverCapacity) {
   }
 }
 
-namespace {
-std::vector<std::pair<int, int>> legacy_callback_state;
-void legacy_callback(void* value, size_t charge) {
-  legacy_callback_state.push_back(
-      {DecodeValue(value), static_cast<int>(charge)});
-}
-};  // namespace
-
-TEST_P(CacheTest, ApplyToAllCacheEntriesTest) {
-  std::vector<std::pair<int, int>> inserted;
-  legacy_callback_state.clear();
-
-  for (int i = 0; i < 10; ++i) {
-    Insert(i, i * 2, i + 1);
-    inserted.push_back({i * 2, i + 1});
-  }
-  cache_->ApplyToAllCacheEntries(legacy_callback, true);
-
-  std::sort(inserted.begin(), inserted.end());
-  std::sort(legacy_callback_state.begin(), legacy_callback_state.end());
-  ASSERT_EQ(inserted.size(), legacy_callback_state.size());
-  for (int i = 0; i < static_cast<int>(inserted.size()); ++i) {
-    EXPECT_EQ(inserted[i], legacy_callback_state[i]);
-  }
-}
-
 TEST_P(CacheTest, ApplyToAllEntriesTest) {
   std::vector<std::string> callback_state;
-  const auto callback = [&](const Slice& key, void* value, size_t charge,
-                            Cache::DeleterFn deleter) {
+  const auto callback = [&](const Slice& key, Cache::ObjectPtr value,
+                            size_t charge,
+                            const Cache::CacheItemHelper* helper) {
     callback_state.push_back(std::to_string(DecodeKey(key)) + "," +
                              std::to_string(DecodeValue(value)) + "," +
                              std::to_string(charge));
-    assert(deleter == &CacheTest::Deleter);
+    assert(helper == &CacheTest::kHelper);
   };
 
   std::vector<std::string> inserted;
@@ -957,8 +925,8 @@ TEST_P(CacheTest, ApplyToAllEntriesDuringResize) {
 
   // For callback
   int special_count = 0;
-  const auto callback = [&](const Slice&, void*, size_t charge,
-                            Cache::DeleterFn) {
+  const auto callback = [&](const Slice&, Cache::ObjectPtr, size_t charge,
+                            const Cache::CacheItemHelper*) {
     if (charge == static_cast<size_t>(kSpecialCharge)) {
       ++special_count;
     }
@@ -1020,7 +988,7 @@ TEST_P(CacheTest, GetChargeAndDeleter) {
   Cache::Handle* h1 = cache_->Lookup(EncodeKey(1));
   ASSERT_EQ(2, DecodeValue(cache_->Value(h1)));
   ASSERT_EQ(1, cache_->GetCharge(h1));
-  ASSERT_EQ(&CacheTest::Deleter, cache_->GetDeleter(h1));
+  ASSERT_EQ(&CacheTest::kHelper, cache_->GetCacheItemHelper(h1));
   cache_->Release(h1);
 }
 

cache/charged_cache.cc

@@ -17,25 +17,10 @@ ChargedCache::ChargedCache(std::shared_ptr<Cache> cache,
               CacheReservationManagerImpl<CacheEntryRole::kBlobCache>>(
               block_cache))) {}
 
-Status ChargedCache::Insert(const Slice& key, void* value, size_t charge,
-                            DeleterFn deleter, Handle** handle,
-                            Priority priority) {
-  Status s = cache_->Insert(key, value, charge, deleter, handle, priority);
-  if (s.ok()) {
-    // Insert may cause the cache entry eviction if the cache is full. So we
-    // directly call the reservation manager to update the total memory used
-    // in the cache.
-    assert(cache_res_mgr_);
-    cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
-        .PermitUncheckedError();
-  }
-  return s;
-}
-
-Status ChargedCache::Insert(const Slice& key, void* value,
+Status ChargedCache::Insert(const Slice& key, ObjectPtr obj,
                             const CacheItemHelper* helper, size_t charge,
                             Handle** handle, Priority priority) {
-  Status s = cache_->Insert(key, value, helper, charge, handle, priority);
+  Status s = cache_->Insert(key, obj, helper, charge, handle, priority);
   if (s.ok()) {
     // Insert may cause the cache entry eviction if the cache is full. So we
     // directly call the reservation manager to update the total memory used
@@ -47,22 +32,21 @@ Status ChargedCache::Insert(const Slice& key, void* value,
   return s;
 }
 
-Cache::Handle* ChargedCache::Lookup(const Slice& key, Statistics* stats) {
-  return cache_->Lookup(key, stats);
-}
-
 Cache::Handle* ChargedCache::Lookup(const Slice& key,
                                     const CacheItemHelper* helper,
-                                    const CreateCallback& create_cb,
+                                    CreateContext* create_context,
                                     Priority priority, bool wait,
                                     Statistics* stats) {
-  auto handle = cache_->Lookup(key, helper, create_cb, priority, wait, stats);
+  auto handle =
+      cache_->Lookup(key, helper, create_context, priority, wait, stats);
   // Lookup may promote the KV pair from the secondary cache to the primary
   // cache. So we directly call the reservation manager to update the total
   // memory used in the cache.
-  assert(cache_res_mgr_);
-  cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
-      .PermitUncheckedError();
+  if (helper && helper->create_cb) {
+    assert(cache_res_mgr_);
+    cache_res_mgr_->UpdateCacheReservation(cache_->GetUsage())
+        .PermitUncheckedError();
+  }
   return handle;
 }
 

cache/charged_cache.h

@@ -23,16 +23,14 @@ class ChargedCache : public Cache {
                std::shared_ptr<Cache> block_cache);
   ~ChargedCache() override = default;
 
-  Status Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter,
-                Handle** handle, Priority priority) override;
-  Status Insert(const Slice& key, void* value, const CacheItemHelper* helper,
+  Status Insert(const Slice& key, ObjectPtr obj, const CacheItemHelper* helper,
                 size_t charge, Handle** handle = nullptr,
                 Priority priority = Priority::LOW) override;
 
-  Cache::Handle* Lookup(const Slice& key, Statistics* stats) override;
   Cache::Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
-                        const CreateCallback& create_cb, Priority priority,
-                        bool wait, Statistics* stats = nullptr) override;
+                        CreateContext* create_context,
+                        Priority priority = Priority::LOW, bool wait = true,
+                        Statistics* stats = nullptr) override;
 
   bool Release(Cache::Handle* handle, bool useful,
                bool erase_if_last_ref = false) override;
@@ -56,7 +54,9 @@ class ChargedCache : public Cache {
     return cache_->HasStrictCapacityLimit();
   }
 
-  void* Value(Cache::Handle* handle) override { return cache_->Value(handle); }
+  ObjectPtr Value(Cache::Handle* handle) override {
+    return cache_->Value(handle);
+  }
 
   bool IsReady(Cache::Handle* handle) override {
     return cache_->IsReady(handle);
@@ -84,22 +84,17 @@ class ChargedCache : public Cache {
     return cache_->GetCharge(handle);
   }
 
-  Cache::DeleterFn GetDeleter(Cache::Handle* handle) const override {
-    return cache_->GetDeleter(handle);
+  const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override {
+    return cache_->GetCacheItemHelper(handle);
   }
 
   void ApplyToAllEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               Cache::DeleterFn deleter)>& callback,
+      const std::function<void(const Slice& key, ObjectPtr value, size_t charge,
+                               const CacheItemHelper* helper)>& callback,
       const Cache::ApplyToAllEntriesOptions& opts) override {
     cache_->ApplyToAllEntries(callback, opts);
   }
 
-  void ApplyToAllCacheEntries(void (*callback)(void* value, size_t charge),
-                              bool thread_safe) override {
-    cache_->ApplyToAllCacheEntries(callback, thread_safe);
-  }
-
   std::string GetPrintableOptions() const override {
     return cache_->GetPrintableOptions();
   }

cache/clock_cache.cc

@@ -50,12 +50,12 @@ inline uint64_t GetInitialCountdown(Cache::Priority priority) {
   }
 }
 
-inline void FreeDataMarkEmpty(ClockHandle& h) {
+inline void FreeDataMarkEmpty(ClockHandle& h, MemoryAllocator* allocator) {
   // NOTE: in theory there's more room for parallelism if we copy the handle
   // data and delay actions like this until after marking the entry as empty,
   // but performance tests only show a regression by copying the few words
   // of data.
-  h.FreeData();
+  h.FreeData(allocator);
 
 #ifndef NDEBUG
   // Mark slot as empty, with assertion
@@ -115,24 +115,23 @@ inline bool ClockUpdate(ClockHandle& h) {
 
 }  // namespace
 
-void ClockHandleBasicData::FreeData() const {
-  if (deleter) {
-    UniqueId64x2 unhashed;
-    (*deleter)(
-        ClockCacheShard<HyperClockTable>::ReverseHash(hashed_key, &unhashed),
-        value);
+void ClockHandleBasicData::FreeData(MemoryAllocator* allocator) const {
+  if (helper->del_cb) {
+    helper->del_cb(value, allocator);
   }
 }
 
 HyperClockTable::HyperClockTable(
     size_t capacity, bool /*strict_capacity_limit*/,
-    CacheMetadataChargePolicy metadata_charge_policy, const Opts& opts)
+    CacheMetadataChargePolicy metadata_charge_policy,
+    MemoryAllocator* allocator, const Opts& opts)
     : length_bits_(CalcHashBits(capacity, opts.estimated_value_size,
                                 metadata_charge_policy)),
       length_bits_mask_((size_t{1} << length_bits_) - 1),
       occupancy_limit_(static_cast<size_t>((uint64_t{1} << length_bits_) *
                                            kStrictLoadFactor)),
-      array_(new HandleImpl[size_t{1} << length_bits_]) {
+      array_(new HandleImpl[size_t{1} << length_bits_]),
+      allocator_(allocator) {
   if (metadata_charge_policy ==
       CacheMetadataChargePolicy::kFullChargeCacheMetadata) {
     usage_ += size_t{GetTableSize()} * sizeof(HandleImpl);
@@ -154,7 +153,7 @@ HyperClockTable::~HyperClockTable() {
       case ClockHandle::kStateInvisible:  // rare but possible
       case ClockHandle::kStateVisible:
         assert(GetRefcount(h.meta) == 0);
-        h.FreeData();
+        h.FreeData(allocator_);
 #ifndef NDEBUG
         Rollback(h.hashed_key, &h);
         ReclaimEntryUsage(h.GetTotalCharge());
@@ -415,7 +414,7 @@ Status HyperClockTable::Insert(const ClockHandleBasicData& proto,
       if (handle == nullptr) {
         // Don't insert the entry but still return ok, as if the entry
         // inserted into cache and evicted immediately.
-        proto.FreeData();
+        proto.FreeData(allocator_);
         return Status::OK();
       } else {
         // Need to track usage of fallback detached insert
@@ -556,7 +555,7 @@ Status HyperClockTable::Insert(const ClockHandleBasicData& proto,
     if (handle == nullptr) {
       revert_usage_fn();
       // As if unrefed entry immdiately evicted
-      proto.FreeData();
+      proto.FreeData(allocator_);
       return Status::OK();
     }
   }
@@ -698,14 +697,14 @@ bool HyperClockTable::Release(HandleImpl* h, bool useful,
     // Took ownership
     size_t total_charge = h->GetTotalCharge();
     if (UNLIKELY(h->IsDetached())) {
-      h->FreeData();
+      h->FreeData(allocator_);
       // Delete detached handle
       delete h;
       detached_usage_.fetch_sub(total_charge, std::memory_order_relaxed);
       usage_.fetch_sub(total_charge, std::memory_order_relaxed);
     } else {
       Rollback(h->hashed_key, h);
-      FreeDataMarkEmpty(*h);
+      FreeDataMarkEmpty(*h, allocator_);
       ReclaimEntryUsage(total_charge);
     }
     return true;
@@ -790,7 +789,7 @@ void HyperClockTable::Erase(const UniqueId64x2& hashed_key) {
                 // Took ownership
                 assert(hashed_key == h->hashed_key);
                 size_t total_charge = h->GetTotalCharge();
-                FreeDataMarkEmpty(*h);
+                FreeDataMarkEmpty(*h, allocator_);
                 ReclaimEntryUsage(total_charge);
                 // We already have a copy of hashed_key in this case, so OK to
                 // delay Rollback until after releasing the entry
@@ -878,7 +877,7 @@ void HyperClockTable::EraseUnRefEntries() {
       // Took ownership
       size_t total_charge = h.GetTotalCharge();
       Rollback(h.hashed_key, &h);
-      FreeDataMarkEmpty(h);
+      FreeDataMarkEmpty(h, allocator_);
       ReclaimEntryUsage(total_charge);
     }
   }
@@ -968,7 +967,7 @@ inline void HyperClockTable::Evict(size_t requested_charge,
         Rollback(h.hashed_key, &h);
         *freed_charge += h.GetTotalCharge();
         *freed_count += 1;
-        FreeDataMarkEmpty(h);
+        FreeDataMarkEmpty(h, allocator_);
       }
     }
 
@@ -990,9 +989,10 @@ template <class Table>
 ClockCacheShard<Table>::ClockCacheShard(
     size_t capacity, bool strict_capacity_limit,
     CacheMetadataChargePolicy metadata_charge_policy,
-    const typename Table::Opts& opts)
+    MemoryAllocator* allocator, const typename Table::Opts& opts)
     : CacheShardBase(metadata_charge_policy),
-      table_(capacity, strict_capacity_limit, metadata_charge_policy, opts),
+      table_(capacity, strict_capacity_limit, metadata_charge_policy, allocator,
+             opts),
       capacity_(capacity),
       strict_capacity_limit_(strict_capacity_limit) {
   // Initial charge metadata should not exceed capacity
@@ -1006,8 +1006,9 @@ void ClockCacheShard<Table>::EraseUnRefEntries() {
 
 template <class Table>
 void ClockCacheShard<Table>::ApplyToSomeEntries(
-    const std::function<void(const Slice& key, void* value, size_t charge,
-                             DeleterFn deleter)>& callback,
+    const std::function<void(const Slice& key, Cache::ObjectPtr value,
+                             size_t charge,
+                             const Cache::CacheItemHelper* helper)>& callback,
     size_t average_entries_per_lock, size_t* state) {
   // The state is essentially going to be the starting hash, which works
   // nicely even if we resize between calls because we use upper-most
@@ -1034,7 +1035,7 @@ void ClockCacheShard<Table>::ApplyToSomeEntries(
       [callback](const HandleImpl& h) {
         UniqueId64x2 unhashed;
         callback(ReverseHash(h.hashed_key, &unhashed), h.value,
-                 h.GetTotalCharge(), h.deleter);
+                 h.GetTotalCharge(), h.helper);
       },
       index_begin, index_end, false);
 }
@@ -1078,9 +1079,9 @@ void ClockCacheShard<Table>::SetStrictCapacityLimit(
 template <class Table>
 Status ClockCacheShard<Table>::Insert(const Slice& key,
                                       const UniqueId64x2& hashed_key,
-                                      void* value, size_t charge,
-                                      Cache::DeleterFn deleter,
-                                      HandleImpl** handle,
+                                      Cache::ObjectPtr value,
+                                      const Cache::CacheItemHelper* helper,
+                                      size_t charge, HandleImpl** handle,
                                       Cache::Priority priority) {
   if (UNLIKELY(key.size() != kCacheKeySize)) {
     return Status::NotSupported("ClockCache only supports key size " +
@@ -1089,7 +1090,7 @@ Status ClockCacheShard<Table>::Insert(const Slice& key,
   ClockHandleBasicData proto;
   proto.hashed_key = hashed_key;
   proto.value = value;
-  proto.deleter = deleter;
+  proto.helper = helper;
   proto.total_charge = charge;
   Status s = table_.Insert(
       proto, handle, priority, capacity_.load(std::memory_order_relaxed),
@@ -1223,15 +1224,16 @@ HyperClockCache::HyperClockCache(
   // TODO: should not need to go through two levels of pointer indirection to
   // get to table entries
   size_t per_shard = GetPerShardCapacity();
+  MemoryAllocator* alloc = this->memory_allocator();
   InitShards([=](Shard* cs) {
     HyperClockTable::Opts opts;
     opts.estimated_value_size = estimated_value_size;
-    new (cs)
-        Shard(per_shard, strict_capacity_limit, metadata_charge_policy, opts);
+    new (cs) Shard(per_shard, strict_capacity_limit, metadata_charge_policy,
+                   alloc, opts);
   });
 }
 
-void* HyperClockCache::Value(Handle* handle) {
+Cache::ObjectPtr HyperClockCache::Value(Handle* handle) {
   return reinterpret_cast<const HandleImpl*>(handle)->value;
 }
 
@@ -1239,9 +1241,10 @@ size_t HyperClockCache::GetCharge(Handle* handle) const {
   return reinterpret_cast<const HandleImpl*>(handle)->GetTotalCharge();
 }
 
-Cache::DeleterFn HyperClockCache::GetDeleter(Handle* handle) const {
+const Cache::CacheItemHelper* HyperClockCache::GetCacheItemHelper(
+    Handle* handle) const {
   auto h = reinterpret_cast<const HandleImpl*>(handle);
-  return h->deleter;
+  return h->helper;
 }
 
 namespace {

cache/clock_cache.h

@@ -305,8 +305,8 @@ constexpr double kLoadFactor = 0.7;
 constexpr double kStrictLoadFactor = 0.84;
 
 struct ClockHandleBasicData {
-  void* value = nullptr;
-  Cache::DeleterFn deleter = nullptr;
+  Cache::ObjectPtr value = nullptr;
+  const Cache::CacheItemHelper* helper = nullptr;
   // A lossless, reversible hash of the fixed-size (16 byte) cache key. This
   // eliminates the need to store a hash separately.
   UniqueId64x2 hashed_key = kNullUniqueId64x2;
@@ -321,7 +321,7 @@ struct ClockHandleBasicData {
   inline size_t GetTotalCharge() const { return total_charge; }
 
   // Calls deleter (if non-null) on cache key and value
-  void FreeData() const;
+  void FreeData(MemoryAllocator* allocator) const;
 
   // Required by concept HandleImpl
   const UniqueId64x2& GetHash() const { return hashed_key; }
@@ -411,7 +411,7 @@ class HyperClockTable {
 
   HyperClockTable(size_t capacity, bool strict_capacity_limit,
                   CacheMetadataChargePolicy metadata_charge_policy,
-                  const Opts& opts);
+                  MemoryAllocator* allocator, const Opts& opts);
   ~HyperClockTable();
 
   Status Insert(const ClockHandleBasicData& proto, HandleImpl** handle,
@@ -519,6 +519,8 @@ class HyperClockTable {
   // Updates `detached_usage_` but not `usage_` nor `occupancy_`.
   inline HandleImpl* DetachedInsert(const ClockHandleBasicData& proto);
 
+  MemoryAllocator* GetAllocator() const { return allocator_; }
+
   // Returns the number of bits used to hash an element in the hash
   // table.
   static int CalcHashBits(size_t capacity, size_t estimated_value_size,
@@ -538,6 +540,9 @@ class HyperClockTable {
   // Array of slots comprising the hash table.
   const std::unique_ptr<HandleImpl[]> array_;
 
+  // From Cache, for deleter
+  MemoryAllocator* const allocator_;
+
   // We partition the following members into different cache lines
   // to avoid false sharing among Lookup, Release, Erase and Insert
   // operations in ClockCacheShard.
@@ -563,7 +568,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
  public:
   ClockCacheShard(size_t capacity, bool strict_capacity_limit,
                   CacheMetadataChargePolicy metadata_charge_policy,
-                  const typename Table::Opts& opts);
+                  MemoryAllocator* allocator, const typename Table::Opts& opts);
 
   // For CacheShard concept
   using HandleImpl = typename Table::HandleImpl;
@@ -600,9 +605,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
 
   void SetStrictCapacityLimit(bool strict_capacity_limit);
 
-  Status Insert(const Slice& key, const UniqueId64x2& hashed_key, void* value,
-                size_t charge, Cache::DeleterFn deleter, HandleImpl** handle,
-                Cache::Priority priority);
+  Status Insert(const Slice& key, const UniqueId64x2& hashed_key,
+                Cache::ObjectPtr value, const Cache::CacheItemHelper* helper,
+                size_t charge, HandleImpl** handle, Cache::Priority priority);
 
   HandleImpl* Lookup(const Slice& key, const UniqueId64x2& hashed_key);
 
@@ -629,25 +634,18 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
   size_t GetTableAddressCount() const;
 
   void ApplyToSomeEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
+      const std::function<void(const Slice& key, Cache::ObjectPtr obj,
+                               size_t charge,
+                               const Cache::CacheItemHelper* helper)>& callback,
       size_t average_entries_per_lock, size_t* state);
 
   void EraseUnRefEntries();
 
   std::string GetPrintableOptions() const { return std::string{}; }
 
-  // SecondaryCache not yet supported
-  Status Insert(const Slice& key, const UniqueId64x2& hashed_key, void* value,
-                const Cache::CacheItemHelper* helper, size_t charge,
-                HandleImpl** handle, Cache::Priority priority) {
-    return Insert(key, hashed_key, value, charge, helper->del_cb, handle,
-                  priority);
-  }
-
   HandleImpl* Lookup(const Slice& key, const UniqueId64x2& hashed_key,
                      const Cache::CacheItemHelper* /*helper*/,
-                     const Cache::CreateCallback& /*create_cb*/,
+                     Cache::CreateContext* /*create_context*/,
                      Cache::Priority /*priority*/, bool /*wait*/,
                      Statistics* /*stats*/) {
     return Lookup(key, hashed_key);
@@ -686,11 +684,11 @@ class HyperClockCache
 
   const char* Name() const override { return "HyperClockCache"; }
 
-  void* Value(Handle* handle) override;
+  Cache::ObjectPtr Value(Handle* handle) override;
 
   size_t GetCharge(Handle* handle) const override;
 
-  DeleterFn GetDeleter(Handle* handle) const override;
+  const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override;
 
   void ReportProblems(
       const std::shared_ptr<Logger>& /*info_log*/) const override;

cache/compressed_secondary_cache.cc

@@ -37,8 +37,10 @@ CompressedSecondaryCache::CompressedSecondaryCache(
 CompressedSecondaryCache::~CompressedSecondaryCache() { cache_.reset(); }
 
 std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
-    const Slice& key, const Cache::CreateCallback& create_cb, bool /*wait*/,
-    bool advise_erase, bool& is_in_sec_cache) {
+    const Slice& key, const Cache::CacheItemHelper* helper,
+    Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase,
+    bool& is_in_sec_cache) {
+  assert(helper);
   std::unique_ptr<SecondaryCacheResultHandle> handle;
   is_in_sec_cache = false;
   Cache::Handle* lru_handle = cache_->Lookup(key);
@@ -64,12 +66,14 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
     ptr = reinterpret_cast<CacheAllocationPtr*>(handle_value);
     handle_value_charge = cache_->GetCharge(lru_handle);
   }
+  MemoryAllocator* allocator = cache_options_.memory_allocator.get();
 
   Status s;
-  void* value{nullptr};
+  Cache::ObjectPtr value{nullptr};
   size_t charge{0};
   if (cache_options_.compression_type == kNoCompression) {
-    s = create_cb(ptr->get(), handle_value_charge, &value, &charge);
+    s = helper->create_cb(Slice(ptr->get(), handle_value_charge),
+                          create_context, allocator, &value, &charge);
   } else {
     UncompressionContext uncompression_context(cache_options_.compression_type);
     UncompressionInfo uncompression_info(uncompression_context,
@@ -79,14 +83,14 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
     size_t uncompressed_size{0};
     CacheAllocationPtr uncompressed = UncompressData(
         uncompression_info, (char*)ptr->get(), handle_value_charge,
-        &uncompressed_size, cache_options_.compress_format_version,
-        cache_options_.memory_allocator.get());
+        &uncompressed_size, cache_options_.compress_format_version, allocator);
 
     if (!uncompressed) {
       cache_->Release(lru_handle, /*erase_if_last_ref=*/true);
       return nullptr;
     }
-    s = create_cb(uncompressed.get(), uncompressed_size, &value, &charge);
+    s = helper->create_cb(Slice(uncompressed.get(), uncompressed_size),
+                          create_context, allocator, &value, &charge);
   }
 
   if (!s.ok()) {
@@ -98,8 +102,9 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
     cache_->Release(lru_handle, /*erase_if_last_ref=*/true);
     // Insert a dummy handle.
     cache_
-        ->Insert(key, /*value=*/nullptr, /*charge=*/0,
-                 GetDeletionCallback(cache_options_.enable_custom_split_merge))
+        ->Insert(key, /*obj=*/nullptr,
+                 GetHelper(cache_options_.enable_custom_split_merge),
+                 /*charge=*/0)
         .PermitUncheckedError();
   } else {
     is_in_sec_cache = true;
@@ -109,19 +114,20 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
   return handle;
 }
 
-Status CompressedSecondaryCache::Insert(const Slice& key, void* value,
+Status CompressedSecondaryCache::Insert(const Slice& key,
+                                        Cache::ObjectPtr value,
                                         const Cache::CacheItemHelper* helper) {
   if (value == nullptr) {
     return Status::InvalidArgument();
   }
 
   Cache::Handle* lru_handle = cache_->Lookup(key);
-  Cache::DeleterFn del_cb =
-      GetDeletionCallback(cache_options_.enable_custom_split_merge);
+  auto internal_helper = GetHelper(cache_options_.enable_custom_split_merge);
   if (lru_handle == nullptr) {
     PERF_COUNTER_ADD(compressed_sec_cache_insert_dummy_count, 1);
     // Insert a dummy handle if the handle is evicted for the first time.
-    return cache_->Insert(key, /*value=*/nullptr, /*charge=*/0, del_cb);
+    return cache_->Insert(key, /*obj=*/nullptr, internal_helper,
+                          /*charge=*/0);
   } else {
     cache_->Release(lru_handle, /*erase_if_last_ref=*/false);
   }
@@ -169,10 +175,10 @@ Status CompressedSecondaryCache::Insert(const Slice& key, void* value,
     size_t charge{0};
     CacheValueChunk* value_chunks_head =
         SplitValueIntoChunks(val, cache_options_.compression_type, charge);
-    return cache_->Insert(key, value_chunks_head, charge, del_cb);
+    return cache_->Insert(key, value_chunks_head, internal_helper, charge);
   } else {
     CacheAllocationPtr* buf = new CacheAllocationPtr(std::move(ptr));
-    return cache_->Insert(key, buf, size, del_cb);
+    return cache_->Insert(key, buf, internal_helper, size);
   }
 }
 
@@ -276,23 +282,29 @@ CacheAllocationPtr CompressedSecondaryCache::MergeChunksIntoValue(
   return ptr;
 }
 
-Cache::DeleterFn CompressedSecondaryCache::GetDeletionCallback(
-    bool enable_custom_split_merge) {
+const Cache::CacheItemHelper* CompressedSecondaryCache::GetHelper(
+    bool enable_custom_split_merge) const {
   if (enable_custom_split_merge) {
-    return [](const Slice& /*key*/, void* obj) {
-      CacheValueChunk* chunks_head = reinterpret_cast<CacheValueChunk*>(obj);
-      while (chunks_head != nullptr) {
-        CacheValueChunk* tmp_chunk = chunks_head;
-        chunks_head = chunks_head->next;
-        tmp_chunk->Free();
-        obj = nullptr;
-      };
-    };
+    static const Cache::CacheItemHelper kHelper{
+        CacheEntryRole::kMisc,
+        [](Cache::ObjectPtr obj, MemoryAllocator* /*alloc*/) {
+          CacheValueChunk* chunks_head = static_cast<CacheValueChunk*>(obj);
+          while (chunks_head != nullptr) {
+            CacheValueChunk* tmp_chunk = chunks_head;
+            chunks_head = chunks_head->next;
+            tmp_chunk->Free();
+            obj = nullptr;
+          };
+        }};
+    return &kHelper;
   } else {
-    return [](const Slice& /*key*/, void* obj) {
-      delete reinterpret_cast<CacheAllocationPtr*>(obj);
-      obj = nullptr;
-    };
+    static const Cache::CacheItemHelper kHelper{
+        CacheEntryRole::kMisc,
+        [](Cache::ObjectPtr obj, MemoryAllocator* /*alloc*/) {
+          delete static_cast<CacheAllocationPtr*>(obj);
+          obj = nullptr;
+        }};
+    return &kHelper;
   }
 }
 

cache/compressed_secondary_cache.h

@@ -21,7 +21,7 @@ namespace ROCKSDB_NAMESPACE {
 
 class CompressedSecondaryCacheResultHandle : public SecondaryCacheResultHandle {
  public:
-  CompressedSecondaryCacheResultHandle(void* value, size_t size)
+  CompressedSecondaryCacheResultHandle(Cache::ObjectPtr value, size_t size)
       : value_(value), size_(size) {}
   ~CompressedSecondaryCacheResultHandle() override = default;
 
@@ -34,12 +34,12 @@ class CompressedSecondaryCacheResultHandle : public SecondaryCacheResultHandle {
 
   void Wait() override {}
 
-  void* Value() override { return value_; }
+  Cache::ObjectPtr Value() override { return value_; }
 
   size_t Size() override { return size_; }
 
  private:
-  void* value_;
+  Cache::ObjectPtr value_;
   size_t size_;
 };
 
@@ -83,12 +83,13 @@ class CompressedSecondaryCache : public SecondaryCache {
 
   const char* Name() const override { return "CompressedSecondaryCache"; }
 
-  Status Insert(const Slice& key, void* value,
+  Status Insert(const Slice& key, Cache::ObjectPtr value,
                 const Cache::CacheItemHelper* helper) override;
 
   std::unique_ptr<SecondaryCacheResultHandle> Lookup(
-      const Slice& key, const Cache::CreateCallback& create_cb, bool /*wait*/,
-      bool advise_erase, bool& is_in_sec_cache) override;
+      const Slice& key, const Cache::CacheItemHelper* helper,
+      Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase,
+      bool& is_in_sec_cache) override;
 
   bool SupportForceErase() const override { return true; }
 
@@ -129,8 +130,8 @@ class CompressedSecondaryCache : public SecondaryCache {
   CacheAllocationPtr MergeChunksIntoValue(const void* chunks_head,
                                           size_t& charge);
 
-  // An implementation of Cache::DeleterFn.
-  static Cache::DeleterFn GetDeletionCallback(bool enable_custom_split_merge);
+  // TODO: clean up to use cleaner interfaces in typed_cache.h
+  const Cache::CacheItemHelper* GetHelper(bool enable_custom_split_merge) const;
   std::shared_ptr<Cache> cache_;
   CompressedSecondaryCacheOptions cache_options_;
   mutable port::Mutex capacity_mutex_;

cache/compressed_secondary_cache_test.cc

@@ -16,7 +16,8 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-class CompressedSecondaryCacheTest : public testing::Test {
+class CompressedSecondaryCacheTest : public testing::Test,
+                                     public Cache::CreateContext {
  public:
   CompressedSecondaryCacheTest() : fail_create_(false) {}
   ~CompressedSecondaryCacheTest() override = default;
@@ -37,13 +38,13 @@ class CompressedSecondaryCacheTest : public testing::Test {
     size_t size_;
   };
 
-  static size_t SizeCallback(void* obj) {
-    return reinterpret_cast<TestItem*>(obj)->Size();
+  static size_t SizeCallback(Cache::ObjectPtr obj) {
+    return static_cast<TestItem*>(obj)->Size();
   }
 
-  static Status SaveToCallback(void* from_obj, size_t from_offset,
-                               size_t length, void* out) {
-    auto item = reinterpret_cast<TestItem*>(from_obj);
+  static Status SaveToCallback(Cache::ObjectPtr from_obj, size_t from_offset,
+                               size_t length, char* out) {
+    auto item = static_cast<TestItem*>(from_obj);
     const char* buf = item->Buf();
     EXPECT_EQ(length, item->Size());
     EXPECT_EQ(from_offset, 0);
@@ -51,30 +52,36 @@ class CompressedSecondaryCacheTest : public testing::Test {
     return Status::OK();
   }
 
-  static void DeletionCallback(const Slice& /*key*/, void* obj) {
-    delete reinterpret_cast<TestItem*>(obj);
+  static void DeletionCallback(Cache::ObjectPtr obj,
+                               MemoryAllocator* /*alloc*/) {
+    delete static_cast<TestItem*>(obj);
     obj = nullptr;
   }
 
-  static Cache::CacheItemHelper helper_;
-
-  static Status SaveToCallbackFail(void* /*obj*/, size_t /*offset*/,
-                                   size_t /*size*/, void* /*out*/) {
+  static Status SaveToCallbackFail(Cache::ObjectPtr /*obj*/, size_t /*offset*/,
+                                   size_t /*size*/, char* /*out*/) {
     return Status::NotSupported();
   }
 
-  static Cache::CacheItemHelper helper_fail_;
-
-  Cache::CreateCallback test_item_creator = [&](const void* buf, size_t size,
-                                                void** out_obj,
-                                                size_t* charge) -> Status {
-    if (fail_create_) {
+  static Status CreateCallback(const Slice& data, Cache::CreateContext* context,
+                               MemoryAllocator* /*allocator*/,
+                               Cache::ObjectPtr* out_obj, size_t* out_charge) {
+    auto t = static_cast<CompressedSecondaryCacheTest*>(context);
+    if (t->fail_create_) {
       return Status::NotSupported();
     }
-    *out_obj = reinterpret_cast<void*>(new TestItem((char*)buf, size));
-    *charge = size;
+    *out_obj = new TestItem(data.data(), data.size());
+    *out_charge = data.size();
     return Status::OK();
-  };
+  }
+
+  static constexpr Cache::CacheItemHelper kHelper{
+      CacheEntryRole::kMisc, &DeletionCallback, &SizeCallback, &SaveToCallback,
+      &CreateCallback};
+
+  static constexpr Cache::CacheItemHelper kHelperFail{
+      CacheEntryRole::kMisc, &DeletionCallback, &SizeCallback,
+      &SaveToCallbackFail, &CreateCallback};
 
   void SetFailCreate(bool fail) { fail_create_ = fail; }
 
@@ -84,7 +91,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
     bool is_in_sec_cache{true};
     // Lookup an non-existent key.
     std::unique_ptr<SecondaryCacheResultHandle> handle0 = sec_cache->Lookup(
-        "k0", test_item_creator, true, /*advise_erase=*/true, is_in_sec_cache);
+        "k0", &kHelper, this, true, /*advise_erase=*/true, is_in_sec_cache);
     ASSERT_EQ(handle0, nullptr);
 
     Random rnd(301);
@@ -92,23 +99,21 @@ class CompressedSecondaryCacheTest : public testing::Test {
     std::string str1(rnd.RandomString(1000));
     TestItem item1(str1.data(), str1.length());
     // A dummy handle is inserted if the item is inserted for the first time.
-    ASSERT_OK(sec_cache->Insert("k1", &item1,
-                                &CompressedSecondaryCacheTest::helper_));
+    ASSERT_OK(sec_cache->Insert("k1", &item1, &kHelper));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1);
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
 
     std::unique_ptr<SecondaryCacheResultHandle> handle1_1 = sec_cache->Lookup(
-        "k1", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache);
+        "k1", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache);
     ASSERT_EQ(handle1_1, nullptr);
 
     // Insert and Lookup the item k1 for the second time and advise erasing it.
-    ASSERT_OK(sec_cache->Insert("k1", &item1,
-                                &CompressedSecondaryCacheTest::helper_));
+    ASSERT_OK(sec_cache->Insert("k1", &item1, &kHelper));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1);
 
     std::unique_ptr<SecondaryCacheResultHandle> handle1_2 = sec_cache->Lookup(
-        "k1", test_item_creator, true, /*advise_erase=*/true, is_in_sec_cache);
+        "k1", &kHelper, this, true, /*advise_erase=*/true, is_in_sec_cache);
     ASSERT_NE(handle1_2, nullptr);
     ASSERT_FALSE(is_in_sec_cache);
     if (sec_cache_is_compressed) {
@@ -128,21 +133,19 @@ class CompressedSecondaryCacheTest : public testing::Test {
 
     // Lookup the item k1 again.
     std::unique_ptr<SecondaryCacheResultHandle> handle1_3 = sec_cache->Lookup(
-        "k1", test_item_creator, true, /*advise_erase=*/true, is_in_sec_cache);
+        "k1", &kHelper, this, true, /*advise_erase=*/true, is_in_sec_cache);
     ASSERT_EQ(handle1_3, nullptr);
 
     // Insert and Lookup the item k2.
     std::string str2(rnd.RandomString(1000));
     TestItem item2(str2.data(), str2.length());
-    ASSERT_OK(sec_cache->Insert("k2", &item2,
-                                &CompressedSecondaryCacheTest::helper_));
+    ASSERT_OK(sec_cache->Insert("k2", &item2, &kHelper));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2);
     std::unique_ptr<SecondaryCacheResultHandle> handle2_1 = sec_cache->Lookup(
-        "k2", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache);
+        "k2", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache);
     ASSERT_EQ(handle2_1, nullptr);
 
-    ASSERT_OK(sec_cache->Insert("k2", &item2,
-                                &CompressedSecondaryCacheTest::helper_));
+    ASSERT_OK(sec_cache->Insert("k2", &item2, &kHelper));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2);
     if (sec_cache_is_compressed) {
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
@@ -154,7 +157,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
     }
     std::unique_ptr<SecondaryCacheResultHandle> handle2_2 = sec_cache->Lookup(
-        "k2", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache);
+        "k2", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache);
     ASSERT_NE(handle2_2, nullptr);
     std::unique_ptr<TestItem> val2 =
         std::unique_ptr<TestItem>(static_cast<TestItem*>(handle2_2->Value()));
@@ -223,28 +226,24 @@ class CompressedSecondaryCacheTest : public testing::Test {
     std::string str1(rnd.RandomString(1000));
     TestItem item1(str1.data(), str1.length());
     // Insert a dummy handle.
-    ASSERT_OK(sec_cache->Insert("k1", &item1,
-                                &CompressedSecondaryCacheTest::helper_));
+    ASSERT_OK(sec_cache->Insert("k1", &item1, &kHelper));
     // Insert k1.
-    ASSERT_OK(sec_cache->Insert("k1", &item1,
-                                &CompressedSecondaryCacheTest::helper_));
+    ASSERT_OK(sec_cache->Insert("k1", &item1, &kHelper));
 
     // Insert and Lookup the second item.
     std::string str2(rnd.RandomString(200));
     TestItem item2(str2.data(), str2.length());
     // Insert a dummy handle, k1 is not evicted.
-    ASSERT_OK(sec_cache->Insert("k2", &item2,
-                                &CompressedSecondaryCacheTest::helper_));
+    ASSERT_OK(sec_cache->Insert("k2", &item2, &kHelper));
     bool is_in_sec_cache{false};
     std::unique_ptr<SecondaryCacheResultHandle> handle1 = sec_cache->Lookup(
-        "k1", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache);
+        "k1", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache);
     ASSERT_EQ(handle1, nullptr);
 
     // Insert k2 and k1 is evicted.
-    ASSERT_OK(sec_cache->Insert("k2", &item2,
-                                &CompressedSecondaryCacheTest::helper_));
+    ASSERT_OK(sec_cache->Insert("k2", &item2, &kHelper));
     std::unique_ptr<SecondaryCacheResultHandle> handle2 = sec_cache->Lookup(
-        "k2", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache);
+        "k2", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache);
     ASSERT_NE(handle2, nullptr);
     std::unique_ptr<TestItem> val2 =
         std::unique_ptr<TestItem>(static_cast<TestItem*>(handle2->Value()));
@@ -252,27 +251,24 @@ class CompressedSecondaryCacheTest : public testing::Test {
     ASSERT_EQ(memcmp(val2->Buf(), item2.Buf(), item2.Size()), 0);
 
     // Insert k1 again and a dummy handle is inserted.
-    ASSERT_OK(sec_cache->Insert("k1", &item1,
-                                &CompressedSecondaryCacheTest::helper_));
+    ASSERT_OK(sec_cache->Insert("k1", &item1, &kHelper));
 
     std::unique_ptr<SecondaryCacheResultHandle> handle1_1 = sec_cache->Lookup(
-        "k1", test_item_creator, true, /*advise_erase=*/false, is_in_sec_cache);
+        "k1", &kHelper, this, true, /*advise_erase=*/false, is_in_sec_cache);
     ASSERT_EQ(handle1_1, nullptr);
 
     // Create Fails.
     SetFailCreate(true);
     std::unique_ptr<SecondaryCacheResultHandle> handle2_1 = sec_cache->Lookup(
-        "k2", test_item_creator, true, /*advise_erase=*/true, is_in_sec_cache);
+        "k2", &kHelper, this, true, /*advise_erase=*/true, is_in_sec_cache);
     ASSERT_EQ(handle2_1, nullptr);
 
     // Save Fails.
     std::string str3 = rnd.RandomString(10);
     TestItem item3(str3.data(), str3.length());
     // The Status is OK because a dummy handle is inserted.
-    ASSERT_OK(sec_cache->Insert("k3", &item3,
-                                &CompressedSecondaryCacheTest::helper_fail_));
-    ASSERT_NOK(sec_cache->Insert("k3", &item3,
-                                 &CompressedSecondaryCacheTest::helper_fail_));
+    ASSERT_OK(sec_cache->Insert("k3", &item3, &kHelperFail));
+    ASSERT_NOK(sec_cache->Insert("k3", &item3, &kHelperFail));
 
     sec_cache.reset();
   }
@@ -309,15 +305,13 @@ class CompressedSecondaryCacheTest : public testing::Test {
     Random rnd(301);
     std::string str1 = rnd.RandomString(1001);
     auto item1_1 = new TestItem(str1.data(), str1.length());
-    ASSERT_OK(cache->Insert(
-        "k1", item1_1, &CompressedSecondaryCacheTest::helper_, str1.length()));
+    ASSERT_OK(cache->Insert("k1", item1_1, &kHelper, str1.length()));
 
     std::string str2 = rnd.RandomString(1012);
     auto item2_1 = new TestItem(str2.data(), str2.length());
     // After this Insert, primary cache contains k2 and secondary cache contains
     // k1's dummy item.
-    ASSERT_OK(cache->Insert(
-        "k2", item2_1, &CompressedSecondaryCacheTest::helper_, str2.length()));
+    ASSERT_OK(cache->Insert("k2", item2_1, &kHelper, str2.length()));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 1);
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes, 0);
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_compressed_bytes, 0);
@@ -326,22 +320,19 @@ class CompressedSecondaryCacheTest : public testing::Test {
     auto item3_1 = new TestItem(str3.data(), str3.length());
     // After this Insert, primary cache contains k3 and secondary cache contains
     // k1's dummy item and k2's dummy item.
-    ASSERT_OK(cache->Insert(
-        "k3", item3_1, &CompressedSecondaryCacheTest::helper_, str3.length()));
+    ASSERT_OK(cache->Insert("k3", item3_1, &kHelper, str3.length()));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 2);
 
     // After this Insert, primary cache contains k1 and secondary cache contains
     // k1's dummy item, k2's dummy item, and k3's dummy item.
     auto item1_2 = new TestItem(str1.data(), str1.length());
-    ASSERT_OK(cache->Insert(
-        "k1", item1_2, &CompressedSecondaryCacheTest::helper_, str1.length()));
+    ASSERT_OK(cache->Insert("k1", item1_2, &kHelper, str1.length()));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 3);
 
     // After this Insert, primary cache contains k2 and secondary cache contains
     // k1's item, k2's dummy item, and k3's dummy item.
     auto item2_2 = new TestItem(str2.data(), str2.length());
-    ASSERT_OK(cache->Insert(
-        "k2", item2_2, &CompressedSecondaryCacheTest::helper_, str2.length()));
+    ASSERT_OK(cache->Insert("k2", item2_2, &kHelper, str2.length()));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 1);
     if (sec_cache_is_compressed) {
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
@@ -356,8 +347,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
     // After this Insert, primary cache contains k3 and secondary cache contains
     // k1's item and k2's item.
     auto item3_2 = new TestItem(str3.data(), str3.length());
-    ASSERT_OK(cache->Insert(
-        "k3", item3_2, &CompressedSecondaryCacheTest::helper_, str3.length()));
+    ASSERT_OK(cache->Insert("k3", item3_2, &kHelper, str3.length()));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 2);
     if (sec_cache_is_compressed) {
       ASSERT_EQ(get_perf_context()->compressed_sec_cache_uncompressed_bytes,
@@ -370,8 +360,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
     }
 
     Cache::Handle* handle;
-    handle = cache->Lookup("k3", &CompressedSecondaryCacheTest::helper_,
-                           test_item_creator, Cache::Priority::LOW, true,
+    handle = cache->Lookup("k3", &kHelper, this, Cache::Priority::LOW, true,
                            stats.get());
     ASSERT_NE(handle, nullptr);
     auto val3 = static_cast<TestItem*>(cache->Value(handle));
@@ -380,15 +369,13 @@ class CompressedSecondaryCacheTest : public testing::Test {
     cache->Release(handle);
 
     // Lookup an non-existent key.
-    handle = cache->Lookup("k0", &CompressedSecondaryCacheTest::helper_,
-                           test_item_creator, Cache::Priority::LOW, true,
+    handle = cache->Lookup("k0", &kHelper, this, Cache::Priority::LOW, true,
                            stats.get());
     ASSERT_EQ(handle, nullptr);
 
     // This Lookup should just insert a dummy handle in the primary cache
     // and the k1 is still in the secondary cache.
-    handle = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_,
-                           test_item_creator, Cache::Priority::LOW, true,
+    handle = cache->Lookup("k1", &kHelper, this, Cache::Priority::LOW, true,
                            stats.get());
     ASSERT_NE(handle, nullptr);
     ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 1);
@@ -400,8 +387,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
     // This Lookup should erase k1 from the secondary cache and insert
     // it into primary cache; then k3 is demoted.
     // k2 and k3 are in secondary cache.
-    handle = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_,
-                           test_item_creator, Cache::Priority::LOW, true,
+    handle = cache->Lookup("k1", &kHelper, this, Cache::Priority::LOW, true,
                            stats.get());
     ASSERT_NE(handle, nullptr);
     ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 1);
@@ -409,8 +395,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
     cache->Release(handle);
 
     // k2 is still in secondary cache.
-    handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_,
-                           test_item_creator, Cache::Priority::LOW, true,
+    handle = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, true,
                            stats.get());
     ASSERT_NE(handle, nullptr);
     ASSERT_EQ(get_perf_context()->block_cache_standalone_handle_count, 2);
@@ -418,8 +403,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
 
     // Testing SetCapacity().
     ASSERT_OK(secondary_cache->SetCapacity(0));
-    handle = cache->Lookup("k3", &CompressedSecondaryCacheTest::helper_,
-                           test_item_creator, Cache::Priority::LOW, true,
+    handle = cache->Lookup("k3", &kHelper, this, Cache::Priority::LOW, true,
                            stats.get());
     ASSERT_EQ(handle, nullptr);
 
@@ -429,35 +413,30 @@ class CompressedSecondaryCacheTest : public testing::Test {
     ASSERT_EQ(capacity, 7000);
     auto item1_3 = new TestItem(str1.data(), str1.length());
     // After this Insert, primary cache contains k1.
-    ASSERT_OK(cache->Insert(
-        "k1", item1_3, &CompressedSecondaryCacheTest::helper_, str2.length()));
+    ASSERT_OK(cache->Insert("k1", item1_3, &kHelper, str2.length()));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 3);
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 4);
 
     auto item2_3 = new TestItem(str2.data(), str2.length());
     // After this Insert, primary cache contains k2 and secondary cache contains
     // k1's dummy item.
-    ASSERT_OK(cache->Insert(
-        "k2", item2_3, &CompressedSecondaryCacheTest::helper_, str1.length()));
+    ASSERT_OK(cache->Insert("k2", item2_3, &kHelper, str1.length()));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 4);
 
     auto item1_4 = new TestItem(str1.data(), str1.length());
     // After this Insert, primary cache contains k1 and secondary cache contains
     // k1's dummy item and k2's dummy item.
-    ASSERT_OK(cache->Insert(
-        "k1", item1_4, &CompressedSecondaryCacheTest::helper_, str2.length()));
+    ASSERT_OK(cache->Insert("k1", item1_4, &kHelper, str2.length()));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_dummy_count, 5);
 
     auto item2_4 = new TestItem(str2.data(), str2.length());
     // After this Insert, primary cache contains k2 and secondary cache contains
     // k1's real item and k2's dummy item.
-    ASSERT_OK(cache->Insert(
-        "k2", item2_4, &CompressedSecondaryCacheTest::helper_, str2.length()));
+    ASSERT_OK(cache->Insert("k2", item2_4, &kHelper, str2.length()));
     ASSERT_EQ(get_perf_context()->compressed_sec_cache_insert_real_count, 5);
     // This Lookup should just insert a dummy handle in the primary cache
     // and the k1 is still in the secondary cache.
-    handle = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_,
-                           test_item_creator, Cache::Priority::LOW, true,
+    handle = cache->Lookup("k1", &kHelper, this, Cache::Priority::LOW, true,
                            stats.get());
 
     ASSERT_NE(handle, nullptr);
@@ -496,18 +475,13 @@ class CompressedSecondaryCacheTest : public testing::Test {
     Random rnd(301);
     std::string str1 = rnd.RandomString(1001);
     auto item1 = std::make_unique<TestItem>(str1.data(), str1.length());
-    ASSERT_NOK(cache->Insert("k1", item1.get(), nullptr, str1.length()));
-    ASSERT_OK(cache->Insert("k1", item1.get(),
-                            &CompressedSecondaryCacheTest::helper_,
-                            str1.length()));
+    ASSERT_OK(cache->Insert("k1", item1.get(), &kHelper, str1.length()));
     item1.release();  // Appease clang-analyze "potential memory leak"
 
     Cache::Handle* handle;
-    handle = cache->Lookup("k2", nullptr, test_item_creator,
-                           Cache::Priority::LOW, true);
+    handle = cache->Lookup("k2", nullptr, this, Cache::Priority::LOW, true);
     ASSERT_EQ(handle, nullptr);
-    handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_,
-                           test_item_creator, Cache::Priority::LOW, false);
+    handle = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, false);
     ASSERT_EQ(handle, nullptr);
 
     cache.reset();
@@ -543,29 +517,25 @@ class CompressedSecondaryCacheTest : public testing::Test {
     Random rnd(301);
     std::string str1 = rnd.RandomString(1001);
     auto item1 = new TestItem(str1.data(), str1.length());
-    ASSERT_OK(cache->Insert("k1", item1,
-                            &CompressedSecondaryCacheTest::helper_fail_,
-                            str1.length()));
+    ASSERT_OK(cache->Insert("k1", item1, &kHelperFail, str1.length()));
 
     std::string str2 = rnd.RandomString(1002);
     auto item2 = new TestItem(str2.data(), str2.length());
     // k1 should be demoted to the secondary cache.
-    ASSERT_OK(cache->Insert("k2", item2,
-                            &CompressedSecondaryCacheTest::helper_fail_,
-                            str2.length()));
+    ASSERT_OK(cache->Insert("k2", item2, &kHelperFail, str2.length()));
 
     Cache::Handle* handle;
-    handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_fail_,
-                           test_item_creator, Cache::Priority::LOW, true);
+    handle =
+        cache->Lookup("k2", &kHelperFail, this, Cache::Priority::LOW, true);
     ASSERT_NE(handle, nullptr);
     cache->Release(handle);
     // This lookup should fail, since k1 demotion would have failed.
-    handle = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_fail_,
-                           test_item_creator, Cache::Priority::LOW, true);
+    handle =
+        cache->Lookup("k1", &kHelperFail, this, Cache::Priority::LOW, true);
     ASSERT_EQ(handle, nullptr);
     // Since k1 was not promoted, k2 should still be in cache.
-    handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_fail_,
-                           test_item_creator, Cache::Priority::LOW, true);
+    handle =
+        cache->Lookup("k2", &kHelperFail, this, Cache::Priority::LOW, true);
     ASSERT_NE(handle, nullptr);
     cache->Release(handle);
 
@@ -602,28 +572,23 @@ class CompressedSecondaryCacheTest : public testing::Test {
     Random rnd(301);
     std::string str1 = rnd.RandomString(1001);
     auto item1 = new TestItem(str1.data(), str1.length());
-    ASSERT_OK(cache->Insert("k1", item1, &CompressedSecondaryCacheTest::helper_,
-                            str1.length()));
+    ASSERT_OK(cache->Insert("k1", item1, &kHelper, str1.length()));
 
     std::string str2 = rnd.RandomString(1002);
     auto item2 = new TestItem(str2.data(), str2.length());
     // k1 should be demoted to the secondary cache.
-    ASSERT_OK(cache->Insert("k2", item2, &CompressedSecondaryCacheTest::helper_,
-                            str2.length()));
+    ASSERT_OK(cache->Insert("k2", item2, &kHelper, str2.length()));
 
     Cache::Handle* handle;
     SetFailCreate(true);
-    handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_,
-                           test_item_creator, Cache::Priority::LOW, true);
+    handle = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, true);
     ASSERT_NE(handle, nullptr);
     cache->Release(handle);
     // This lookup should fail, since k1 creation would have failed
-    handle = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_,
-                           test_item_creator, Cache::Priority::LOW, true);
+    handle = cache->Lookup("k1", &kHelper, this, Cache::Priority::LOW, true);
     ASSERT_EQ(handle, nullptr);
     // Since k1 didn't get promoted, k2 should still be in cache
-    handle = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_,
-                           test_item_creator, Cache::Priority::LOW, true);
+    handle = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, true);
     ASSERT_NE(handle, nullptr);
     cache->Release(handle);
 
@@ -660,32 +625,27 @@ class CompressedSecondaryCacheTest : public testing::Test {
     Random rnd(301);
     std::string str1 = rnd.RandomString(1001);
     auto item1_1 = new TestItem(str1.data(), str1.length());
-    ASSERT_OK(cache->Insert(
-        "k1", item1_1, &CompressedSecondaryCacheTest::helper_, str1.length()));
+    ASSERT_OK(cache->Insert("k1", item1_1, &kHelper, str1.length()));
 
     std::string str2 = rnd.RandomString(1002);
     std::string str2_clone{str2};
     auto item2 = new TestItem(str2.data(), str2.length());
     // After this Insert, primary cache contains k2 and secondary cache contains
     // k1's dummy item.
-    ASSERT_OK(cache->Insert("k2", item2, &CompressedSecondaryCacheTest::helper_,
-                            str2.length()));
+    ASSERT_OK(cache->Insert("k2", item2, &kHelper, str2.length()));
 
     // After this Insert, primary cache contains k1 and secondary cache contains
     // k1's dummy item and k2's dummy item.
     auto item1_2 = new TestItem(str1.data(), str1.length());
-    ASSERT_OK(cache->Insert(
-        "k1", item1_2, &CompressedSecondaryCacheTest::helper_, str1.length()));
+    ASSERT_OK(cache->Insert("k1", item1_2, &kHelper, str1.length()));
 
     auto item2_2 = new TestItem(str2.data(), str2.length());
     // After this Insert, primary cache contains k2 and secondary cache contains
     // k1's item and k2's dummy item.
-    ASSERT_OK(cache->Insert(
-        "k2", item2_2, &CompressedSecondaryCacheTest::helper_, str2.length()));
+    ASSERT_OK(cache->Insert("k2", item2_2, &kHelper, str2.length()));
 
     Cache::Handle* handle2;
-    handle2 = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_,
-                            test_item_creator, Cache::Priority::LOW, true);
+    handle2 = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, true);
     ASSERT_NE(handle2, nullptr);
     cache->Release(handle2);
 
@@ -693,14 +653,12 @@ class CompressedSecondaryCacheTest : public testing::Test {
     // strict_capacity_limit is true, but the lookup should still succeed.
     // A k1's dummy item is inserted into primary cache.
     Cache::Handle* handle1;
-    handle1 = cache->Lookup("k1", &CompressedSecondaryCacheTest::helper_,
-                            test_item_creator, Cache::Priority::LOW, true);
+    handle1 = cache->Lookup("k1", &kHelper, this, Cache::Priority::LOW, true);
     ASSERT_NE(handle1, nullptr);
     cache->Release(handle1);
 
     // Since k1 didn't get inserted, k2 should still be in cache
-    handle2 = cache->Lookup("k2", &CompressedSecondaryCacheTest::helper_,
-                            test_item_creator, Cache::Priority::LOW, true);
+    handle2 = cache->Lookup("k2", &kHelper, this, Cache::Priority::LOW, true);
     ASSERT_NE(handle2, nullptr);
     cache->Release(handle2);
 
@@ -741,7 +699,7 @@ class CompressedSecondaryCacheTest : public testing::Test {
     current_chunk = current_chunk->next;
     ASSERT_EQ(current_chunk->size, 98);
 
-    sec_cache->GetDeletionCallback(true)("dummy", chunks_head);
+    sec_cache->GetHelper(true)->del_cb(chunks_head, /*alloc*/ nullptr);
   }
 
   void MergeChunksIntoValueTest() {
@@ -822,23 +780,13 @@ class CompressedSecondaryCacheTest : public testing::Test {
     std::string value_str{value.get(), charge};
     ASSERT_EQ(strcmp(value_str.data(), str.data()), 0);
 
-    sec_cache->GetDeletionCallback(true)("dummy", chunks_head);
+    sec_cache->GetHelper(true)->del_cb(chunks_head, /*alloc*/ nullptr);
   }
 
  private:
   bool fail_create_;
 };
 
-Cache::CacheItemHelper CompressedSecondaryCacheTest::helper_(
-    CompressedSecondaryCacheTest::SizeCallback,
-    CompressedSecondaryCacheTest::SaveToCallback,
-    CompressedSecondaryCacheTest::DeletionCallback);
-
-Cache::CacheItemHelper CompressedSecondaryCacheTest::helper_fail_(
-    CompressedSecondaryCacheTest::SizeCallback,
-    CompressedSecondaryCacheTest::SaveToCallbackFail,
-    CompressedSecondaryCacheTest::DeletionCallback);
-
 class CompressedSecCacheTestWithCompressAndAllocatorParam
     : public CompressedSecondaryCacheTest,
       public ::testing::WithParamInterface<std::tuple<bool, bool>> {

cache/lru_cache.cc

@@ -22,20 +22,28 @@
 namespace ROCKSDB_NAMESPACE {
 namespace lru_cache {
 
+namespace {
 // A distinct pointer value for marking "dummy" cache entries
-void* const kDummyValueMarker = const_cast<char*>("kDummyValueMarker");
+struct DummyValue {
+  char val[12] = "kDummyValue";
+};
+DummyValue kDummyValue{};
+}  // namespace
 
-LRUHandleTable::LRUHandleTable(int max_upper_hash_bits)
+LRUHandleTable::LRUHandleTable(int max_upper_hash_bits,
+                               MemoryAllocator* allocator)
     : length_bits_(/* historical starting size*/ 4),
       list_(new LRUHandle* [size_t{1} << length_bits_] {}),
       elems_(0),
-      max_length_bits_(max_upper_hash_bits) {}
+      max_length_bits_(max_upper_hash_bits),
+      allocator_(allocator) {}
 
 LRUHandleTable::~LRUHandleTable() {
+  auto alloc = allocator_;
   ApplyToEntriesRange(
-      [](LRUHandle* h) {
+      [alloc](LRUHandle* h) {
         if (!h->HasRefs()) {
-          h->Free();
+          h->Free(alloc);
         }
       },
       0, size_t{1} << length_bits_);
@@ -118,6 +126,7 @@ LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit,
                              double low_pri_pool_ratio, bool use_adaptive_mutex,
                              CacheMetadataChargePolicy metadata_charge_policy,
                              int max_upper_hash_bits,
+                             MemoryAllocator* allocator,
                              SecondaryCache* secondary_cache)
     : CacheShardBase(metadata_charge_policy),
       capacity_(0),
@@ -128,7 +137,7 @@ LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit,
       high_pri_pool_capacity_(0),
       low_pri_pool_ratio_(low_pri_pool_ratio),
       low_pri_pool_capacity_(0),
-      table_(max_upper_hash_bits),
+      table_(max_upper_hash_bits, allocator),
       usage_(0),
       lru_usage_(0),
       mutex_(use_adaptive_mutex),
@@ -159,13 +168,14 @@ void LRUCacheShard::EraseUnRefEntries() {
   }
 
   for (auto entry : last_reference_list) {
-    entry->Free();
+    entry->Free(table_.GetAllocator());
   }
 }
 
 void LRUCacheShard::ApplyToSomeEntries(
-    const std::function<void(const Slice& key, void* value, size_t charge,
-                             DeleterFn deleter)>& callback,
+    const std::function<void(const Slice& key, Cache::ObjectPtr value,
+                             size_t charge,
+                             const Cache::CacheItemHelper* helper)>& callback,
     size_t average_entries_per_lock, size_t* state) {
   // The state is essentially going to be the starting hash, which works
   // nicely even if we resize between calls because we use upper-most
@@ -192,11 +202,8 @@ void LRUCacheShard::ApplyToSomeEntries(
   table_.ApplyToEntriesRange(
       [callback,
        metadata_charge_policy = metadata_charge_policy_](LRUHandle* h) {
-        DeleterFn deleter = h->IsSecondaryCacheCompatible()
-                                ? h->info_.helper->del_cb
-                                : h->info_.deleter;
         callback(h->key(), h->value, h->GetCharge(metadata_charge_policy),
-                 deleter);
+                 h->helper);
       },
       index_begin, index_end);
 }
@@ -339,11 +346,11 @@ void LRUCacheShard::TryInsertIntoSecondaryCache(
   for (auto entry : evicted_handles) {
     if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
         !entry->IsInSecondaryCache()) {
-      secondary_cache_->Insert(entry->key(), entry->value, entry->info_.helper)
+      secondary_cache_->Insert(entry->key(), entry->value, entry->helper)
           .PermitUncheckedError();
     }
     // Free the entries here outside of mutex for performance reasons.
-    entry->Free();
+    entry->Free(table_.GetAllocator());
   }
 }
 
@@ -464,7 +471,7 @@ void LRUCacheShard::Promote(LRUHandle* e) {
       TryInsertIntoSecondaryCache(last_reference_list);
       if (free_standalone_handle) {
         e->Unref();
-        e->Free();
+        e->Free(table_.GetAllocator());
         e = nullptr;
       } else {
         PERF_COUNTER_ADD(block_cache_standalone_handle_count, 1);
@@ -476,9 +483,9 @@ void LRUCacheShard::Promote(LRUHandle* e) {
       // rare case that one exists
       Cache::Priority priority =
           e->IsHighPri() ? Cache::Priority::HIGH : Cache::Priority::LOW;
-      s = Insert(e->key(), e->hash, kDummyValueMarker, /*charge=*/0,
-                 /*deleter=*/nullptr, /*helper=*/nullptr, /*handle=*/nullptr,
-                 priority);
+      s = Insert(e->key(), e->hash, &kDummyValue, &kNoopCacheItemHelper,
+                 /*charge=*/0,
+                 /*handle=*/nullptr, priority);
     } else {
       e->SetInCache(true);
       LRUHandle* handle = e;
@@ -508,7 +515,7 @@ void LRUCacheShard::Promote(LRUHandle* e) {
 
 LRUHandle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash,
                                  const Cache::CacheItemHelper* helper,
-                                 const Cache::CreateCallback& create_cb,
+                                 Cache::CreateContext* create_context,
                                  Cache::Priority priority, bool wait,
                                  Statistics* stats) {
   LRUHandle* e = nullptr;
@@ -518,7 +525,7 @@ LRUHandle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash,
     e = table_.Lookup(key, hash);
     if (e != nullptr) {
       assert(e->InCache());
-      if (e->value == kDummyValueMarker) {
+      if (e->value == &kDummyValue) {
         // For a dummy handle, if it was retrieved from secondary cache,
         // it may still exist in secondary cache.
         // If the handle exists in secondary cache, the value should be
@@ -547,24 +554,17 @@ LRUHandle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash,
   // standalone handle is returned to the caller. Only if the block is hit
   // again, we erase it from CompressedSecondaryCache and add it into the
   // primary cache.
-  if (!e && secondary_cache_ && helper && helper->saveto_cb) {
-    // For objects from the secondary cache, we expect the caller to provide
-    // a way to create/delete the primary cache object. The only case where
-    // a deleter would not be required is for dummy entries inserted for
-    // accounting purposes, which we won't demote to the secondary cache
-    // anyway.
-    assert(create_cb && helper->del_cb);
+  if (!e && secondary_cache_ && helper && helper->create_cb) {
     bool is_in_sec_cache{false};
     std::unique_ptr<SecondaryCacheResultHandle> secondary_handle =
-        secondary_cache_->Lookup(key, create_cb, wait, found_dummy_entry,
-                                 is_in_sec_cache);
+        secondary_cache_->Lookup(key, helper, create_context, wait,
+                                 found_dummy_entry, is_in_sec_cache);
     if (secondary_handle != nullptr) {
       e = static_cast<LRUHandle*>(malloc(sizeof(LRUHandle) - 1 + key.size()));
 
       e->m_flags = 0;
       e->im_flags = 0;
-      e->SetSecondaryCacheCompatible(true);
-      e->info_.helper = helper;
+      e->helper = helper;
       e->key_length = key.size();
       e->hash = hash;
       e->refs = 0;
@@ -585,7 +585,7 @@ LRUHandle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash,
           if (!e->value) {
             // The secondary cache returned a handle, but the lookup failed.
             e->Unref();
-            e->Free();
+            e->Free(table_.GetAllocator());
             e = nullptr;
           } else {
             PERF_COUNTER_ADD(secondary_cache_hit_count, 1);
@@ -669,16 +669,18 @@ bool LRUCacheShard::Release(LRUHandle* e, bool /*useful*/,
 
   // Free the entry here outside of mutex for performance reasons.
   if (last_reference) {
-    e->Free();
+    e->Free(table_.GetAllocator());
   }
   return last_reference;
 }
 
-Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
-                             size_t charge,
-                             void (*deleter)(const Slice& key, void* value),
+Status LRUCacheShard::Insert(const Slice& key, uint32_t hash,
+                             Cache::ObjectPtr value,
                              const Cache::CacheItemHelper* helper,
-                             LRUHandle** handle, Cache::Priority priority) {
+                             size_t charge, LRUHandle** handle,
+                             Cache::Priority priority) {
+  assert(helper);
+
   // Allocate the memory here outside of the mutex.
   // If the cache is full, we'll have to release it.
   // It shouldn't happen very often though.
@@ -688,17 +690,7 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
   e->value = value;
   e->m_flags = 0;
   e->im_flags = 0;
-  if (helper) {
-    // Use only one of the two parameters
-    assert(deleter == nullptr);
-    // value == nullptr is reserved for indicating failure for when secondary
-    // cache compatible
-    assert(value != nullptr);
-    e->SetSecondaryCacheCompatible(true);
-    e->info_.helper = helper;
-  } else {
-    e->info_.deleter = deleter;
-  }
+  e->helper = helper;
   e->key_length = key.size();
   e->hash = hash;
   e->refs = 0;
@@ -708,6 +700,10 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
   memcpy(e->key_data, key.data(), key.size());
   e->CalcTotalCharge(charge, metadata_charge_policy_);
 
+  // value == nullptr is reserved for indicating failure for when secondary
+  // cache compatible
+  assert(!(e->IsSecondaryCacheCompatible() && value == nullptr));
+
   return InsertItem(e, handle, /* free_handle_on_fail */ true);
 }
 
@@ -733,7 +729,7 @@ void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
   // Free the entry here outside of mutex for performance reasons.
   // last_reference will only be true if e != nullptr.
   if (last_reference) {
-    e->Free();
+    e->Free(table_.GetAllocator());
   }
 }
 
@@ -793,18 +789,19 @@ LRUCache::LRUCache(size_t capacity, int num_shard_bits,
       secondary_cache_(std::move(_secondary_cache)) {
   size_t per_shard = GetPerShardCapacity();
   SecondaryCache* secondary_cache = secondary_cache_.get();
+  MemoryAllocator* alloc = memory_allocator();
   InitShards([=](LRUCacheShard* cs) {
     new (cs) LRUCacheShard(
         per_shard, strict_capacity_limit, high_pri_pool_ratio,
         low_pri_pool_ratio, use_adaptive_mutex, metadata_charge_policy,
-        /* max_upper_hash_bits */ 32 - num_shard_bits, secondary_cache);
+        /* max_upper_hash_bits */ 32 - num_shard_bits, alloc, secondary_cache);
   });
 }
 
-void* LRUCache::Value(Handle* handle) {
+Cache::ObjectPtr LRUCache::Value(Handle* handle) {
   auto h = reinterpret_cast<const LRUHandle*>(handle);
   assert(!h->IsPending() || h->value == nullptr);
-  assert(h->value != kDummyValueMarker);
+  assert(h->value != &kDummyValue);
   return h->value;
 }
 
@@ -813,13 +810,10 @@ size_t LRUCache::GetCharge(Handle* handle) const {
       GetShard(0).metadata_charge_policy_);
 }
 
-Cache::DeleterFn LRUCache::GetDeleter(Handle* handle) const {
+const Cache::CacheItemHelper* LRUCache::GetCacheItemHelper(
+    Handle* handle) const {
   auto h = reinterpret_cast<const LRUHandle*>(handle);
-  if (h->IsSecondaryCacheCompatible()) {
-    return h->info_.helper->del_cb;
-  } else {
-    return h->info_.deleter;
-  }
+  return h->helper;
 }
 
 size_t LRUCache::TEST_GetLRUSize() {

cache/lru_cache.h

@@ -13,6 +13,7 @@
 
 #include "cache/sharded_cache.h"
 #include "port/lang.h"
+#include "port/likely.h"
 #include "port/malloc.h"
 #include "port/port.h"
 #include "rocksdb/secondary_cache.h"
@@ -48,13 +49,8 @@ namespace lru_cache {
 // While refs > 0, public properties like value and deleter must not change.
 
 struct LRUHandle {
-  void* value;
-  union Info {
-    Info() {}
-    ~Info() {}
-    Cache::DeleterFn deleter;
-    const Cache::CacheItemHelper* helper;
-  } info_;
+  Cache::ObjectPtr value;
+  const Cache::CacheItemHelper* helper;
   // An entry is not added to the LRUHandleTable until the secondary cache
   // lookup is complete, so its safe to have this union.
   union {
@@ -93,14 +89,12 @@ struct LRUHandle {
     IM_IS_HIGH_PRI = (1 << 0),
     // Whether this entry is low priority entry.
     IM_IS_LOW_PRI = (1 << 1),
-    // Can this be inserted into the secondary cache.
-    IM_IS_SECONDARY_CACHE_COMPATIBLE = (1 << 2),
     // Is the handle still being read from a lower tier.
-    IM_IS_PENDING = (1 << 3),
+    IM_IS_PENDING = (1 << 2),
     // Whether this handle is still in a lower tier
-    IM_IS_IN_SECONDARY_CACHE = (1 << 4),
+    IM_IS_IN_SECONDARY_CACHE = (1 << 3),
     // Marks result handles that should not be inserted into cache
-    IM_IS_STANDALONE = (1 << 5),
+    IM_IS_STANDALONE = (1 << 4),
   };
 
   // Beginning of the key (MUST BE THE LAST FIELD IN THIS STRUCT!)
@@ -130,9 +124,7 @@ struct LRUHandle {
   bool IsLowPri() const { return im_flags & IM_IS_LOW_PRI; }
   bool InLowPriPool() const { return m_flags & M_IN_LOW_PRI_POOL; }
   bool HasHit() const { return m_flags & M_HAS_HIT; }
-  bool IsSecondaryCacheCompatible() const {
-    return im_flags & IM_IS_SECONDARY_CACHE_COMPATIBLE;
-  }
+  bool IsSecondaryCacheCompatible() const { return helper->size_cb != nullptr; }
   bool IsPending() const { return im_flags & IM_IS_PENDING; }
   bool IsInSecondaryCache() const {
     return im_flags & IM_IS_IN_SECONDARY_CACHE;
@@ -178,14 +170,6 @@ struct LRUHandle {
 
   void SetHit() { m_flags |= M_HAS_HIT; }
 
-  void SetSecondaryCacheCompatible(bool compat) {
-    if (compat) {
-      im_flags |= IM_IS_SECONDARY_CACHE_COMPATIBLE;
-    } else {
-      im_flags &= ~IM_IS_SECONDARY_CACHE_COMPATIBLE;
-    }
-  }
-
   void SetIsPending(bool pending) {
     if (pending) {
       im_flags |= IM_IS_PENDING;
@@ -210,22 +194,19 @@ struct LRUHandle {
     }
   }
 
-  void Free() {
+  void Free(MemoryAllocator* allocator) {
     assert(refs == 0);
 
-    if (!IsSecondaryCacheCompatible() && info_.deleter) {
-      (*info_.deleter)(key(), value);
-    } else if (IsSecondaryCacheCompatible()) {
-      if (IsPending()) {
-        assert(sec_handle != nullptr);
-        SecondaryCacheResultHandle* tmp_sec_handle = sec_handle;
-        tmp_sec_handle->Wait();
-        value = tmp_sec_handle->Value();
-        delete tmp_sec_handle;
-      }
-      if (value) {
-        (*info_.helper->del_cb)(key(), value);
-      }
+    if (UNLIKELY(IsPending())) {
+      assert(sec_handle != nullptr);
+      SecondaryCacheResultHandle* tmp_sec_handle = sec_handle;
+      tmp_sec_handle->Wait();
+      value = tmp_sec_handle->Value();
+      delete tmp_sec_handle;
+    }
+    assert(helper);
+    if (helper->del_cb) {
+      helper->del_cb(value, allocator);
     }
 
     free(this);
@@ -267,7 +248,7 @@ struct LRUHandle {
 // 4.4.3's builtin hashtable.
 class LRUHandleTable {
  public:
-  explicit LRUHandleTable(int max_upper_hash_bits);
+  explicit LRUHandleTable(int max_upper_hash_bits, MemoryAllocator* allocator);
   ~LRUHandleTable();
 
   LRUHandle* Lookup(const Slice& key, uint32_t hash);
@@ -291,6 +272,8 @@ class LRUHandleTable {
 
   size_t GetOccupancyCount() const { return elems_; }
 
+  MemoryAllocator* GetAllocator() const { return allocator_; }
+
  private:
   // Return a pointer to slot that points to a cache entry that
   // matches key/hash.  If there is no such cache entry, return a
@@ -312,6 +295,9 @@ class LRUHandleTable {
 
   // Set from max_upper_hash_bits (see constructor).
   const int max_length_bits_;
+
+  // From Cache, needed for delete
+  MemoryAllocator* const allocator_;
 };
 
 // A single shard of sharded cache.
@@ -321,7 +307,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShardBase {
                 double high_pri_pool_ratio, double low_pri_pool_ratio,
                 bool use_adaptive_mutex,
                 CacheMetadataChargePolicy metadata_charge_policy,
-                int max_upper_hash_bits, SecondaryCache* secondary_cache);
+                int max_upper_hash_bits, MemoryAllocator* allocator,
+                SecondaryCache* secondary_cache);
 
  public:  // Type definitions expected as parameter to ShardedCache
   using HandleImpl = LRUHandle;
@@ -348,26 +335,15 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShardBase {
   void SetLowPriorityPoolRatio(double low_pri_pool_ratio);
 
   // Like Cache methods, but with an extra "hash" parameter.
-  inline Status Insert(const Slice& key, uint32_t hash, void* value,
-                       size_t charge, Cache::DeleterFn deleter,
-                       LRUHandle** handle, Cache::Priority priority) {
-    return Insert(key, hash, value, charge, deleter, nullptr, handle, priority);
-  }
-  inline Status Insert(const Slice& key, uint32_t hash, void* value,
-                       const Cache::CacheItemHelper* helper, size_t charge,
-                       LRUHandle** handle, Cache::Priority priority) {
-    assert(helper);
-    return Insert(key, hash, value, charge, nullptr, helper, handle, priority);
-  }
-  // If helper_cb is null, the values of the following arguments don't matter.
+  Status Insert(const Slice& key, uint32_t hash, Cache::ObjectPtr value,
+                const Cache::CacheItemHelper* helper, size_t charge,
+                LRUHandle** handle, Cache::Priority priority);
+
   LRUHandle* Lookup(const Slice& key, uint32_t hash,
                     const Cache::CacheItemHelper* helper,
-                    const Cache::CreateCallback& create_cb,
+                    Cache::CreateContext* create_context,
                     Cache::Priority priority, bool wait, Statistics* stats);
-  inline LRUHandle* Lookup(const Slice& key, uint32_t hash) {
-    return Lookup(key, hash, nullptr, nullptr, Cache::Priority::LOW, true,
-                  nullptr);
-  }
+
   bool Release(LRUHandle* handle, bool useful, bool erase_if_last_ref);
   bool IsReady(LRUHandle* /*handle*/);
   void Wait(LRUHandle* /*handle*/) {}
@@ -384,8 +360,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShardBase {
   size_t GetTableAddressCount() const;
 
   void ApplyToSomeEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
+      const std::function<void(const Slice& key, Cache::ObjectPtr value,
+                               size_t charge,
+                               const Cache::CacheItemHelper* helper)>& callback,
       size_t average_entries_per_lock, size_t* state);
 
   void EraseUnRefEntries();
@@ -414,9 +391,6 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShardBase {
   // nullptr.
   Status InsertItem(LRUHandle* item, LRUHandle** handle,
                     bool free_handle_on_fail);
-  Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
-                DeleterFn deleter, const Cache::CacheItemHelper* helper,
-                LRUHandle** handle, Cache::Priority priority);
   // Promote an item looked up from the secondary cache to the LRU cache.
   // The item may be still in the secondary cache.
   // It is only inserted into the hash table and not the LRU list, and only
@@ -521,9 +495,9 @@ class LRUCache
                kDontChargeCacheMetadata,
            std::shared_ptr<SecondaryCache> secondary_cache = nullptr);
   const char* Name() const override { return "LRUCache"; }
-  void* Value(Handle* handle) override;
+  ObjectPtr Value(Handle* handle) override;
   size_t GetCharge(Handle* handle) const override;
-  DeleterFn GetDeleter(Handle* handle) const override;
+  const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override;
   void WaitAll(std::vector<Handle*>& handles) override;
 
   // Retrieves number of elements in LRU, for unit test purpose only.

cache/lru_cache_test.cc

@@ -10,6 +10,7 @@
 
 #include "cache/cache_key.h"
 #include "cache/clock_cache.h"
+#include "cache_helpers.h"
 #include "db/db_test_util.h"
 #include "file/sst_file_manager_impl.h"
 #include "port/port.h"
@@ -19,6 +20,7 @@
 #include "rocksdb/sst_file_manager.h"
 #include "rocksdb/utilities/cache_dump_load.h"
 #include "test_util/testharness.h"
+#include "typed_cache.h"
 #include "util/coding.h"
 #include "util/random.h"
 #include "utilities/cache_dump_load_impl.h"
@@ -49,14 +51,15 @@ class LRUCacheTest : public testing::Test {
                                high_pri_pool_ratio, low_pri_pool_ratio,
                                use_adaptive_mutex, kDontChargeCacheMetadata,
                                /*max_upper_hash_bits=*/24,
+                               /*allocator*/ nullptr,
                                /*secondary_cache=*/nullptr);
   }
 
   void Insert(const std::string& key,
               Cache::Priority priority = Cache::Priority::LOW) {
-    EXPECT_OK(cache_->Insert(key, 0 /*hash*/, nullptr /*value*/, 1 /*charge*/,
-                             nullptr /*deleter*/, nullptr /*handle*/,
-                             priority));
+    EXPECT_OK(cache_->Insert(key, 0 /*hash*/, nullptr /*value*/,
+                             &kNoopCacheItemHelper, 1 /*charge*/,
+                             nullptr /*handle*/, priority));
   }
 
   void Insert(char key, Cache::Priority priority = Cache::Priority::LOW) {
@@ -64,7 +67,8 @@ class LRUCacheTest : public testing::Test {
   }
 
   bool Lookup(const std::string& key) {
-    auto handle = cache_->Lookup(key, 0 /*hash*/);
+    auto handle = cache_->Lookup(key, 0 /*hash*/, nullptr, nullptr,
+                                 Cache::Priority::LOW, true, nullptr);
     if (handle) {
       cache_->Release(handle, true /*useful*/, false /*erase*/);
       return true;
@@ -389,15 +393,15 @@ class ClockCacheTest : public testing::Test {
 
     Table::Opts opts;
     opts.estimated_value_size = 1;
-    new (shard_)
-        Shard(capacity, strict_capacity_limit, kDontChargeCacheMetadata, opts);
+    new (shard_) Shard(capacity, strict_capacity_limit,
+                       kDontChargeCacheMetadata, /*allocator*/ nullptr, opts);
   }
 
   Status Insert(const UniqueId64x2& hashed_key,
                 Cache::Priority priority = Cache::Priority::LOW) {
     return shard_->Insert(TestKey(hashed_key), hashed_key, nullptr /*value*/,
-                          1 /*charge*/, nullptr /*deleter*/, nullptr /*handle*/,
-                          priority);
+                          &kNoopCacheItemHelper, 1 /*charge*/,
+                          nullptr /*handle*/, priority);
   }
 
   Status Insert(char key, Cache::Priority priority = Cache::Priority::LOW) {
@@ -407,8 +411,8 @@ class ClockCacheTest : public testing::Test {
   Status InsertWithLen(char key, size_t len) {
     std::string skey(len, key);
     return shard_->Insert(skey, TestHashedKey(key), nullptr /*value*/,
-                          1 /*charge*/, nullptr /*deleter*/, nullptr /*handle*/,
-                          Cache::Priority::LOW);
+                          &kNoopCacheItemHelper, 1 /*charge*/,
+                          nullptr /*handle*/, Cache::Priority::LOW);
   }
 
   bool Lookup(const Slice& key, const UniqueId64x2& hashed_key,
@@ -482,7 +486,7 @@ TEST_F(ClockCacheTest, Limits) {
     // Single entry charge beyond capacity
     {
       Status s = shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/,
-                                5 /*charge*/, nullptr /*deleter*/,
+                                &kNoopCacheItemHelper, 5 /*charge*/,
                                 nullptr /*handle*/, Cache::Priority::LOW);
       if (strict_capacity_limit) {
         EXPECT_TRUE(s.IsMemoryLimit());
@@ -495,7 +499,7 @@ TEST_F(ClockCacheTest, Limits) {
     {
       HandleImpl* h;
       ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/,
-                               3 /*charge*/, nullptr /*deleter*/, &h,
+                               &kNoopCacheItemHelper, 3 /*charge*/, &h,
                                Cache::Priority::LOW));
       // Try to insert more
       Status s = Insert('a');
@@ -519,8 +523,9 @@ TEST_F(ClockCacheTest, Limits) {
       for (size_t i = 0; i < n && s.ok(); ++i) {
         hkey[1] = i;
         s = shard_->Insert(TestKey(hkey), hkey, nullptr /*value*/,
-                           (i + kCapacity < n) ? 0 : 1 /*charge*/,
-                           nullptr /*deleter*/, &ha[i], Cache::Priority::LOW);
+                           &kNoopCacheItemHelper,
+                           (i + kCapacity < n) ? 0 : 1 /*charge*/, &ha[i],
+                           Cache::Priority::LOW);
         if (i == 0) {
           EXPECT_OK(s);
         }
@@ -658,18 +663,25 @@ TEST_F(ClockCacheTest, ClockEvictionTest) {
   }
 }
 
-void IncrementIntDeleter(const Slice& /*key*/, void* value) {
-  *reinterpret_cast<int*>(value) += 1;
-}
+namespace {
+struct DeleteCounter {
+  int deleted = 0;
+};
+const Cache::CacheItemHelper kDeleteCounterHelper{
+    CacheEntryRole::kMisc,
+    [](Cache::ObjectPtr value, MemoryAllocator* /*alloc*/) {
+      static_cast<DeleteCounter*>(value)->deleted += 1;
+    }};
+}  // namespace
 
 // Testing calls to CorrectNearOverflow in Release
 TEST_F(ClockCacheTest, ClockCounterOverflowTest) {
   NewShard(6, /*strict_capacity_limit*/ false);
   HandleImpl* h;
-  int deleted = 0;
+  DeleteCounter val;
   UniqueId64x2 hkey = TestHashedKey('x');
-  ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, &deleted, 1,
-                           IncrementIntDeleter, &h, Cache::Priority::HIGH));
+  ASSERT_OK(shard_->Insert(TestKey(hkey), hkey, &val, &kDeleteCounterHelper, 1,
+                           &h, Cache::Priority::HIGH));
 
   // Some large number outstanding
   shard_->TEST_RefN(h, 123456789);
@@ -689,18 +701,18 @@ TEST_F(ClockCacheTest, ClockCounterOverflowTest) {
   // Free all but last 1
   shard_->TEST_ReleaseN(h, 123456789);
   // Still alive
-  ASSERT_EQ(deleted, 0);
+  ASSERT_EQ(val.deleted, 0);
   // Free last ref, which will finalize erasure
   shard_->Release(h);
   // Deleted
-  ASSERT_EQ(deleted, 1);
+  ASSERT_EQ(val.deleted, 1);
 }
 
 // This test is mostly to exercise some corner case logic, by forcing two
 // keys to have the same hash, and more
 TEST_F(ClockCacheTest, CollidingInsertEraseTest) {
   NewShard(6, /*strict_capacity_limit*/ false);
-  int deleted = 0;
+  DeleteCounter val;
   UniqueId64x2 hkey1 = TestHashedKey('x');
   Slice key1 = TestKey(hkey1);
   UniqueId64x2 hkey2 = TestHashedKey('y');
@@ -708,13 +720,13 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) {
   UniqueId64x2 hkey3 = TestHashedKey('z');
   Slice key3 = TestKey(hkey3);
   HandleImpl* h1;
-  ASSERT_OK(shard_->Insert(key1, hkey1, &deleted, 1, IncrementIntDeleter, &h1,
+  ASSERT_OK(shard_->Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, &h1,
                            Cache::Priority::HIGH));
   HandleImpl* h2;
-  ASSERT_OK(shard_->Insert(key2, hkey2, &deleted, 1, IncrementIntDeleter, &h2,
+  ASSERT_OK(shard_->Insert(key2, hkey2, &val, &kDeleteCounterHelper, 1, &h2,
                            Cache::Priority::HIGH));
   HandleImpl* h3;
-  ASSERT_OK(shard_->Insert(key3, hkey3, &deleted, 1, IncrementIntDeleter, &h3,
+  ASSERT_OK(shard_->Insert(key3, hkey3, &val, &kDeleteCounterHelper, 1, &h3,
                            Cache::Priority::HIGH));
 
   // Can repeatedly lookup+release despite the hash collision
@@ -739,7 +751,7 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) {
   shard_->Erase(key1, hkey1);
 
   // All still alive
-  ASSERT_EQ(deleted, 0);
+  ASSERT_EQ(val.deleted, 0);
 
   // Invisible to Lookup
   tmp_h = shard_->Lookup(key1, hkey1);
@@ -757,8 +769,8 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) {
   }
 
   // Also Insert with invisible entry there
-  ASSERT_OK(shard_->Insert(key1, hkey1, &deleted, 1, IncrementIntDeleter,
-                           nullptr, Cache::Priority::HIGH));
+  ASSERT_OK(shard_->Insert(key1, hkey1, &val, &kDeleteCounterHelper, 1, nullptr,
+                           Cache::Priority::HIGH));
   tmp_h = shard_->Lookup(key1, hkey1);
   // Found but distinct handle
   ASSERT_NE(nullptr, tmp_h);
@@ -766,13 +778,13 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) {
   ASSERT_TRUE(shard_->Release(tmp_h, /*erase_if_last_ref*/ true));
 
   // tmp_h deleted
-  ASSERT_EQ(deleted--, 1);
+  ASSERT_EQ(val.deleted--, 1);
 
   // Release last ref on h1 (already invisible)
   ASSERT_TRUE(shard_->Release(h1, /*erase_if_last_ref*/ false));
 
   // h1 deleted
-  ASSERT_EQ(deleted--, 1);
+  ASSERT_EQ(val.deleted--, 1);
   h1 = nullptr;
 
   // Can still find h2, h3
@@ -790,7 +802,7 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) {
   ASSERT_FALSE(shard_->Release(h2, /*erase_if_last_ref*/ false));
 
   // h2 still not deleted (unreferenced in cache)
-  ASSERT_EQ(deleted, 0);
+  ASSERT_EQ(val.deleted, 0);
 
   // Can still find it
   tmp_h = shard_->Lookup(key2, hkey2);
@@ -800,7 +812,7 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) {
   ASSERT_TRUE(shard_->Release(h2, /*erase_if_last_ref*/ true));
 
   // h2 deleted
-  ASSERT_EQ(deleted--, 1);
+  ASSERT_EQ(val.deleted--, 1);
   tmp_h = shard_->Lookup(key2, hkey2);
   ASSERT_EQ(nullptr, tmp_h);
 
@@ -815,13 +827,13 @@ TEST_F(ClockCacheTest, CollidingInsertEraseTest) {
   ASSERT_FALSE(shard_->Release(h3, /*erase_if_last_ref*/ false));
 
   // h3 still not deleted (unreferenced in cache)
-  ASSERT_EQ(deleted, 0);
+  ASSERT_EQ(val.deleted, 0);
 
   // Explicit erase
   shard_->Erase(key3, hkey3);
 
   // h3 deleted
-  ASSERT_EQ(deleted--, 1);
+  ASSERT_EQ(val.deleted--, 1);
   tmp_h = shard_->Lookup(key3, hkey3);
   ASSERT_EQ(nullptr, tmp_h);
 }
@@ -884,12 +896,12 @@ class TestSecondaryCache : public SecondaryCache {
   using ResultMap = std::unordered_map<std::string, ResultType>;
 
   explicit TestSecondaryCache(size_t capacity)
-      : num_inserts_(0), num_lookups_(0), inject_failure_(false) {
-    cache_ =
-        NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */, nullptr,
-                    kDefaultToAdaptiveMutex, kDontChargeCacheMetadata);
-  }
-  ~TestSecondaryCache() override { cache_.reset(); }
+      : cache_(NewLRUCache(capacity, 0, false, 0.5 /* high_pri_pool_ratio */,
+                           nullptr, kDefaultToAdaptiveMutex,
+                           kDontChargeCacheMetadata)),
+        num_inserts_(0),
+        num_lookups_(0),
+        inject_failure_(false) {}
 
   const char* Name() const override { return "TestSecondaryCache"; }
 
@@ -897,7 +909,7 @@ class TestSecondaryCache : public SecondaryCache {
 
   void ResetInjectFailure() { inject_failure_ = false; }
 
-  Status Insert(const Slice& key, void* value,
+  Status Insert(const Slice& key, Cache::ObjectPtr value,
                 const Cache::CacheItemHelper* helper) override {
     if (inject_failure_) {
       return Status::Corruption("Insertion Data Corrupted");
@@ -916,14 +928,12 @@ class TestSecondaryCache : public SecondaryCache {
       delete[] buf;
       return s;
     }
-    return cache_->Insert(key, buf, size,
-                          [](const Slice& /*key*/, void* val) -> void {
-                            delete[] static_cast<char*>(val);
-                          });
+    return cache_.Insert(key, buf, size);
   }
 
   std::unique_ptr<SecondaryCacheResultHandle> Lookup(
-      const Slice& key, const Cache::CreateCallback& create_cb, bool /*wait*/,
+      const Slice& key, const Cache::CacheItemHelper* helper,
+      Cache::CreateContext* create_context, bool /*wait*/,
       bool /*advise_erase*/, bool& is_in_sec_cache) override {
     std::string key_str = key.ToString();
     TEST_SYNC_POINT_CALLBACK("TestSecondaryCache::Lookup", &key_str);
@@ -939,24 +949,25 @@ class TestSecondaryCache : public SecondaryCache {
       return secondary_handle;
     }
 
-    Cache::Handle* handle = cache_->Lookup(key);
+    TypedHandle* handle = cache_.Lookup(key);
     num_lookups_++;
     if (handle) {
-      void* value = nullptr;
+      Cache::ObjectPtr value = nullptr;
       size_t charge = 0;
       Status s;
       if (type != ResultType::DEFER_AND_FAIL) {
-        char* ptr = (char*)cache_->Value(handle);
+        char* ptr = cache_.Value(handle);
         size_t size = DecodeFixed64(ptr);
         ptr += sizeof(uint64_t);
-        s = create_cb(ptr, size, &value, &charge);
+        s = helper->create_cb(Slice(ptr, size), create_context,
+                              /*alloc*/ nullptr, &value, &charge);
       }
       if (s.ok()) {
         secondary_handle.reset(new TestSecondaryCacheResultHandle(
             cache_.get(), handle, value, charge, type));
         is_in_sec_cache = true;
       } else {
-        cache_->Release(handle);
+        cache_.Release(handle);
       }
     }
     return secondary_handle;
@@ -995,7 +1006,8 @@ class TestSecondaryCache : public SecondaryCache {
   class TestSecondaryCacheResultHandle : public SecondaryCacheResultHandle {
    public:
     TestSecondaryCacheResultHandle(Cache* cache, Cache::Handle* handle,
-                                   void* value, size_t size, ResultType type)
+                                   Cache::ObjectPtr value, size_t size,
+                                   ResultType type)
         : cache_(cache),
           handle_(handle),
           value_(value),
@@ -1012,7 +1024,7 @@ class TestSecondaryCache : public SecondaryCache {
 
     void Wait() override {}
 
-    void* Value() override {
+    Cache::ObjectPtr Value() override {
       assert(is_ready_);
       return value_;
     }
@@ -1024,12 +1036,15 @@ class TestSecondaryCache : public SecondaryCache {
    private:
     Cache* cache_;
     Cache::Handle* handle_;
-    void* value_;
+    Cache::ObjectPtr value_;
     size_t size_;
     bool is_ready_;
   };
 
-  std::shared_ptr<Cache> cache_;
+  using SharedCache =
+      BasicTypedSharedCacheInterface<char[], CacheEntryRole::kMisc>;
+  using TypedHandle = SharedCache::TypedHandle;
+  SharedCache cache_;
   uint32_t num_inserts_;
   uint32_t num_lookups_;
   bool inject_failure_;
@@ -1049,7 +1064,8 @@ class DBSecondaryCacheTest : public DBTestBase {
   std::unique_ptr<Env> fault_env_;
 };
 
-class LRUCacheSecondaryCacheTest : public LRUCacheTest {
+class LRUCacheSecondaryCacheTest : public LRUCacheTest,
+                                   public Cache::CreateContext {
  public:
   LRUCacheSecondaryCacheTest() : fail_create_(false) {}
   ~LRUCacheSecondaryCacheTest() {}
@@ -1071,13 +1087,13 @@ class LRUCacheSecondaryCacheTest : public LRUCacheTest {
     size_t size_;
   };
 
-  static size_t SizeCallback(void* obj) {
-    return reinterpret_cast<TestItem*>(obj)->Size();
+  static size_t SizeCallback(Cache::ObjectPtr obj) {
+    return static_cast<TestItem*>(obj)->Size();
   }
 
-  static Status SaveToCallback(void* from_obj, size_t from_offset,
-                               size_t length, void* out) {
-    TestItem* item = reinterpret_cast<TestItem*>(from_obj);
+  static Status SaveToCallback(Cache::ObjectPtr from_obj, size_t from_offset,
+                               size_t length, char* out) {
+    TestItem* item = static_cast<TestItem*>(from_obj);
     char* buf = item->Buf();
     EXPECT_EQ(length, item->Size());
     EXPECT_EQ(from_offset, 0);
@@ -1085,27 +1101,30 @@ class LRUCacheSecondaryCacheTest : public LRUCacheTest {
     return Status::OK();
   }
 
-  static void DeletionCallback(const Slice& /*key*/, void* obj) {
-    delete reinterpret_cast<TestItem*>(obj);
+  static void DeletionCallback(Cache::ObjectPtr obj,
+                               MemoryAllocator* /*alloc*/) {
+    delete static_cast<TestItem*>(obj);
   }
 
   static Cache::CacheItemHelper helper_;
 
-  static Status SaveToCallbackFail(void* /*obj*/, size_t /*offset*/,
-                                   size_t /*size*/, void* /*out*/) {
+  static Status SaveToCallbackFail(Cache::ObjectPtr /*from_obj*/,
+                                   size_t /*from_offset*/, size_t /*length*/,
+                                   char* /*out*/) {
     return Status::NotSupported();
   }
 
   static Cache::CacheItemHelper helper_fail_;
 
-  Cache::CreateCallback test_item_creator = [&](const void* buf, size_t size,
-                                                void** out_obj,
-                                                size_t* charge) -> Status {
-    if (fail_create_) {
+  static Status CreateCallback(const Slice& data, Cache::CreateContext* context,
+                               MemoryAllocator* /*allocator*/,
+                               Cache::ObjectPtr* out_obj, size_t* out_charge) {
+    auto t = static_cast<LRUCacheSecondaryCacheTest*>(context);
+    if (t->fail_create_) {
       return Status::NotSupported();
     }
-    *out_obj = reinterpret_cast<void*>(new TestItem((char*)buf, size));
-    *charge = size;
+    *out_obj = new TestItem(data.data(), data.size());
+    *out_charge = data.size();
     return Status::OK();
   };
 
@@ -1115,15 +1134,17 @@ class LRUCacheSecondaryCacheTest : public LRUCacheTest {
   bool fail_create_;
 };
 
-Cache::CacheItemHelper LRUCacheSecondaryCacheTest::helper_(
+Cache::CacheItemHelper LRUCacheSecondaryCacheTest::helper_{
+    CacheEntryRole::kMisc, LRUCacheSecondaryCacheTest::DeletionCallback,
     LRUCacheSecondaryCacheTest::SizeCallback,
     LRUCacheSecondaryCacheTest::SaveToCallback,
-    LRUCacheSecondaryCacheTest::DeletionCallback);
+    LRUCacheSecondaryCacheTest::CreateCallback};
 
-Cache::CacheItemHelper LRUCacheSecondaryCacheTest::helper_fail_(
+Cache::CacheItemHelper LRUCacheSecondaryCacheTest::helper_fail_{
+    CacheEntryRole::kMisc, LRUCacheSecondaryCacheTest::DeletionCallback,
     LRUCacheSecondaryCacheTest::SizeCallback,
     LRUCacheSecondaryCacheTest::SaveToCallbackFail,
-    LRUCacheSecondaryCacheTest::DeletionCallback);
+    LRUCacheSecondaryCacheTest::CreateCallback};
 
 TEST_F(LRUCacheSecondaryCacheTest, BasicTest) {
   LRUCacheOptions opts(1024 /* capacity */, 0 /* num_shard_bits */,
@@ -1159,7 +1180,7 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicTest) {
   Cache::Handle* handle;
   handle =
       cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_,
-                    test_item_creator, Cache::Priority::LOW, true, stats.get());
+                    /*context*/ this, Cache::Priority::LOW, true, stats.get());
   ASSERT_NE(handle, nullptr);
   ASSERT_EQ(static_cast<TestItem*>(cache->Value(handle))->Size(), str2.size());
   cache->Release(handle);
@@ -1167,7 +1188,7 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicTest) {
   // This lookup should promote k1 and demote k2
   handle =
       cache->Lookup(k1.AsSlice(), &LRUCacheSecondaryCacheTest::helper_,
-                    test_item_creator, Cache::Priority::LOW, true, stats.get());
+                    /*context*/ this, Cache::Priority::LOW, true, stats.get());
   ASSERT_NE(handle, nullptr);
   ASSERT_EQ(static_cast<TestItem*>(cache->Value(handle))->Size(), str1.size());
   cache->Release(handle);
@@ -1175,7 +1196,7 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicTest) {
   // This lookup should promote k3 and demote k1
   handle =
       cache->Lookup(k3.AsSlice(), &LRUCacheSecondaryCacheTest::helper_,
-                    test_item_creator, Cache::Priority::LOW, true, stats.get());
+                    /*context*/ this, Cache::Priority::LOW, true, stats.get());
   ASSERT_NE(handle, nullptr);
   ASSERT_EQ(static_cast<TestItem*>(cache->Value(handle))->Size(), str3.size());
   cache->Release(handle);
@@ -1207,18 +1228,19 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicFailTest) {
   Random rnd(301);
   std::string str1 = rnd.RandomString(1020);
   auto item1 = std::make_unique<TestItem>(str1.data(), str1.length());
-  ASSERT_TRUE(cache->Insert(k1.AsSlice(), item1.get(), nullptr, str1.length())
-                  .IsInvalidArgument());
+  // NOTE: changed to assert helper != nullptr for efficiency / code size
+  // ASSERT_TRUE(cache->Insert(k1.AsSlice(), item1.get(), nullptr,
+  //                           str1.length()).IsInvalidArgument());
   ASSERT_OK(cache->Insert(k1.AsSlice(), item1.get(),
                           &LRUCacheSecondaryCacheTest::helper_, str1.length()));
   item1.release();  // Appease clang-analyze "potential memory leak"
 
   Cache::Handle* handle;
-  handle = cache->Lookup(k2.AsSlice(), nullptr, test_item_creator,
+  handle = cache->Lookup(k2.AsSlice(), nullptr, /*context*/ this,
                          Cache::Priority::LOW, true);
   ASSERT_EQ(handle, nullptr);
   handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_,
-                         test_item_creator, Cache::Priority::LOW, false);
+                         /*context*/ this, Cache::Priority::LOW, false);
   ASSERT_EQ(handle, nullptr);
 
   cache.reset();
@@ -1256,18 +1278,18 @@ TEST_F(LRUCacheSecondaryCacheTest, SaveFailTest) {
   Cache::Handle* handle;
   handle =
       cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_fail_,
-                    test_item_creator, Cache::Priority::LOW, true);
+                    /*context*/ this, Cache::Priority::LOW, true);
   ASSERT_NE(handle, nullptr);
   cache->Release(handle);
   // This lookup should fail, since k1 demotion would have failed
   handle =
       cache->Lookup(k1.AsSlice(), &LRUCacheSecondaryCacheTest::helper_fail_,
-                    test_item_creator, Cache::Priority::LOW, true);
+                    /*context*/ this, Cache::Priority::LOW, true);
   ASSERT_EQ(handle, nullptr);
   // Since k1 didn't get promoted, k2 should still be in cache
   handle =
       cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_fail_,
-                    test_item_creator, Cache::Priority::LOW, true);
+                    /*context*/ this, Cache::Priority::LOW, true);
   ASSERT_NE(handle, nullptr);
   cache->Release(handle);
   ASSERT_EQ(secondary_cache->num_inserts(), 1u);
@@ -1304,16 +1326,16 @@ TEST_F(LRUCacheSecondaryCacheTest, CreateFailTest) {
   Cache::Handle* handle;
   SetFailCreate(true);
   handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_,
-                         test_item_creator, Cache::Priority::LOW, true);
+                         /*context*/ this, Cache::Priority::LOW, true);
   ASSERT_NE(handle, nullptr);
   cache->Release(handle);
   // This lookup should fail, since k1 creation would have failed
   handle = cache->Lookup(k1.AsSlice(), &LRUCacheSecondaryCacheTest::helper_,
-                         test_item_creator, Cache::Priority::LOW, true);
+                         /*context*/ this, Cache::Priority::LOW, true);
   ASSERT_EQ(handle, nullptr);
   // Since k1 didn't get promoted, k2 should still be in cache
   handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_,
-                         test_item_creator, Cache::Priority::LOW, true);
+                         /*context*/ this, Cache::Priority::LOW, true);
   ASSERT_NE(handle, nullptr);
   cache->Release(handle);
   ASSERT_EQ(secondary_cache->num_inserts(), 1u);
@@ -1349,19 +1371,19 @@ TEST_F(LRUCacheSecondaryCacheTest, FullCapacityTest) {
 
   Cache::Handle* handle;
   handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_,
-                         test_item_creator, Cache::Priority::LOW, true);
+                         /*context*/ this, Cache::Priority::LOW, true);
   ASSERT_NE(handle, nullptr);
   // k1 promotion should fail due to the block cache being at capacity,
   // but the lookup should still succeed
   Cache::Handle* handle2;
   handle2 = cache->Lookup(k1.AsSlice(), &LRUCacheSecondaryCacheTest::helper_,
-                          test_item_creator, Cache::Priority::LOW, true);
+                          /*context*/ this, Cache::Priority::LOW, true);
   ASSERT_NE(handle2, nullptr);
   // Since k1 didn't get inserted, k2 should still be in cache
   cache->Release(handle);
   cache->Release(handle2);
   handle = cache->Lookup(k2.AsSlice(), &LRUCacheSecondaryCacheTest::helper_,
-                         test_item_creator, Cache::Priority::LOW, true);
+                         /*context*/ this, Cache::Priority::LOW, true);
   ASSERT_NE(handle, nullptr);
   cache->Release(handle);
   ASSERT_EQ(secondary_cache->num_inserts(), 1u);
@@ -1838,7 +1860,7 @@ TEST_F(LRUCacheSecondaryCacheTest, BasicWaitAllTest) {
   for (int i = 0; i < 6; ++i) {
     results.emplace_back(cache->Lookup(
         ock.WithOffset(i).AsSlice(), &LRUCacheSecondaryCacheTest::helper_,
-        test_item_creator, Cache::Priority::LOW, false));
+        /*context*/ this, Cache::Priority::LOW, false));
   }
   cache->WaitAll(results);
   for (int i = 0; i < 6; ++i) {
@@ -1964,26 +1986,18 @@ class LRUCacheWithStat : public LRUCache {
   }
   ~LRUCacheWithStat() {}
 
-  Status Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter,
-                Handle** handle, Priority priority) override {
-    insert_count_++;
-    return LRUCache::Insert(key, value, charge, deleter, handle, priority);
-  }
-  Status Insert(const Slice& key, void* value, const CacheItemHelper* helper,
-                size_t charge, Handle** handle = nullptr,
+  Status Insert(const Slice& key, Cache::ObjectPtr value,
+                const CacheItemHelper* helper, size_t charge,
+                Handle** handle = nullptr,
                 Priority priority = Priority::LOW) override {
     insert_count_++;
     return LRUCache::Insert(key, value, helper, charge, handle, priority);
   }
-  Handle* Lookup(const Slice& key, Statistics* stats) override {
-    lookup_count_++;
-    return LRUCache::Lookup(key, stats);
-  }
   Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
-                 const CreateCallback& create_cb, Priority priority, bool wait,
+                 CreateContext* create_context, Priority priority, bool wait,
                  Statistics* stats = nullptr) override {
     lookup_count_++;
-    return LRUCache::Lookup(key, helper, create_cb, priority, wait, stats);
+    return LRUCache::Lookup(key, helper, create_context, priority, wait, stats);
   }
 
   uint32_t GetInsertCount() { return insert_count_; }

cache/secondary_cache.cc

@@ -11,20 +11,29 @@ namespace ROCKSDB_NAMESPACE {
 
 namespace {
 
-size_t SliceSize(void* obj) { return static_cast<Slice*>(obj)->size(); }
+void NoopDelete(Cache::ObjectPtr, MemoryAllocator*) {}
 
-Status SliceSaveTo(void* from_obj, size_t from_offset, size_t length,
-                   void* out) {
+size_t SliceSize(Cache::ObjectPtr obj) {
+  return static_cast<Slice*>(obj)->size();
+}
+
+Status SliceSaveTo(Cache::ObjectPtr from_obj, size_t from_offset, size_t length,
+                   char* out) {
   const Slice& slice = *static_cast<Slice*>(from_obj);
   std::memcpy(out, slice.data() + from_offset, length);
   return Status::OK();
 }
 
+Status FailCreate(const Slice&, Cache::CreateContext*, MemoryAllocator*,
+                  Cache::ObjectPtr*, size_t*) {
+  return Status::NotSupported("Only for dumping data into SecondaryCache");
+}
+
 }  // namespace
 
 Status SecondaryCache::InsertSaved(const Slice& key, const Slice& saved) {
-  static Cache::CacheItemHelper helper{
-      &SliceSize, &SliceSaveTo, GetNoopDeleterForRole<CacheEntryRole::kMisc>()};
+  static Cache::CacheItemHelper helper{CacheEntryRole::kMisc, &NoopDelete,
+                                       &SliceSize, &SliceSaveTo, &FailCreate};
   // NOTE: depends on Insert() being synchronous, not keeping pointer `&saved`
   return Insert(key, const_cast<Slice*>(&saved), &helper);
 }

cache/sharded_cache.h

@@ -49,16 +49,12 @@ class CacheShardBase {
     HashCref GetHash() const;
     ...
   };
-  Status Insert(const Slice& key, HashCref hash, void* value, size_t charge,
-                DeleterFn deleter, HandleImpl** handle,
-                Cache::Priority priority) = 0;
-  Status Insert(const Slice& key, HashCref hash, void* value,
+  Status Insert(const Slice& key, HashCref hash, Cache::ObjectPtr value,
                 const Cache::CacheItemHelper* helper, size_t charge,
                 HandleImpl** handle, Cache::Priority priority) = 0;
-  HandleImpl* Lookup(const Slice& key, HashCref hash) = 0;
   HandleImpl* Lookup(const Slice& key, HashCref hash,
                         const Cache::CacheItemHelper* helper,
-                        const Cache::CreateCallback& create_cb,
+                        Cache::CreateContext* create_context,
                         Cache::Priority priority, bool wait,
                         Statistics* stats) = 0;
   bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref) = 0;
@@ -77,8 +73,9 @@ class CacheShardBase {
   // *state == 0 and implementation sets *state = SIZE_MAX to indicate
   // completion.
   void ApplyToSomeEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
+      const std::function<void(const Slice& key, ObjectPtr value,
+                               size_t charge,
+                               const Cache::CacheItemHelper* helper)>& callback,
       size_t average_entries_per_lock, size_t* state) = 0;
   void EraseUnRefEntries() = 0;
   */
@@ -172,36 +169,24 @@ class ShardedCache : public ShardedCacheBase {
         [s_c_l](CacheShard* cs) { cs->SetStrictCapacityLimit(s_c_l); });
   }
 
-  Status Insert(const Slice& key, void* value, size_t charge, DeleterFn deleter,
-                Handle** handle, Priority priority) override {
-    HashVal hash = CacheShard::ComputeHash(key);
-    auto h_out = reinterpret_cast<HandleImpl**>(handle);
-    return GetShard(hash).Insert(key, hash, value, charge, deleter, h_out,
-                                 priority);
-  }
-  Status Insert(const Slice& key, void* value, const CacheItemHelper* helper,
-                size_t charge, Handle** handle = nullptr,
+  Status Insert(const Slice& key, ObjectPtr value,
+                const CacheItemHelper* helper, size_t charge,
+                Handle** handle = nullptr,
                 Priority priority = Priority::LOW) override {
-    if (!helper) {
-      return Status::InvalidArgument();
-    }
+    assert(helper);
     HashVal hash = CacheShard::ComputeHash(key);
     auto h_out = reinterpret_cast<HandleImpl**>(handle);
     return GetShard(hash).Insert(key, hash, value, helper, charge, h_out,
                                  priority);
   }
 
-  Handle* Lookup(const Slice& key, Statistics* /*stats*/) override {
-    HashVal hash = CacheShard::ComputeHash(key);
-    HandleImpl* result = GetShard(hash).Lookup(key, hash);
-    return reinterpret_cast<Handle*>(result);
-  }
-  Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
-                 const CreateCallback& create_cb, Priority priority, bool wait,
+  Handle* Lookup(const Slice& key, const CacheItemHelper* helper = nullptr,
+                 CreateContext* create_context = nullptr,
+                 Priority priority = Priority::LOW, bool wait = true,
                  Statistics* stats = nullptr) override {
     HashVal hash = CacheShard::ComputeHash(key);
-    HandleImpl* result = GetShard(hash).Lookup(key, hash, helper, create_cb,
-                                               priority, wait, stats);
+    HandleImpl* result = GetShard(hash).Lookup(
+        key, hash, helper, create_context, priority, wait, stats);
     return reinterpret_cast<Handle*>(result);
   }
 
@@ -244,8 +229,8 @@ class ShardedCache : public ShardedCacheBase {
     return SumOverShards2(&CacheShard::GetTableAddressCount);
   }
   void ApplyToAllEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
+      const std::function<void(const Slice& key, ObjectPtr value, size_t charge,
+                               const CacheItemHelper* helper)>& callback,
       const ApplyToAllEntriesOptions& opts) override {
     uint32_t num_shards = GetNumShards();
     // Iterate over part of each shard, rotating between shards, to

cache/typed_cache.h

@@ -0,0 +1,339 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+// APIs for accessing Cache in a type-safe and convenient way. Cache is kept
+// at a low, thin level of abstraction so that different implementations can
+// be plugged in, but these wrappers provide clean, convenient access to the
+// most common operations.
+//
+// A number of template classes are needed for sharing common structure. The
+// key classes are these:
+//
+// * PlaceholderCacheInterface - Used for making cache reservations, with
+// entries that have a charge but no value.
+// * BasicTypedCacheInterface<TValue> - Used for primary cache storage of
+// objects of type TValue.
+// * FullTypedCacheHelper<TValue, TCreateContext> - Used for secondary cache
+// compatible storage of objects of type TValue.
+// * For each of these, there's a "Shared" version
+// (e.g. FullTypedSharedCacheInterface) that holds a shared_ptr to the Cache,
+// rather than assuming external ownership by holding only a raw `Cache*`.
+
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+#include <memory>
+#include <type_traits>
+
+#include "cache/cache_helpers.h"
+#include "rocksdb/advanced_options.h"
+#include "rocksdb/cache.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// For future consideration:
+// * Pass in value to Insert with std::unique_ptr& to simplify ownership
+//   transfer logic in callers
+// * Make key type a template parameter (e.g. useful for table cache)
+// * Closer integration with CacheHandleGuard (opt-in, so not always
+//   paying the extra overhead)
+
+#define CACHE_TYPE_DEFS()                     \
+  using Priority = Cache::Priority;           \
+  using Handle = Cache::Handle;               \
+  using ObjectPtr = Cache::ObjectPtr;         \
+  using CreateContext = Cache::CreateContext; \
+  using CacheItemHelper = Cache::CacheItemHelper /* caller ; */
+
+template <typename CachePtr>
+class BaseCacheInterface {
+ public:
+  CACHE_TYPE_DEFS();
+
+  /*implicit*/ BaseCacheInterface(CachePtr cache) : cache_(std::move(cache)) {}
+
+  inline void Release(Handle* handle) { cache_->Release(handle); }
+
+  inline void ReleaseAndEraseIfLastRef(Handle* handle) {
+    cache_->Release(handle, /*erase_if_last_ref*/ true);
+  }
+
+  inline void RegisterReleaseAsCleanup(Handle* handle, Cleanable& cleanable) {
+    cleanable.RegisterCleanup(&ReleaseCacheHandleCleanup, get(), handle);
+  }
+
+  inline Cache* get() const { return &*cache_; }
+
+  explicit inline operator bool() const noexcept { return cache_ != nullptr; }
+
+ protected:
+  CachePtr cache_;
+};
+
+// PlaceholderCacheInterface - Used for making cache reservations, with
+// entries that have a charge but no value. CacheEntryRole is required as
+// a template parameter.
+template <CacheEntryRole kRole, typename CachePtr = Cache*>
+class PlaceholderCacheInterface : public BaseCacheInterface<CachePtr> {
+ public:
+  CACHE_TYPE_DEFS();
+  using BaseCacheInterface<CachePtr>::BaseCacheInterface;
+
+  inline Status Insert(const Slice& key, size_t charge, Handle** handle) {
+    return this->cache_->Insert(key, /*value=*/nullptr, &kHelper, charge,
+                                handle);
+  }
+
+  static constexpr Cache::CacheItemHelper kHelper{kRole};
+};
+
+template <CacheEntryRole kRole>
+using PlaceholderSharedCacheInterface =
+    PlaceholderCacheInterface<kRole, std::shared_ptr<Cache>>;
+
+template <class TValue>
+class BasicTypedCacheHelperFns {
+ public:
+  CACHE_TYPE_DEFS();
+  // E.g. char* for char[]
+  using TValuePtr = std::remove_extent_t<TValue>*;
+
+ protected:
+  inline static ObjectPtr UpCastValue(TValuePtr value) { return value; }
+  inline static TValuePtr DownCastValue(ObjectPtr value) {
+    return static_cast<TValuePtr>(value);
+  }
+
+  static void Delete(ObjectPtr value, MemoryAllocator* allocator) {
+    // FIXME: Currently, no callers actually allocate the ObjectPtr objects
+    // using the custom allocator, just subobjects that keep a reference to
+    // the allocator themselves (with CacheAllocationPtr).
+    if (/*DISABLED*/ false && allocator) {
+      if constexpr (std::is_destructible_v<TValue>) {
+        DownCastValue(value)->~TValue();
+      }
+      allocator->Deallocate(value);
+    } else {
+      // Like delete but properly handles TValue=char[] etc.
+      std::default_delete<TValue>{}(DownCastValue(value));
+    }
+  }
+};
+
+// In its own class to try to minimize the number of distinct CacheItemHelper
+// instances (e.g. don't vary by CachePtr)
+template <class TValue, CacheEntryRole kRole>
+class BasicTypedCacheHelper : public BasicTypedCacheHelperFns<TValue> {
+ public:
+  static constexpr Cache::CacheItemHelper kBasicHelper{
+      kRole, &BasicTypedCacheHelper::Delete};
+};
+
+// BasicTypedCacheInterface - Used for primary cache storage of objects of
+// type TValue, which can be cleaned up with std::default_delete<TValue>. The
+// role is provided by TValue::kCacheEntryRole or given in an optional
+// template parameter.
+template <class TValue, CacheEntryRole kRole = TValue::kCacheEntryRole,
+          typename CachePtr = Cache*>
+class BasicTypedCacheInterface : public BaseCacheInterface<CachePtr>,
+                                 public BasicTypedCacheHelper<TValue, kRole> {
+ public:
+  CACHE_TYPE_DEFS();
+  using typename BasicTypedCacheHelperFns<TValue>::TValuePtr;
+  struct TypedHandle : public Handle {};
+  using BasicTypedCacheHelper<TValue, kRole>::kBasicHelper;
+  // ctor
+  using BaseCacheInterface<CachePtr>::BaseCacheInterface;
+
+  inline Status Insert(const Slice& key, TValuePtr value, size_t charge,
+                       TypedHandle** handle = nullptr,
+                       Priority priority = Priority::LOW) {
+    auto untyped_handle = reinterpret_cast<Handle**>(handle);
+    return this->cache_->Insert(
+        key, BasicTypedCacheHelperFns<TValue>::UpCastValue(value),
+        &kBasicHelper, charge, untyped_handle, priority);
+  }
+
+  inline TypedHandle* Lookup(const Slice& key, Statistics* stats = nullptr) {
+    return reinterpret_cast<TypedHandle*>(
+        this->cache_->BasicLookup(key, stats));
+  }
+
+  inline CacheHandleGuard<TValue> Guard(TypedHandle* handle) {
+    if (handle) {
+      return CacheHandleGuard<TValue>(&*this->cache_, handle);
+    } else {
+      return {};
+    }
+  }
+
+  inline std::shared_ptr<TValue> SharedGuard(TypedHandle* handle) {
+    if (handle) {
+      return MakeSharedCacheHandleGuard<TValue>(&*this->cache_, handle);
+    } else {
+      return {};
+    }
+  }
+
+  inline TValuePtr Value(TypedHandle* handle) {
+    return BasicTypedCacheHelperFns<TValue>::DownCastValue(
+        this->cache_->Value(handle));
+  }
+};
+
+// BasicTypedSharedCacheInterface - Like BasicTypedCacheInterface but with a
+// shared_ptr<Cache> for keeping Cache alive.
+template <class TValue, CacheEntryRole kRole = TValue::kCacheEntryRole>
+using BasicTypedSharedCacheInterface =
+    BasicTypedCacheInterface<TValue, kRole, std::shared_ptr<Cache>>;
+
+// TValue must implement ContentSlice() and ~TValue
+// TCreateContext must implement Create(std::unique_ptr<TValue>*, ...)
+template <class TValue, class TCreateContext>
+class FullTypedCacheHelperFns : public BasicTypedCacheHelperFns<TValue> {
+ public:
+  CACHE_TYPE_DEFS();
+
+ protected:
+  using typename BasicTypedCacheHelperFns<TValue>::TValuePtr;
+  using BasicTypedCacheHelperFns<TValue>::DownCastValue;
+  using BasicTypedCacheHelperFns<TValue>::UpCastValue;
+
+  static size_t Size(ObjectPtr v) {
+    TValuePtr value = DownCastValue(v);
+    auto slice = value->ContentSlice();
+    return slice.size();
+  }
+
+  static Status SaveTo(ObjectPtr v, size_t from_offset, size_t length,
+                       char* out) {
+    TValuePtr value = DownCastValue(v);
+    auto slice = value->ContentSlice();
+    assert(from_offset < slice.size());
+    assert(from_offset + length <= slice.size());
+    std::copy_n(slice.data() + from_offset, length, out);
+    return Status::OK();
+  }
+
+  static Status Create(const Slice& data, CreateContext* context,
+                       MemoryAllocator* allocator, ObjectPtr* out_obj,
+                       size_t* out_charge) {
+    std::unique_ptr<TValue> value = nullptr;
+    if constexpr (sizeof(TCreateContext) > 0) {
+      TCreateContext* tcontext = static_cast<TCreateContext*>(context);
+      tcontext->Create(&value, out_charge, data, allocator);
+    } else {
+      TCreateContext::Create(&value, out_charge, data, allocator);
+    }
+    *out_obj = UpCastValue(value.release());
+    return Status::OK();
+  }
+};
+
+// In its own class to try to minimize the number of distinct CacheItemHelper
+// instances (e.g. don't vary by CachePtr)
+template <class TValue, class TCreateContext, CacheEntryRole kRole>
+class FullTypedCacheHelper
+    : public FullTypedCacheHelperFns<TValue, TCreateContext> {
+ public:
+  static constexpr Cache::CacheItemHelper kFullHelper{
+      kRole, &FullTypedCacheHelper::Delete, &FullTypedCacheHelper::Size,
+      &FullTypedCacheHelper::SaveTo, &FullTypedCacheHelper::Create};
+};
+
+// FullTypedCacheHelper - Used for secondary cache compatible storage of
+// objects of type TValue. In addition to BasicTypedCacheInterface constraints,
+// we require TValue::ContentSlice() to return persistable data. This
+// simplifies usage for the normal case of simple secondary cache compatibility
+// (can give you a Slice to the data already in memory). In addition to
+// TCreateContext performing the role of Cache::CreateContext, it is also
+// expected to provide a function Create(std::unique_ptr<TValue>* value,
+// size_t* out_charge, const Slice& data, MemoryAllocator* allocator) for
+// creating new TValue.
+template <class TValue, class TCreateContext,
+          CacheEntryRole kRole = TValue::kCacheEntryRole,
+          typename CachePtr = Cache*>
+class FullTypedCacheInterface
+    : public BasicTypedCacheInterface<TValue, kRole, CachePtr>,
+      public FullTypedCacheHelper<TValue, TCreateContext, kRole> {
+ public:
+  CACHE_TYPE_DEFS();
+  using typename BasicTypedCacheInterface<TValue, kRole, CachePtr>::TypedHandle;
+  using typename BasicTypedCacheHelperFns<TValue>::TValuePtr;
+  using BasicTypedCacheHelper<TValue, kRole>::kBasicHelper;
+  using FullTypedCacheHelper<TValue, TCreateContext, kRole>::kFullHelper;
+  using BasicTypedCacheHelperFns<TValue>::UpCastValue;
+  using BasicTypedCacheHelperFns<TValue>::DownCastValue;
+  // ctor
+  using BasicTypedCacheInterface<TValue, kRole,
+                                 CachePtr>::BasicTypedCacheInterface;
+
+  // Insert with SecondaryCache compatibility (subject to CacheTier).
+  // (Basic Insert() also inherited.)
+  inline Status InsertFull(
+      const Slice& key, TValuePtr value, size_t charge,
+      TypedHandle** handle = nullptr, Priority priority = Priority::LOW,
+      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) {
+    auto untyped_handle = reinterpret_cast<Handle**>(handle);
+    auto helper = lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier
+                      ? &kFullHelper
+                      : &kBasicHelper;
+    return this->cache_->Insert(key, UpCastValue(value), helper, charge,
+                                untyped_handle, priority);
+  }
+
+  // Like SecondaryCache::InsertSaved, with SecondaryCache compatibility
+  // (subject to CacheTier).
+  inline Status InsertSaved(
+      const Slice& key, const Slice& data, TCreateContext* create_context,
+      Priority priority = Priority::LOW,
+      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier,
+      size_t* out_charge = nullptr) {
+    ObjectPtr value;
+    size_t charge;
+    Status st = kFullHelper.create_cb(data, create_context,
+                                      this->cache_->memory_allocator(), &value,
+                                      &charge);
+    if (out_charge) {
+      *out_charge = charge;
+    }
+    if (st.ok()) {
+      st = InsertFull(key, DownCastValue(value), charge, nullptr /*handle*/,
+                      priority, lowest_used_cache_tier);
+    } else {
+      kFullHelper.del_cb(value, this->cache_->memory_allocator());
+    }
+    return st;
+  }
+
+  // Lookup with SecondaryCache support (subject to CacheTier).
+  // (Basic Lookup() also inherited.)
+  inline TypedHandle* LookupFull(
+      const Slice& key, TCreateContext* create_context = nullptr,
+      Priority priority = Priority::LOW, bool wait = true,
+      Statistics* stats = nullptr,
+      CacheTier lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier) {
+    if (lowest_used_cache_tier == CacheTier::kNonVolatileBlockTier) {
+      return reinterpret_cast<TypedHandle*>(this->cache_->Lookup(
+          key, &kFullHelper, create_context, priority, wait, stats));
+    } else {
+      return BasicTypedCacheInterface<TValue, kRole, CachePtr>::Lookup(key,
+                                                                       stats);
+    }
+  }
+};
+
+// FullTypedSharedCacheInterface - Like FullTypedCacheInterface but with a
+// shared_ptr<Cache> for keeping Cache alive.
+template <class TValue, class TCreateContext,
+          CacheEntryRole kRole = TValue::kCacheEntryRole>
+using FullTypedSharedCacheInterface =
+    FullTypedCacheInterface<TValue, TCreateContext, kRole,
+                            std::shared_ptr<Cache>>;
+
+#undef CACHE_TYPE_DEFS
+
+}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_contents.cc

@@ -13,12 +13,6 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-std::unique_ptr<BlobContents> BlobContents::Create(
-    CacheAllocationPtr&& allocation, size_t size) {
-  return std::unique_ptr<BlobContents>(
-      new BlobContents(std::move(allocation), size));
-}
-
 size_t BlobContents::ApproximateMemoryUsage() const {
   size_t usage = 0;
 
@@ -45,46 +39,4 @@ size_t BlobContents::ApproximateMemoryUsage() const {
   return usage;
 }
 
-size_t BlobContents::SizeCallback(void* obj) {
-  assert(obj);
-
-  return static_cast<const BlobContents*>(obj)->size();
-}
-
-Status BlobContents::SaveToCallback(void* from_obj, size_t from_offset,
-                                    size_t length, void* out) {
-  assert(from_obj);
-
-  const BlobContents* buf = static_cast<const BlobContents*>(from_obj);
-  assert(buf->size() >= from_offset + length);
-
-  memcpy(out, buf->data().data() + from_offset, length);
-
-  return Status::OK();
-}
-
-Cache::CacheItemHelper* BlobContents::GetCacheItemHelper() {
-  static Cache::CacheItemHelper cache_helper(
-      &SizeCallback, &SaveToCallback,
-      GetCacheEntryDeleterForRole<BlobContents, CacheEntryRole::kBlobValue>());
-
-  return &cache_helper;
-}
-
-Status BlobContents::CreateCallback(CacheAllocationPtr&& allocation,
-                                    const void* buf, size_t size,
-                                    void** out_obj, size_t* charge) {
-  assert(allocation);
-
-  memcpy(allocation.get(), buf, size);
-
-  std::unique_ptr<BlobContents> obj = Create(std::move(allocation), size);
-  BlobContents* const contents = obj.release();
-
-  *out_obj = contents;
-  *charge = contents->ApproximateMemoryUsage();
-
-  return Status::OK();
-}
-
 }  // namespace ROCKSDB_NAMESPACE

db/blob/blob_contents.h

@@ -18,8 +18,8 @@ namespace ROCKSDB_NAMESPACE {
 // A class representing a single uncompressed value read from a blob file.
 class BlobContents {
  public:
-  static std::unique_ptr<BlobContents> Create(CacheAllocationPtr&& allocation,
-                                              size_t size);
+  BlobContents(CacheAllocationPtr&& allocation, size_t size)
+      : allocation_(std::move(allocation)), data_(allocation_.get(), size) {}
 
   BlobContents(const BlobContents&) = delete;
   BlobContents& operator=(const BlobContents&) = delete;
@@ -34,23 +34,26 @@ class BlobContents {
 
   size_t ApproximateMemoryUsage() const;
 
-  // Callbacks for secondary cache
-  static size_t SizeCallback(void* obj);
-
-  static Status SaveToCallback(void* from_obj, size_t from_offset,
-                               size_t length, void* out);
-
-  static Cache::CacheItemHelper* GetCacheItemHelper();
-
-  static Status CreateCallback(CacheAllocationPtr&& allocation, const void* buf,
-                               size_t size, void** out_obj, size_t* charge);
+  // For TypedCacheInterface
+  const Slice& ContentSlice() const { return data_; }
+  static constexpr CacheEntryRole kCacheEntryRole = CacheEntryRole::kBlobValue;
 
  private:
-  BlobContents(CacheAllocationPtr&& allocation, size_t size)
-      : allocation_(std::move(allocation)), data_(allocation_.get(), size) {}
-
   CacheAllocationPtr allocation_;
   Slice data_;
 };
 
+class BlobContentsCreator : public Cache::CreateContext {
+ public:
+  static void Create(std::unique_ptr<BlobContents>* out, size_t* out_charge,
+                     const Slice& contents, MemoryAllocator* alloc) {
+    auto raw = new BlobContents(AllocateAndCopyBlock(contents, alloc),
+                                contents.size());
+    out->reset(raw);
+    if (out_charge) {
+      *out_charge = raw->ApproximateMemoryUsage();
+    }
+  }
+};
+
 }  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_builder.cc

@@ -13,6 +13,7 @@
 #include "db/blob/blob_index.h"
 #include "db/blob/blob_log_format.h"
 #include "db/blob/blob_log_writer.h"
+#include "db/blob/blob_source.h"
 #include "db/event_helpers.h"
 #include "db/version_set.h"
 #include "file/filename.h"
@@ -393,7 +394,7 @@ Status BlobFileBuilder::PutBlobIntoCacheIfNeeded(const Slice& blob,
                                                  uint64_t blob_offset) const {
   Status s = Status::OK();
 
-  auto blob_cache = immutable_options_->blob_cache;
+  BlobSource::SharedCacheInterface blob_cache{immutable_options_->blob_cache};
   auto statistics = immutable_options_->statistics.get();
   bool warm_cache =
       prepopulate_blob_cache_ == PrepopulateBlobCache::kFlushOnly &&
@@ -407,34 +408,12 @@ Status BlobFileBuilder::PutBlobIntoCacheIfNeeded(const Slice& blob,
 
     const Cache::Priority priority = Cache::Priority::BOTTOM;
 
-    // Objects to be put into the cache have to be heap-allocated and
-    // self-contained, i.e. own their contents. The Cache has to be able to
-    // take unique ownership of them.
-    CacheAllocationPtr allocation =
-        AllocateBlock(blob.size(), blob_cache->memory_allocator());
-    memcpy(allocation.get(), blob.data(), blob.size());
-    std::unique_ptr<BlobContents> buf =
-        BlobContents::Create(std::move(allocation), blob.size());
-
-    Cache::CacheItemHelper* const cache_item_helper =
-        BlobContents::GetCacheItemHelper();
-    assert(cache_item_helper);
-
-    if (immutable_options_->lowest_used_cache_tier ==
-        CacheTier::kNonVolatileBlockTier) {
-      s = blob_cache->Insert(key, buf.get(), cache_item_helper,
-                             buf->ApproximateMemoryUsage(),
-                             nullptr /* cache_handle */, priority);
-    } else {
-      s = blob_cache->Insert(key, buf.get(), buf->ApproximateMemoryUsage(),
-                             cache_item_helper->del_cb,
-                             nullptr /* cache_handle */, priority);
-    }
+    s = blob_cache.InsertSaved(key, blob, nullptr /*context*/, priority,
+                               immutable_options_->lowest_used_cache_tier);
 
     if (s.ok()) {
       RecordTick(statistics, BLOB_DB_CACHE_ADD);
-      RecordTick(statistics, BLOB_DB_CACHE_BYTES_WRITE, buf->size());
-      buf.release();
+      RecordTick(statistics, BLOB_DB_CACHE_BYTES_WRITE, blob.size());
     } else {
       RecordTick(statistics, BLOB_DB_CACHE_ADD_FAILURES);
     }

db/blob/blob_file_cache.cc

@@ -42,13 +42,13 @@ Status BlobFileCache::GetBlobFileReader(
   assert(blob_file_reader);
   assert(blob_file_reader->IsEmpty());
 
-  const Slice key = GetSlice(&blob_file_number);
+  const Slice key = GetSliceForKey(&blob_file_number);
 
   assert(cache_);
 
-  Cache::Handle* handle = cache_->Lookup(key);
+  TypedHandle* handle = cache_.Lookup(key);
   if (handle) {
-    *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
+    *blob_file_reader = cache_.Guard(handle);
     return Status::OK();
   }
 
@@ -57,9 +57,9 @@ Status BlobFileCache::GetBlobFileReader(
   // Check again while holding mutex
   MutexLock lock(mutex_.get(key));
 
-  handle = cache_->Lookup(key);
+  handle = cache_.Lookup(key);
   if (handle) {
-    *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
+    *blob_file_reader = cache_.Guard(handle);
     return Status::OK();
   }
 
@@ -84,8 +84,7 @@ Status BlobFileCache::GetBlobFileReader(
   {
     constexpr size_t charge = 1;
 
-    const Status s = cache_->Insert(key, reader.get(), charge,
-                                    &DeleteCacheEntry<BlobFileReader>, &handle);
+    const Status s = cache_.Insert(key, reader.get(), charge, &handle);
     if (!s.ok()) {
       RecordTick(statistics, NO_FILE_ERRORS);
       return s;
@@ -94,7 +93,7 @@ Status BlobFileCache::GetBlobFileReader(
 
   reader.release();
 
-  *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
+  *blob_file_reader = cache_.Guard(handle);
 
   return Status::OK();
 }

db/blob/blob_file_cache.h

@@ -7,7 +7,8 @@
 
 #include <cinttypes>
 
-#include "cache/cache_helpers.h"
+#include "cache/typed_cache.h"
+#include "db/blob/blob_file_reader.h"
 #include "rocksdb/rocksdb_namespace.h"
 #include "util/mutexlock.h"
 
@@ -18,7 +19,6 @@ struct ImmutableOptions;
 struct FileOptions;
 class HistogramImpl;
 class Status;
-class BlobFileReader;
 class Slice;
 class IOTracer;
 
@@ -36,7 +36,10 @@ class BlobFileCache {
                            CacheHandleGuard<BlobFileReader>* blob_file_reader);
 
  private:
-  Cache* cache_;
+  using CacheInterface =
+      BasicTypedCacheInterface<BlobFileReader, CacheEntryRole::kMisc>;
+  using TypedHandle = CacheInterface::TypedHandle;
+  CacheInterface cache_;
   // Note: mutex_ below is used to guard against multiple threads racing to open
   // the same file.
   Striped<port::Mutex, Slice> mutex_;

db/blob/blob_file_reader.cc

@@ -569,12 +569,7 @@ Status BlobFileReader::UncompressBlobIfNeeded(
   assert(result);
 
   if (compression_type == kNoCompression) {
-    CacheAllocationPtr allocation =
-        AllocateBlock(value_slice.size(), allocator);
-    memcpy(allocation.get(), value_slice.data(), value_slice.size());
-
-    *result = BlobContents::Create(std::move(allocation), value_slice.size());
-
+    BlobContentsCreator::Create(result, nullptr, value_slice, allocator);
     return Status::OK();
   }
 
@@ -602,7 +597,7 @@ Status BlobFileReader::UncompressBlobIfNeeded(
     return Status::Corruption("Unable to uncompress blob");
   }
 
-  *result = BlobContents::Create(std::move(output), uncompressed_size);
+  result->reset(new BlobContents(std::move(output), uncompressed_size));
 
   return Status::OK();
 }

db/blob/blob_source.cc

@@ -36,8 +36,8 @@ BlobSource::BlobSource(const ImmutableOptions* immutable_options,
   if (bbto &&
       bbto->cache_usage_options.options_overrides.at(CacheEntryRole::kBlobCache)
               .charged == CacheEntryRoleOptions::Decision::kEnabled) {
-    blob_cache_ = std::make_shared<ChargedCache>(immutable_options->blob_cache,
-                                                 bbto->block_cache);
+    blob_cache_ = SharedCacheInterface{std::make_shared<ChargedCache>(
+        immutable_options->blob_cache, bbto->block_cache)};
   }
 #endif  // ROCKSDB_LITE
 }
@@ -82,9 +82,8 @@ Status BlobSource::PutBlobIntoCache(
   assert(cached_blob);
   assert(cached_blob->IsEmpty());
 
-  Cache::Handle* cache_handle = nullptr;
+  TypedHandle* cache_handle = nullptr;
   const Status s = InsertEntryIntoCache(cache_key, blob->get(),
-                                        (*blob)->ApproximateMemoryUsage(),
                                         &cache_handle, Cache::Priority::BOTTOM);
   if (s.ok()) {
     blob->release();
@@ -106,26 +105,10 @@ Status BlobSource::PutBlobIntoCache(
   return s;
 }
 
-Cache::Handle* BlobSource::GetEntryFromCache(const Slice& key) const {
-  Cache::Handle* cache_handle = nullptr;
-
-  if (lowest_used_cache_tier_ == CacheTier::kNonVolatileBlockTier) {
-    Cache::CreateCallback create_cb =
-        [allocator = blob_cache_->memory_allocator()](
-            const void* buf, size_t size, void** out_obj,
-            size_t* charge) -> Status {
-      return BlobContents::CreateCallback(AllocateBlock(size, allocator), buf,
-                                          size, out_obj, charge);
-    };
-
-    cache_handle = blob_cache_->Lookup(key, BlobContents::GetCacheItemHelper(),
-                                       create_cb, Cache::Priority::BOTTOM,
-                                       true /* wait_for_cache */, statistics_);
-  } else {
-    cache_handle = blob_cache_->Lookup(key, statistics_);
-  }
-
-  return cache_handle;
+BlobSource::TypedHandle* BlobSource::GetEntryFromCache(const Slice& key) const {
+  return blob_cache_.LookupFull(
+      key, nullptr /* context */, Cache::Priority::BOTTOM,
+      true /* wait_for_cache */, statistics_, lowest_used_cache_tier_);
 }
 
 void BlobSource::PinCachedBlob(CacheHandleGuard<BlobContents>* cached_blob,
@@ -166,24 +149,11 @@ void BlobSource::PinOwnedBlob(std::unique_ptr<BlobContents>* owned_blob,
 }
 
 Status BlobSource::InsertEntryIntoCache(const Slice& key, BlobContents* value,
-                                        size_t charge,
-                                        Cache::Handle** cache_handle,
+                                        TypedHandle** cache_handle,
                                         Cache::Priority priority) const {
-  Status s;
-
-  Cache::CacheItemHelper* const cache_item_helper =
-      BlobContents::GetCacheItemHelper();
-  assert(cache_item_helper);
-
-  if (lowest_used_cache_tier_ == CacheTier::kNonVolatileBlockTier) {
-    s = blob_cache_->Insert(key, value, cache_item_helper, charge, cache_handle,
-                            priority);
-  } else {
-    s = blob_cache_->Insert(key, value, charge, cache_item_helper->del_cb,
-                            cache_handle, priority);
-  }
-
-  return s;
+  return blob_cache_.InsertFull(key, value, value->ApproximateMemoryUsage(),
+                                cache_handle, priority,
+                                lowest_used_cache_tier_);
 }
 
 Status BlobSource::GetBlob(const ReadOptions& read_options,
@@ -252,9 +222,10 @@ Status BlobSource::GetBlob(const ReadOptions& read_options,
       return Status::Corruption("Compression type mismatch when reading blob");
     }
 
-    MemoryAllocator* const allocator = (blob_cache_ && read_options.fill_cache)
-                                           ? blob_cache_->memory_allocator()
-                                           : nullptr;
+    MemoryAllocator* const allocator =
+        (blob_cache_ && read_options.fill_cache)
+            ? blob_cache_.get()->memory_allocator()
+            : nullptr;
 
     uint64_t read_size = 0;
     s = blob_file_reader.GetValue()->GetBlob(
@@ -418,9 +389,10 @@ void BlobSource::MultiGetBlobFromOneFile(const ReadOptions& read_options,
 
     assert(blob_file_reader.GetValue());
 
-    MemoryAllocator* const allocator = (blob_cache_ && read_options.fill_cache)
-                                           ? blob_cache_->memory_allocator()
-                                           : nullptr;
+    MemoryAllocator* const allocator =
+        (blob_cache_ && read_options.fill_cache)
+            ? blob_cache_.get()->memory_allocator()
+            : nullptr;
 
     blob_file_reader.GetValue()->MultiGetBlob(read_options, allocator,
                                               _blob_reqs, &_bytes_read);

db/blob/blob_source.h

@@ -8,8 +8,9 @@
 #include <cinttypes>
 #include <memory>
 
-#include "cache/cache_helpers.h"
 #include "cache/cache_key.h"
+#include "cache/typed_cache.h"
+#include "db/blob/blob_contents.h"
 #include "db/blob/blob_file_cache.h"
 #include "db/blob/blob_read_request.h"
 #include "rocksdb/cache.h"
@@ -23,7 +24,6 @@ struct ImmutableOptions;
 class Status;
 class FilePrefetchBuffer;
 class Slice;
-class BlobContents;
 
 // BlobSource is a class that provides universal access to blobs, regardless of
 // whether they are in the blob cache, secondary cache, or (remote) storage.
@@ -106,6 +106,14 @@ class BlobSource {
   bool TEST_BlobInCache(uint64_t file_number, uint64_t file_size,
                         uint64_t offset, size_t* charge = nullptr) const;
 
+  // For TypedSharedCacheInterface
+  void Create(BlobContents** out, const char* buf, size_t size,
+              MemoryAllocator* alloc);
+
+  using SharedCacheInterface =
+      FullTypedSharedCacheInterface<BlobContents, BlobContentsCreator>;
+  using TypedHandle = SharedCacheInterface::TypedHandle;
+
  private:
   Status GetBlobFromCache(const Slice& cache_key,
                           CacheHandleGuard<BlobContents>* cached_blob) const;
@@ -120,10 +128,10 @@ class BlobSource {
   static void PinOwnedBlob(std::unique_ptr<BlobContents>* owned_blob,
                            PinnableSlice* value);
 
-  Cache::Handle* GetEntryFromCache(const Slice& key) const;
+  TypedHandle* GetEntryFromCache(const Slice& key) const;
 
   Status InsertEntryIntoCache(const Slice& key, BlobContents* value,
-                              size_t charge, Cache::Handle** cache_handle,
+                              TypedHandle** cache_handle,
                               Cache::Priority priority) const;
 
   inline CacheKey GetCacheKey(uint64_t file_number, uint64_t /*file_size*/,
@@ -141,7 +149,7 @@ class BlobSource {
   BlobFileCache* blob_file_cache_;
 
   // A cache to store uncompressed blobs.
-  std::shared_ptr<Cache> blob_cache_;
+  mutable SharedCacheInterface blob_cache_;
 
   // The control option of how the cache tiers will be used. Currently rocksdb
   // support block/blob cache (volatile tier) and secondary cache (this tier

db/blob/blob_source_test.cc

@@ -1150,15 +1150,6 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
   auto blob_cache = options_.blob_cache;
   auto secondary_cache = lru_cache_opts_.secondary_cache;
 
-  Cache::CreateCallback create_cb = [](const void* buf, size_t size,
-                                       void** out_obj,
-                                       size_t* charge) -> Status {
-    CacheAllocationPtr allocation(new char[size]);
-
-    return BlobContents::CreateCallback(std::move(allocation), buf, size,
-                                        out_obj, charge);
-  };
-
   {
     // GetBlob
     std::vector<PinnableSlice> values(keys.size());
@@ -1219,14 +1210,15 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
     {
       CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[0]);
       const Slice key0 = cache_key.AsSlice();
-      auto handle0 = blob_cache->Lookup(key0, statistics);
+      auto handle0 = blob_cache->BasicLookup(key0, statistics);
       ASSERT_EQ(handle0, nullptr);
 
       // key0's item should be in the secondary cache.
       bool is_in_sec_cache = false;
-      auto sec_handle0 =
-          secondary_cache->Lookup(key0, create_cb, true,
-                                  /*advise_erase=*/true, is_in_sec_cache);
+      auto sec_handle0 = secondary_cache->Lookup(
+          key0, &BlobSource::SharedCacheInterface::kFullHelper,
+          /*context*/ nullptr, true,
+          /*advise_erase=*/true, is_in_sec_cache);
       ASSERT_FALSE(is_in_sec_cache);
       ASSERT_NE(sec_handle0, nullptr);
       ASSERT_TRUE(sec_handle0->IsReady());
@@ -1246,14 +1238,15 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
     {
       CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[1]);
       const Slice key1 = cache_key.AsSlice();
-      auto handle1 = blob_cache->Lookup(key1, statistics);
+      auto handle1 = blob_cache->BasicLookup(key1, statistics);
       ASSERT_NE(handle1, nullptr);
       blob_cache->Release(handle1);
 
       bool is_in_sec_cache = false;
-      auto sec_handle1 =
-          secondary_cache->Lookup(key1, create_cb, true,
-                                  /*advise_erase=*/true, is_in_sec_cache);
+      auto sec_handle1 = secondary_cache->Lookup(
+          key1, &BlobSource::SharedCacheInterface::kFullHelper,
+          /*context*/ nullptr, true,
+          /*advise_erase=*/true, is_in_sec_cache);
       ASSERT_FALSE(is_in_sec_cache);
       ASSERT_EQ(sec_handle1, nullptr);
 
@@ -1276,7 +1269,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
       // key0 should be in the primary cache.
       CacheKey cache_key0 = base_cache_key.WithOffset(blob_offsets[0]);
       const Slice key0 = cache_key0.AsSlice();
-      auto handle0 = blob_cache->Lookup(key0, statistics);
+      auto handle0 = blob_cache->BasicLookup(key0, statistics);
       ASSERT_NE(handle0, nullptr);
       auto value = static_cast<BlobContents*>(blob_cache->Value(handle0));
       ASSERT_NE(value, nullptr);
@@ -1286,12 +1279,12 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
       // key1 is not in the primary cache and is in the secondary cache.
       CacheKey cache_key1 = base_cache_key.WithOffset(blob_offsets[1]);
       const Slice key1 = cache_key1.AsSlice();
-      auto handle1 = blob_cache->Lookup(key1, statistics);
+      auto handle1 = blob_cache->BasicLookup(key1, statistics);
       ASSERT_EQ(handle1, nullptr);
 
       // erase key0 from the primary cache.
       blob_cache->Erase(key0);
-      handle0 = blob_cache->Lookup(key0, statistics);
+      handle0 = blob_cache->BasicLookup(key0, statistics);
       ASSERT_EQ(handle0, nullptr);
 
       // key1 promotion should succeed due to the primary cache being empty. we
@@ -1307,7 +1300,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
       // in the secondary cache. So, the primary cache's Lookup() without
       // secondary cache support cannot see it. (NOTE: The dummy handle used
       // to be a leaky abstraction but not anymore.)
-      handle1 = blob_cache->Lookup(key1, statistics);
+      handle1 = blob_cache->BasicLookup(key1, statistics);
       ASSERT_EQ(handle1, nullptr);
 
       // But after another access, it is promoted to primary cache
@@ -1315,7 +1308,7 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
                                                blob_offsets[1]));
 
       // And Lookup() can find it (without secondary cache support)
-      handle1 = blob_cache->Lookup(key1, statistics);
+      handle1 = blob_cache->BasicLookup(key1, statistics);
       ASSERT_NE(handle1, nullptr);
       ASSERT_NE(blob_cache->Value(handle1), nullptr);
       blob_cache->Release(handle1);

db/builder.cc

@@ -71,8 +71,9 @@ Status BuildTable(
     int job_id, const Env::IOPriority io_priority,
     TableProperties* table_properties, Env::WriteLifeTimeHint write_hint,
     const std::string* full_history_ts_low,
-    BlobFileCompletionCallback* blob_callback, uint64_t* num_input_entries,
-    uint64_t* memtable_payload_bytes, uint64_t* memtable_garbage_bytes) {
+    BlobFileCompletionCallback* blob_callback, Version* version,
+    uint64_t* num_input_entries, uint64_t* memtable_payload_bytes,
+    uint64_t* memtable_garbage_bytes) {
   assert((tboptions.column_family_id ==
           TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
          tboptions.column_family_name.empty());
@@ -175,10 +176,10 @@ Status BuildTable(
       builder = NewTableBuilder(tboptions, file_writer.get());
     }
 
+    auto ucmp = tboptions.internal_comparator.user_comparator();
     MergeHelper merge(
-        env, tboptions.internal_comparator.user_comparator(),
-        ioptions.merge_operator.get(), compaction_filter.get(), ioptions.logger,
-        true /* internal key corruption is not ok */,
+        env, ucmp, ioptions.merge_operator.get(), compaction_filter.get(),
+        ioptions.logger, true /* internal key corruption is not ok */,
         snapshots.empty() ? 0 : snapshots.back(), snapshot_checker);
 
     std::unique_ptr<BlobFileBuilder> blob_file_builder(
@@ -196,9 +197,8 @@ Status BuildTable(
 
     const std::atomic<bool> kManualCompactionCanceledFalse{false};
     CompactionIterator c_iter(
-        iter, tboptions.internal_comparator.user_comparator(), &merge,
-        kMaxSequenceNumber, &snapshots, earliest_write_conflict_snapshot,
-        job_snapshot, snapshot_checker, env,
+        iter, ucmp, &merge, kMaxSequenceNumber, &snapshots,
+        earliest_write_conflict_snapshot, job_snapshot, snapshot_checker, env,
         ShouldReportDetailedTime(env, ioptions.stats),
         true /* internal key corruption is not ok */, range_del_agg.get(),
         blob_file_builder.get(), ioptions.allow_data_in_errors,
@@ -241,14 +241,28 @@ Status BuildTable(
 
     if (s.ok()) {
       auto range_del_it = range_del_agg->NewIterator();
+      Slice last_tombstone_start_user_key{};
       for (range_del_it->SeekToFirst(); range_del_it->Valid();
            range_del_it->Next()) {
         auto tombstone = range_del_it->Tombstone();
         auto kv = tombstone.Serialize();
         builder->Add(kv.first.Encode(), kv.second);
-        meta->UpdateBoundariesForRange(kv.first, tombstone.SerializeEndKey(),
-                                       tombstone.seq_,
+        InternalKey tombstone_end = tombstone.SerializeEndKey();
+        meta->UpdateBoundariesForRange(kv.first, tombstone_end, tombstone.seq_,
                                        tboptions.internal_comparator);
+        if (version) {
+          if (last_tombstone_start_user_key.empty() ||
+              ucmp->CompareWithoutTimestamp(last_tombstone_start_user_key,
+                                            range_del_it->start_key()) < 0) {
+            SizeApproximationOptions approx_opts;
+            approx_opts.files_size_error_margin = 0.1;
+            meta->compensated_range_deletion_size += versions->ApproximateSize(
+                approx_opts, version, kv.first.Encode(), tombstone_end.Encode(),
+                0 /* start_level */, -1 /* end_level */,
+                TableReaderCaller::kFlush);
+          }
+          last_tombstone_start_user_key = range_del_it->start_key();
+        }
       }
     }
 

db/builder.h

@@ -13,6 +13,7 @@
 #include "db/range_tombstone_fragmenter.h"
 #include "db/seqno_to_time_mapping.h"
 #include "db/table_properties_collector.h"
+#include "db/version_set.h"
 #include "logging/event_logger.h"
 #include "options/cf_options.h"
 #include "rocksdb/comparator.h"
@@ -70,7 +71,7 @@ extern Status BuildTable(
     Env::WriteLifeTimeHint write_hint = Env::WLTH_NOT_SET,
     const std::string* full_history_ts_low = nullptr,
     BlobFileCompletionCallback* blob_callback = nullptr,
-    uint64_t* num_input_entries = nullptr,
+    Version* version = nullptr, uint64_t* num_input_entries = nullptr,
     uint64_t* memtable_payload_bytes = nullptr,
     uint64_t* memtable_garbage_bytes = nullptr);
 

db/c.cc

@@ -2588,6 +2588,12 @@ void rocksdb_block_based_options_set_partition_filters(
   options->rep.partition_filters = partition_filters;
 }
 
+void rocksdb_block_based_options_set_optimize_filters_for_memory(
+    rocksdb_block_based_table_options_t* options,
+    unsigned char optimize_filters_for_memory) {
+  options->rep.optimize_filters_for_memory = optimize_filters_for_memory;
+}
+
 void rocksdb_block_based_options_set_use_delta_encoding(
     rocksdb_block_based_table_options_t* options,
     unsigned char use_delta_encoding) {
@@ -4443,6 +4449,15 @@ rocksdb_readoptions_get_io_timeout(rocksdb_readoptions_t* opt) {
   return opt->rep.io_timeout.count();
 }
 
+void rocksdb_readoptions_set_async_io(rocksdb_readoptions_t* opt,
+                                      unsigned char v) {
+  opt->rep.async_io = v;
+}
+
+unsigned char rocksdb_readoptions_get_async_io(rocksdb_readoptions_t* opt) {
+  return opt->rep.async_io;
+}
+
 void rocksdb_readoptions_set_timestamp(rocksdb_readoptions_t* opt,
                                        const char* ts, size_t tslen) {
   if (ts == nullptr) {

db/c_test.c

@@ -2572,6 +2572,9 @@ int main(int argc, char** argv) {
     rocksdb_readoptions_set_io_timeout(ro, 400);
     CheckCondition(400 == rocksdb_readoptions_get_io_timeout(ro));
 
+    rocksdb_readoptions_set_async_io(ro, 1);
+    CheckCondition(1 == rocksdb_readoptions_get_async_io(ro));
+
     rocksdb_readoptions_destroy(ro);
   }
 

db/column_family.cc

@@ -557,7 +557,6 @@ ColumnFamilyData::ColumnFamilyData(
       next_(nullptr),
       prev_(nullptr),
       log_number_(0),
-      flush_reason_(FlushReason::kOthers),
       column_family_set_(column_family_set),
       queued_for_flush_(false),
       queued_for_compaction_(false),
@@ -565,7 +564,8 @@ ColumnFamilyData::ColumnFamilyData(
       allow_2pc_(db_options.allow_2pc),
       last_memtable_id_(0),
       db_paths_registered_(false),
-      mempurge_used_(false) {
+      mempurge_used_(false),
+      next_epoch_number_(1) {
   if (id_ != kDummyColumnFamilyDataId) {
     // TODO(cc): RegisterDbPaths can be expensive, considering moving it
     // outside of this constructor which might be called with db mutex held.
@@ -1128,12 +1128,9 @@ bool ColumnFamilyData::NeedsCompaction() const {
 Compaction* ColumnFamilyData::PickCompaction(
     const MutableCFOptions& mutable_options,
     const MutableDBOptions& mutable_db_options, LogBuffer* log_buffer) {
-  SequenceNumber earliest_mem_seqno =
-      std::min(mem_->GetEarliestSequenceNumber(),
-               imm_.current()->GetEarliestSequenceNumber(false));
   auto* result = compaction_picker_->PickCompaction(
       GetName(), mutable_options, mutable_db_options, current_->storage_info(),
-      log_buffer, earliest_mem_seqno);
+      log_buffer);
   if (result != nullptr) {
     result->SetInputVersion(current_);
   }
@@ -1220,6 +1217,7 @@ Compaction* ColumnFamilyData::CompactRange(
   if (result != nullptr) {
     result->SetInputVersion(current_);
   }
+  TEST_SYNC_POINT("ColumnFamilyData::CompactRange:Return");
   return result;
 }
 
@@ -1520,6 +1518,13 @@ FSDirectory* ColumnFamilyData::GetDataDir(size_t path_id) const {
   return data_dirs_[path_id].get();
 }
 
+void ColumnFamilyData::RecoverEpochNumbers() {
+  assert(current_);
+  auto* vstorage = current_->storage_info();
+  assert(vstorage);
+  vstorage->RecoverEpochNumbers(this);
+}
+
 ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
                                  const ImmutableDBOptions* db_options,
                                  const FileOptions& file_options,

db/column_family.h

@@ -310,10 +310,6 @@ class ColumnFamilyData {
   void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
   uint64_t GetLogNumber() const { return log_number_; }
 
-  void SetFlushReason(FlushReason flush_reason) {
-    flush_reason_ = flush_reason;
-  }
-  FlushReason GetFlushReason() const { return flush_reason_; }
   // thread-safe
   const FileOptions* soptions() const;
   const ImmutableOptions* ioptions() const { return &ioptions_; }
@@ -533,6 +529,24 @@ class ColumnFamilyData {
   void SetMempurgeUsed() { mempurge_used_ = true; }
   bool GetMempurgeUsed() { return mempurge_used_; }
 
+  // Allocate and return a new epoch number
+  uint64_t NewEpochNumber() { return next_epoch_number_.fetch_add(1); }
+
+  // Get the next epoch number to be assigned
+  uint64_t GetNextEpochNumber() const { return next_epoch_number_.load(); }
+
+  // Set the next epoch number to be assigned
+  void SetNextEpochNumber(uint64_t next_epoch_number) {
+    next_epoch_number_.store(next_epoch_number);
+  }
+
+  // Reset the next epoch number to be assigned
+  void ResetNextEpochNumber() { next_epoch_number_.store(1); }
+
+  // Recover the next epoch number of this CF and epoch number
+  // of its files (if missing)
+  void RecoverEpochNumbers();
+
  private:
   friend class ColumnFamilySet;
   ColumnFamilyData(uint32_t id, const std::string& name,
@@ -598,8 +612,6 @@ class ColumnFamilyData {
   // recovered from
   uint64_t log_number_;
 
-  std::atomic<FlushReason> flush_reason_;
-
   // An object that keeps all the compaction stats
   // and picks the next compaction
   std::unique_ptr<CompactionPicker> compaction_picker_;
@@ -634,6 +646,8 @@ class ColumnFamilyData {
   // a Version associated with this CFD
   std::shared_ptr<CacheReservationManager> file_metadata_cache_res_mgr_;
   bool mempurge_used_;
+
+  std::atomic<uint64_t> next_epoch_number_;
 };
 
 // ColumnFamilySet has interesting thread-safety requirements

db/compaction/compaction.cc

@@ -238,12 +238,19 @@ Compaction::Compaction(
       inputs_(PopulateWithAtomicBoundaries(vstorage, std::move(_inputs))),
       grandparents_(std::move(_grandparents)),
       score_(_score),
-      bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
+      bottommost_level_(
+          // For simplicity, we don't support the concept of "bottommost level"
+          // with
+          // `CompactionReason::kExternalSstIngestion` and
+          // `CompactionReason::kRefitLevel`
+          (_compaction_reason == CompactionReason::kExternalSstIngestion ||
+           _compaction_reason == CompactionReason::kRefitLevel)
+              ? false
+              : IsBottommostLevel(output_level_, vstorage, inputs_)),
       is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
       is_manual_compaction_(_manual_compaction),
       trim_ts_(_trim_ts),
       is_trivial_move_(false),
-
       compaction_reason_(_compaction_reason),
       notify_on_compaction_completion_(false),
       enable_blob_garbage_collection_(
@@ -258,8 +265,15 @@ Compaction::Compaction(
                   _blob_garbage_collection_age_cutoff > 1
               ? mutable_cf_options()->blob_garbage_collection_age_cutoff
               : _blob_garbage_collection_age_cutoff),
-      penultimate_level_(EvaluatePenultimateLevel(
-          vstorage, immutable_options_, start_level_, output_level_)) {
+      penultimate_level_(
+          // For simplicity, we don't support the concept of "penultimate level"
+          // with `CompactionReason::kExternalSstIngestion` and
+          // `CompactionReason::kRefitLevel`
+          _compaction_reason == CompactionReason::kExternalSstIngestion ||
+                  _compaction_reason == CompactionReason::kRefitLevel
+              ? Compaction::kInvalidLevel
+              : EvaluatePenultimateLevel(vstorage, immutable_options_,
+                                         start_level_, output_level_)) {
   MarkFilesBeingCompacted(true);
   if (is_manual_compaction_) {
     compaction_reason_ = CompactionReason::kManualCompaction;
@@ -332,6 +346,7 @@ void Compaction::PopulatePenultimateLevelOutputRange() {
   // the case that the penultimate level is empty).
   if (immutable_options_.compaction_style == kCompactionStyleUniversal) {
     exclude_level = kInvalidLevel;
+    penultimate_output_range_type_ = PenultimateOutputRangeType::kFullRange;
     std::set<uint64_t> penultimate_inputs;
     for (const auto& input_lvl : inputs_) {
       if (input_lvl.level == penultimate_level_) {
@@ -345,7 +360,8 @@ void Compaction::PopulatePenultimateLevelOutputRange() {
       if (penultimate_inputs.find(file->fd.GetNumber()) ==
           penultimate_inputs.end()) {
         exclude_level = number_levels_ - 1;
-        penultimate_output_range_type_ = PenultimateOutputRangeType::kFullRange;
+        penultimate_output_range_type_ =
+            PenultimateOutputRangeType::kNonLastRange;
         break;
       }
     }
@@ -354,35 +370,6 @@ void Compaction::PopulatePenultimateLevelOutputRange() {
   GetBoundaryKeys(input_vstorage_, inputs_,
                   &penultimate_level_smallest_user_key_,
                   &penultimate_level_largest_user_key_, exclude_level);
-
-  // If there's a case that the penultimate level output range is overlapping
-  // with the existing files, disable the penultimate level output by setting
-  // the range to empty. One example is the range delete could have overlap
-  // boundary with the next file. (which is actually a false overlap)
-  // TODO: Exclude such false overlap, so it won't disable the penultimate
-  //  output.
-  std::set<uint64_t> penultimate_inputs;
-  for (const auto& input_lvl : inputs_) {
-    if (input_lvl.level == penultimate_level_) {
-      for (const auto& file : input_lvl.files) {
-        penultimate_inputs.emplace(file->fd.GetNumber());
-      }
-    }
-  }
-
-  auto penultimate_files = input_vstorage_->LevelFiles(penultimate_level_);
-  for (const auto& file : penultimate_files) {
-    if (penultimate_inputs.find(file->fd.GetNumber()) ==
-            penultimate_inputs.end() &&
-        OverlapPenultimateLevelOutputRange(file->smallest.user_key(),
-                                           file->largest.user_key())) {
-      // basically disable the penultimate range output. which should be rare
-      // or a false overlap caused by range del
-      penultimate_level_smallest_user_key_ = "";
-      penultimate_level_largest_user_key_ = "";
-      penultimate_output_range_type_ = PenultimateOutputRangeType::kDisabled;
-    }
-  }
 }
 
 Compaction::~Compaction() {
@@ -807,6 +794,16 @@ uint64_t Compaction::MinInputFileOldestAncesterTime(
   return min_oldest_ancester_time;
 }
 
+uint64_t Compaction::MinInputFileEpochNumber() const {
+  uint64_t min_epoch_number = std::numeric_limits<uint64_t>::max();
+  for (const auto& inputs_per_level : inputs_) {
+    for (const auto& file : inputs_per_level.files) {
+      min_epoch_number = std::min(min_epoch_number, file->epoch_number);
+    }
+  }
+  return min_epoch_number;
+}
+
 int Compaction::EvaluatePenultimateLevel(
     const VersionStorageInfo* vstorage,
     const ImmutableOptions& immutable_options, const int start_level,

db/compaction/compaction.h

@@ -378,6 +378,9 @@ class Compaction {
   // This is used to filter out some input files' ancester's time range.
   uint64_t MinInputFileOldestAncesterTime(const InternalKey* start,
                                           const InternalKey* end) const;
+  // Return the minimum epoch number among
+  // input files' associated with this compaction
+  uint64_t MinInputFileEpochNumber() const;
 
   // Called by DBImpl::NotifyOnCompactionCompleted to make sure number of
   // compaction begin and compaction completion callbacks match.

db/compaction/compaction_iterator.cc

@@ -124,6 +124,9 @@ CompactionIterator::CompactionIterator(
          timestamp_size_ == full_history_ts_low_->size());
 #endif
   input_.SetPinnedItersMgr(&pinned_iters_mgr_);
+  // The default `merge_until_status_` does not need to be checked since it is
+  // overwritten as soon as `MergeUntil()` is called
+  merge_until_status_.PermitUncheckedError();
   TEST_SYNC_POINT_CALLBACK("CompactionIterator:AfterInit", compaction_.get());
 }
 
@@ -178,6 +181,20 @@ void CompactionIterator::Next() {
       ikey_.user_key = current_key_.GetUserKey();
       validity_info_.SetValid(ValidContext::kMerge1);
     } else {
+      if (merge_until_status_.IsMergeInProgress()) {
+        // `Status::MergeInProgress()` tells us that the previous `MergeUntil()`
+        // produced only merge operands. Those merge operands were accessed and
+        // written out using `merge_out_iter_`. Since `merge_out_iter_` is
+        // exhausted at this point, all merge operands have been written out.
+        //
+        // Still, there may be a base value (PUT, DELETE, SINGLEDEL, etc.) that
+        // needs to be written out. Normally, `CompactionIterator` would skip it
+        // on the basis that it has already output something in the same
+        // snapshot stripe. To prevent this, we reset `has_current_user_key_` to
+        // trick the future iteration from finding out the snapshot stripe is
+        // unchanged.
+        has_current_user_key_ = false;
+      }
       // We consumed all pinned merge operands, release pinned iterators
       pinned_iters_mgr_.ReleasePinnedData();
       // MergeHelper moves the iterator to the first record after the merged
@@ -880,14 +897,15 @@ void CompactionIterator::NextFromInput() {
       // have hit (A)
       // We encapsulate the merge related state machine in a different
       // object to minimize change to the existing flow.
-      Status s = merge_helper_->MergeUntil(
+      merge_until_status_ = merge_helper_->MergeUntil(
           &input_, range_del_agg_, prev_snapshot, bottommost_level_,
           allow_data_in_errors_, blob_fetcher_.get(), full_history_ts_low_,
           prefetch_buffers_.get(), &iter_stats_);
       merge_out_iter_.SeekToFirst();
 
-      if (!s.ok() && !s.IsMergeInProgress()) {
-        status_ = s;
+      if (!merge_until_status_.ok() &&
+          !merge_until_status_.IsMergeInProgress()) {
+        status_ = merge_until_status_;
         return;
       } else if (merge_out_iter_.Valid()) {
         // NOTE: key, value, and ikey_ refer to old entries.

db/compaction/compaction_iterator.h

@@ -432,6 +432,7 @@ class CompactionIterator {
   bool clear_and_output_next_key_ = false;
 
   MergeOutputIterator merge_out_iter_;
+  Status merge_until_status_;
   // PinnedIteratorsManager used to pin input_ Iterator blocks while reading
   // merge operands and then releasing them after consuming them.
   PinnedIteratorsManager pinned_iters_mgr_;

db/compaction/compaction_job.cc

@@ -99,6 +99,8 @@ const char* GetCompactionReasonString(CompactionReason compaction_reason) {
       return "ForcedBlobGC";
     case CompactionReason::kRoundRobinTtl:
       return "RoundRobinTtl";
+    case CompactionReason::kRefitLevel:
+      return "RefitLevel";
     case CompactionReason::kNumOfReasons:
       // fall through
     default:
@@ -1834,12 +1836,14 @@ Status CompactionJob::OpenCompactionOutputFile(SubcompactionState* sub_compact,
   }
 
   // Initialize a SubcompactionState::Output and add it to sub_compact->outputs
+  uint64_t epoch_number = sub_compact->compaction->MinInputFileEpochNumber();
   {
     FileMetaData meta;
     meta.fd = FileDescriptor(file_number,
                              sub_compact->compaction->output_path_id(), 0);
     meta.oldest_ancester_time = oldest_ancester_time;
     meta.file_creation_time = current_time;
+    meta.epoch_number = epoch_number;
     meta.temperature = temperature;
     assert(!db_id_.empty());
     assert(!db_session_id_.empty());

db/compaction/compaction_job.h

@@ -402,6 +402,7 @@ struct CompactionServiceOutputFile {
   std::string largest_internal_key;
   uint64_t oldest_ancester_time;
   uint64_t file_creation_time;
+  uint64_t epoch_number;
   uint64_t paranoid_hash;
   bool marked_for_compaction;
   UniqueId64x2 unique_id;
@@ -411,8 +412,8 @@ struct CompactionServiceOutputFile {
       const std::string& name, SequenceNumber smallest, SequenceNumber largest,
       std::string _smallest_internal_key, std::string _largest_internal_key,
       uint64_t _oldest_ancester_time, uint64_t _file_creation_time,
-      uint64_t _paranoid_hash, bool _marked_for_compaction,
-      UniqueId64x2 _unique_id)
+      uint64_t _epoch_number, uint64_t _paranoid_hash,
+      bool _marked_for_compaction, UniqueId64x2 _unique_id)
       : file_name(name),
         smallest_seqno(smallest),
         largest_seqno(largest),
@@ -420,6 +421,7 @@ struct CompactionServiceOutputFile {
         largest_internal_key(std::move(_largest_internal_key)),
         oldest_ancester_time(_oldest_ancester_time),
         file_creation_time(_file_creation_time),
+        epoch_number(_epoch_number),
         paranoid_hash(_paranoid_hash),
         marked_for_compaction(_marked_for_compaction),
         unique_id(std::move(_unique_id)) {}

db/compaction/compaction_job_test.cc

@@ -380,11 +380,14 @@ class CompactionJobTestBase : public testing::Test {
     }
 
     VersionEdit edit;
-    edit.AddFile(level, file_number, 0, file_size, smallest_key, largest_key,
-                 smallest_seqno, largest_seqno, false, Temperature::kUnknown,
-                 oldest_blob_file_number, kUnknownOldestAncesterTime,
-                 kUnknownFileCreationTime, kUnknownFileChecksum,
-                 kUnknownFileChecksumFuncName, kNullUniqueId64x2);
+    edit.AddFile(
+        level, file_number, 0, file_size, smallest_key, largest_key,
+        smallest_seqno, largest_seqno, false, Temperature::kUnknown,
+        oldest_blob_file_number, kUnknownOldestAncesterTime,
+        kUnknownFileCreationTime,
+        versions_->GetColumnFamilySet()->GetDefault()->NewEpochNumber(),
+        kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2,
+        0);
 
     mutex_.Lock();
     EXPECT_OK(
@@ -1655,7 +1658,7 @@ TEST_F(CompactionJobTest, ResultSerialization) {
         rnd.RandomBinaryString(rnd.Uniform(kStrMaxLen)),
         rnd.RandomBinaryString(rnd.Uniform(kStrMaxLen)),
         rnd64.Uniform(UINT64_MAX), rnd64.Uniform(UINT64_MAX),
-        rnd64.Uniform(UINT64_MAX), rnd.OneIn(2), id);
+        rnd64.Uniform(UINT64_MAX), rnd64.Uniform(UINT64_MAX), rnd.OneIn(2), id);
   }
   result.output_level = rnd.Uniform(10);
   result.output_path = rnd.RandomString(rnd.Uniform(kStrMaxLen));

db/compaction/compaction_outputs.cc

@@ -76,6 +76,46 @@ IOStatus CompactionOutputs::WriterSyncClose(const Status& input_status,
   return io_s;
 }
 
+bool CompactionOutputs::UpdateFilesToCutForTTLStates(
+    const Slice& internal_key) {
+  if (!files_to_cut_for_ttl_.empty()) {
+    const InternalKeyComparator* icmp =
+        &compaction_->column_family_data()->internal_comparator();
+    if (cur_files_to_cut_for_ttl_ != -1) {
+      // Previous key is inside the range of a file
+      if (icmp->Compare(internal_key,
+                        files_to_cut_for_ttl_[cur_files_to_cut_for_ttl_]
+                            ->largest.Encode()) > 0) {
+        next_files_to_cut_for_ttl_ = cur_files_to_cut_for_ttl_ + 1;
+        cur_files_to_cut_for_ttl_ = -1;
+        return true;
+      }
+    } else {
+      // Look for the key position
+      while (next_files_to_cut_for_ttl_ <
+             static_cast<int>(files_to_cut_for_ttl_.size())) {
+        if (icmp->Compare(internal_key,
+                          files_to_cut_for_ttl_[next_files_to_cut_for_ttl_]
+                              ->smallest.Encode()) >= 0) {
+          if (icmp->Compare(internal_key,
+                            files_to_cut_for_ttl_[next_files_to_cut_for_ttl_]
+                                ->largest.Encode()) <= 0) {
+            // With in the current file
+            cur_files_to_cut_for_ttl_ = next_files_to_cut_for_ttl_;
+            return true;
+          }
+          // Beyond the current file
+          next_files_to_cut_for_ttl_++;
+        } else {
+          // Still fall into the gap
+          break;
+        }
+      }
+    }
+  }
+  return false;
+}
+
 size_t CompactionOutputs::UpdateGrandparentBoundaryInfo(
     const Slice& internal_key) {
   size_t curr_key_boundary_switched_num = 0;
@@ -185,18 +225,30 @@ uint64_t CompactionOutputs::GetCurrentKeyGrandparentOverlappedBytes(
 
 bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) {
   assert(c_iter.Valid());
-
-  // always update grandparent information like overlapped file number, size
-  // etc.
   const Slice& internal_key = c_iter.key();
   const uint64_t previous_overlapped_bytes = grandparent_overlapped_bytes_;
-  size_t num_grandparent_boundaries_crossed =
-      UpdateGrandparentBoundaryInfo(internal_key);
+  const InternalKeyComparator* icmp =
+      &compaction_->column_family_data()->internal_comparator();
+  size_t num_grandparent_boundaries_crossed = 0;
+  bool should_stop_for_ttl = false;
+  // Always update grandparent information like overlapped file number, size
+  // etc., and TTL states.
+  // If compaction_->output_level() == 0, there is no need to update grandparent
+  // info, and that `grandparent` should be empty.
+  if (compaction_->output_level() > 0) {
+    num_grandparent_boundaries_crossed =
+        UpdateGrandparentBoundaryInfo(internal_key);
+    should_stop_for_ttl = UpdateFilesToCutForTTLStates(internal_key);
+  }
 
   if (!HasBuilder()) {
     return false;
   }
 
+  if (should_stop_for_ttl) {
+    return true;
+  }
+
   // If there's user defined partitioner, check that first
   if (partitioner_ && partitioner_->ShouldPartition(PartitionerRequest(
                           last_key_for_partitioner_, c_iter.user_key(),
@@ -214,9 +266,6 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) {
     return true;
   }
 
-  const InternalKeyComparator* icmp =
-      &compaction_->column_family_data()->internal_comparator();
-
   // Check if it needs to split for RoundRobin
   // Invalid local_output_split_key indicates that we do not need to split
   if (local_output_split_key_ != nullptr && !is_split_) {
@@ -290,41 +339,6 @@ bool CompactionOutputs::ShouldStopBefore(const CompactionIterator& c_iter) {
     }
   }
 
-  // check ttl file boundaries if there's any
-  if (!files_to_cut_for_ttl_.empty()) {
-    if (cur_files_to_cut_for_ttl_ != -1) {
-      // Previous key is inside the range of a file
-      if (icmp->Compare(internal_key,
-                        files_to_cut_for_ttl_[cur_files_to_cut_for_ttl_]
-                            ->largest.Encode()) > 0) {
-        next_files_to_cut_for_ttl_ = cur_files_to_cut_for_ttl_ + 1;
-        cur_files_to_cut_for_ttl_ = -1;
-        return true;
-      }
-    } else {
-      // Look for the key position
-      while (next_files_to_cut_for_ttl_ <
-             static_cast<int>(files_to_cut_for_ttl_.size())) {
-        if (icmp->Compare(internal_key,
-                          files_to_cut_for_ttl_[next_files_to_cut_for_ttl_]
-                              ->smallest.Encode()) >= 0) {
-          if (icmp->Compare(internal_key,
-                            files_to_cut_for_ttl_[next_files_to_cut_for_ttl_]
-                                ->largest.Encode()) <= 0) {
-            // With in the current file
-            cur_files_to_cut_for_ttl_ = next_files_to_cut_for_ttl_;
-            return true;
-          }
-          // Beyond the current file
-          next_files_to_cut_for_ttl_++;
-        } else {
-          // Still fall into the gap
-          break;
-        }
-      }
-    }
-  }
-
   return false;
 }
 
@@ -399,6 +413,17 @@ Status CompactionOutputs::AddRangeDels(
   const Slice *lower_bound, *upper_bound;
   bool lower_bound_from_sub_compact = false;
 
+  // The following example does not happen since
+  // CompactionOutput::ShouldStopBefore() always return false for the first
+  // point key. But we should consider removing this dependency. Suppose for the
+  // first compaction output file,
+  //  - next_table_min_key.user_key == comp_start_user_key
+  //  - no point key is in the output file
+  //  - there is a range tombstone @seqno to be added that covers
+  //  comp_start_user_key
+  // Then meta.smallest will be set to comp_start_user_key@seqno
+  // and meta.largest will be set to comp_start_user_key@kMaxSequenceNumber
+  // which violates the assumption that meta.smallest should be <= meta.largest.
   size_t output_size = outputs_.size();
   if (output_size == 1) {
     // For the first output table, include range tombstones before the min
@@ -459,20 +484,34 @@ Status CompactionOutputs::AddRangeDels(
   } else {
     it->SeekToFirst();
   }
+  Slice last_tombstone_start_user_key{};
   for (; it->Valid(); it->Next()) {
     auto tombstone = it->Tombstone();
     if (upper_bound != nullptr) {
       int cmp =
           ucmp->CompareWithoutTimestamp(*upper_bound, tombstone.start_key_);
-      if ((has_overlapping_endpoints && cmp < 0) ||
-          (!has_overlapping_endpoints && cmp <= 0)) {
-        // Tombstones starting after upper_bound only need to be included in
-        // the next table. If the current SST ends before upper_bound, i.e.,
-        // `has_overlapping_endpoints == false`, we can also skip over range
-        // tombstones that start exactly at upper_bound. Such range
-        // tombstones will be included in the next file and are not relevant
-        // to the point keys or endpoints of the current file.
+      // Tombstones starting after upper_bound only need to be included in
+      // the next table.
+      // If the current SST ends before upper_bound, i.e.,
+      // `has_overlapping_endpoints == false`, we can also skip over range
+      // tombstones that start exactly at upper_bound. Such range
+      // tombstones will be included in the next file and are not relevant
+      // to the point keys or endpoints of the current file.
+      // If the current SST ends at the same user key at upper_bound,
+      // i.e., `has_overlapping_endpoints == true`, AND the tombstone has
+      // the same start key as upper_bound, i.e., cmp == 0, then
+      // the tombstone is relevant only if the tombstone's sequence number
+      // is no larger than this file's largest key's sequence number. This
+      // is because the upper bound to truncate this file's range tombstone
+      // will be meta.largest in this case, and any tombstone that starts after
+      // it will not be relevant.
+      if (cmp < 0) {
         break;
+      } else if (cmp == 0) {
+        if (!has_overlapping_endpoints ||
+            tombstone.seq_ < GetInternalKeySeqno(meta.largest.Encode())) {
+          break;
+        }
       }
     }
 
@@ -500,30 +539,26 @@ Status CompactionOutputs::AddRangeDels(
            ucmp->CompareWithoutTimestamp(*lower_bound, kv.second) < 0);
     // Range tombstone is not supported by output validator yet.
     builder_->Add(kv.first.Encode(), kv.second);
-    InternalKey smallest_candidate = std::move(kv.first);
+    InternalKey tombstone_start = std::move(kv.first);
+    InternalKey smallest_candidate{tombstone_start};
     if (lower_bound != nullptr &&
         ucmp->CompareWithoutTimestamp(smallest_candidate.user_key(),
                                       *lower_bound) <= 0) {
       // Pretend the smallest key has the same user key as lower_bound
       // (the max key in the previous table or subcompaction) in order for
       // files to appear key-space partitioned.
-      //
-      // When lower_bound is chosen by a subcompaction, we know that
-      // subcompactions over smaller keys cannot contain any keys at
-      // lower_bound. We also know that smaller subcompactions exist,
-      // because otherwise the subcompaction woud be unbounded on the left.
-      // As a result, we know that no other files on the output level will
-      // contain actual keys at lower_bound (an output file may have a
-      // largest key of lower_bound@kMaxSequenceNumber, but this only
-      // indicates a large range tombstone was truncated). Therefore, it is
-      // safe to use the tombstone's sequence number, to ensure that keys at
-      // lower_bound at lower levels are covered by truncated tombstones.
-      //
-      // If lower_bound was chosen by the smallest data key in the file,
-      // choose lowest seqnum so this file's smallest internal key comes
-      // after the previous file's largest. The fake seqnum is OK because
-      // the read path's file-picking code only considers user key.
       if (lower_bound_from_sub_compact) {
+        // When lower_bound is chosen by a subcompaction
+        // (lower_bound_from_sub_compact), we know that subcompactions over
+        // smaller keys cannot contain any keys at lower_bound. We also know
+        // that smaller subcompactions exist, because otherwise the
+        // subcompaction woud be unbounded on the left. As a result, we know
+        // that no other files on the output level will contain actual keys at
+        // lower_bound (an output file may have a largest key of
+        // lower_bound@kMaxSequenceNumber, but this only indicates a large range
+        // tombstone was truncated). Therefore, it is safe to use the
+        // tombstone's sequence number, to ensure that keys at lower_bound at
+        // lower levels are covered by truncated tombstones.
         if (ts_sz) {
           assert(tombstone.ts_.size() == ts_sz);
           smallest_candidate = InternalKey(*lower_bound, tombstone.seq_,
@@ -533,10 +568,15 @@ Status CompactionOutputs::AddRangeDels(
               InternalKey(*lower_bound, tombstone.seq_, kTypeRangeDeletion);
         }
       } else {
+        // If lower_bound was chosen by the smallest data key in the file,
+        // choose lowest seqnum so this file's smallest internal key comes
+        // after the previous file's largest. The fake seqnum is OK because
+        // the read path's file-picking code only considers user key.
         smallest_candidate = InternalKey(*lower_bound, 0, kTypeRangeDeletion);
       }
     }
-    InternalKey largest_candidate = tombstone.SerializeEndKey();
+    InternalKey tombstone_end = tombstone.SerializeEndKey();
+    InternalKey largest_candidate{tombstone_end};
     if (upper_bound != nullptr &&
         ucmp->CompareWithoutTimestamp(*upper_bound,
                                       largest_candidate.user_key()) <= 0) {
@@ -570,21 +610,174 @@ Status CompactionOutputs::AddRangeDels(
             InternalKey(*upper_bound, kMaxSequenceNumber, kTypeRangeDeletion);
       }
     }
-#ifndef NDEBUG
-    SequenceNumber smallest_ikey_seqnum = kMaxSequenceNumber;
-    if (meta.smallest.size() > 0) {
-      smallest_ikey_seqnum = GetInternalKeySeqno(meta.smallest.Encode());
-    }
-#endif
     meta.UpdateBoundariesForRange(smallest_candidate, largest_candidate,
                                   tombstone.seq_, icmp);
-    // The smallest key in a file is used for range tombstone truncation, so
-    // it cannot have a seqnum of 0 (unless the smallest data key in a file
-    // has a seqnum of 0). Otherwise, the truncated tombstone may expose
-    // deleted keys at lower levels.
-    assert(smallest_ikey_seqnum == 0 ||
-           ExtractInternalKeyFooter(meta.smallest.Encode()) !=
-               PackSequenceAndType(0, kTypeRangeDeletion));
+    if (!bottommost_level) {
+      bool start_user_key_changed =
+          last_tombstone_start_user_key.empty() ||
+          ucmp->CompareWithoutTimestamp(last_tombstone_start_user_key,
+                                        it->start_key()) < 0;
+      last_tombstone_start_user_key = it->start_key();
+      // Range tombstones are truncated at file boundaries
+      if (icmp.Compare(tombstone_start, meta.smallest) < 0) {
+        tombstone_start = meta.smallest;
+      }
+      if (icmp.Compare(tombstone_end, meta.largest) > 0) {
+        tombstone_end = meta.largest;
+      }
+      // this assertion validates invariant (2) in the comment below.
+      assert(icmp.Compare(tombstone_start, tombstone_end) <= 0);
+      if (start_user_key_changed) {
+        // if tombstone_start >= tombstone_end, then either no key range is
+        // covered, or that they have the same user key. If they have the same
+        // user key, then the internal key range should only be within this
+        // level, and no keys from older levels is covered.
+        if (ucmp->CompareWithoutTimestamp(tombstone_start.user_key(),
+                                          tombstone_end.user_key()) < 0) {
+          SizeApproximationOptions approx_opts;
+          approx_opts.files_size_error_margin = 0.1;
+          auto approximate_covered_size =
+              compaction_->input_version()->version_set()->ApproximateSize(
+                  approx_opts, compaction_->input_version(),
+                  tombstone_start.Encode(), tombstone_end.Encode(),
+                  compaction_->output_level() + 1 /* start_level */,
+                  -1 /* end_level */, kCompaction);
+          meta.compensated_range_deletion_size += approximate_covered_size;
+        }
+      }
+    }
+    // TODO: show invariants that ensure all necessary range tombstones are
+    // added
+    //  and that file boundaries ensure no coverage is lost.
+    // Each range tombstone with internal key range [tombstone_start,
+    // tombstone_end] is being added to the current compaction output file here.
+    // The range tombstone is going to be truncated at range [meta.smallest,
+    // meta.largest] during reading/scanning. We should maintain invariants
+    // (1) meta.smallest <= meta.largest and,
+    // (2) [tombstone_start, tombstone_end] and [meta.smallest, meta.largest]
+    // overlaps, as there is no point adding range tombstone with a range
+    // outside the file's range.
+    // Since `tombstone_end` is always some user_key@kMaxSeqno, it is okay to
+    // use either open or closed range. Using closed range here to make
+    // reasoning easier, and it is more consistent with an ongoing work that
+    // tries to simplify this method.
+    //
+    // There are two cases:
+    // Case 1. Output file has no point key:
+    //   First we show this case only happens when the entire compaction output
+    //   is range tombstone only. This is true if CompactionIterator does not
+    //   emit any point key. Suppose CompactionIterator emits some point key.
+    //   Based on the assumption that CompactionOutputs::ShouldStopBefore()
+    //   always return false for the first point key, the first compaction
+    //   output file always contains a point key. Each new compaction output
+    //   file is created if there is a point key for which ShouldStopBefore()
+    //   returns true, and the point key would be added to the new compaction
+    //   output file. So each new compaction file always contains a point key.
+    //   So Case 1 only happens when CompactionIterator does not emit any
+    //   point key.
+    //
+    //   To show (1) meta.smallest <= meta.largest:
+    //   Since the compaction output is range tombstone only, `lower_bound` and
+    //   `upper_bound` are either null or comp_start/end_user_key respectively.
+    //   According to how UpdateBoundariesForRange() is implemented, it blindly
+    //   updates meta.smallest and meta.largest to smallest_candidate and
+    //   largest_candidate the first time it is called. Subsequently, it
+    //   compares input parameter with meta.smallest and meta.largest and only
+    //   updates them when input is smaller/larger. So we only need to show
+    //   smallest_candidate <= largest_candidate the first time
+    //   UpdateBoundariesForRange() is called. Here we show something stronger
+    //   that smallest_candidate.user_key < largest_candidate.user_key always
+    //   hold for Case 1.
+    //   We assume comp_start_user_key < comp_end_user_key, if provided. We
+    //   assume that tombstone_start < tombstone_end. This assumption is based
+    //   on that each fragment in FragmentedTombstoneList has
+    //   start_key < end_key (user_key) and that
+    //   FragmentedTombstoneIterator::Tombstone() returns the pair
+    //   (start_key@tombstone_seqno with op_type kTypeRangeDeletion, end_key).
+    //   The logic in this loop sets smallest_candidate to
+    //   max(tombstone_start.user_key, comp_start_user_key)@tombstone.seq_ with
+    //   op_type kTypeRangeDeletion, largest_candidate to
+    //   min(tombstone_end.user_key, comp_end_user_key)@kMaxSequenceNumber with
+    //   op_type kTypeRangeDeletion. When a bound is null, there is no
+    //   truncation on that end. To show that smallest_candidate.user_key <
+    //   largest_candidate.user_key, it suffices to show
+    //   tombstone_start.user_key < comp_end_user_key (if not null) AND
+    //   comp_start_user_key (if not null) < tombstone_end.user_key.
+    //   Since the file has no point key, `has_overlapping_endpoints` is false.
+    //   In the first sanity check of this for-loop, we compare
+    //   tombstone_start.user_key against upper_bound = comp_end_user_key,
+    //   and only proceed if tombstone_start.user_key < comp_end_user_key.
+    //   We assume FragmentedTombstoneIterator::Seek(k) lands
+    //   on a tombstone with end_key > k. So the call it->Seek(*lower_bound)
+    //   above implies compact_start_user_key < tombstone_end.user_key.
+    //
+    //   To show (2) [tombstone_start, tombstone_end] and [meta.smallest,
+    //   meta.largest] overlaps (after the call to UpdateBoundariesForRange()):
+    //   In the proof for (1) we have shown that
+    //   smallest_candidate <= largest_candidate. Since tombstone_start <=
+    //   smallest_candidate <= largest_candidate <= tombstone_end, for (2) to
+    //   hold, it suffices to show that [smallest_candidate, largest_candidate]
+    //   overlaps with [meta.smallest, meta.largest]. too.
+    //   Given meta.smallest <= meta.largest shown above, we need to show
+    //   that it is impossible to have largest_candidate < meta.smallest or
+    //   meta.largest < smallest_candidate. If the above
+    //   meta.UpdateBoundariesForRange(smallest_candidate, largest_candidate)
+    //   updates meta.largest or meta.smallest, then the two ranges overlap.
+    //   So we assume meta.UpdateBoundariesForRange(smallest_candidate,
+    //   largest_candidate) did not update meta.smallest nor meta.largest, which
+    //   means meta.smallest < smallest_candidate and largest_candidate <
+    //   meta.largest.
+    //
+    // Case 2. Output file has >= 1 point key. This means meta.smallest and
+    // meta.largest are not empty when AddRangeDels() is called.
+    //   To show (1) meta.smallest <= meta.largest:
+    //   Assume meta.smallest <= meta.largest when AddRangeDels() is called,
+    //   this follow from how UpdateBoundariesForRange() is implemented where it
+    //   takes min or max to update meta.smallest or meta.largest.
+    //
+    //   To show (2) [tombstone_start, tombstone_end] and [meta.smallest,
+    //   meta.largest] overlaps (after the call to UpdateBoundariesForRange()):
+    //   When smallest_candidate <= largest_candidate, the proof in Case 1
+    //   applies, so we only need to show (2) holds when smallest_candidate >
+    //   largest_candidate. When both bounds are either null or from
+    //   subcompaction boundary, the proof in Case 1 applies, so we only need to
+    //   show (2) holds when at least one bound is from a point key (either
+    //   meta.smallest for lower bound or next_table_min_key for upper bound).
+    //
+    //   Suppose lower bound is meta.smallest.user_key. The call
+    //   it->Seek(*lower_bound) implies tombstone_end.user_key >
+    //   meta.smallest.user_key. We have smallest_candidate.user_key =
+    //   max(tombstone_start.user_key, meta.smallest.user_key). For
+    //   smallest_candidate to be > largest_candidate, we need
+    //   largest_candidate.user_key = upper_bound = smallest_candidate.user_key,
+    //   where tombstone_end is truncated to largest_candidate.
+    //   Subcase 1:
+    //   Suppose largest_candidate.user_key = comp_end_user_key (there is no
+    //   next point key). Subcompaction ensures any point key from this
+    //   subcompaction has a user_key < comp_end_user_key, so 1)
+    //   meta.smallest.user_key < comp_end_user_key, 2)
+    //   `has_overlapping_endpoints` is false, and the first if condition in
+    //   this for-loop ensures tombstone_start.user_key < comp_end_user_key. So
+    //   smallest_candidate.user_key < largest_candidate.user_key. This case
+    //   cannot happen when smallest > largest_candidate.
+    //   Subcase 2:
+    //   Suppose largest_candidate.user_key = next_table_min_key.user_key.
+    //   The first if condition in this for-loop together with
+    //   smallest_candidate.user_key = next_table_min_key.user_key =
+    //   upper_bound implies `has_overlapping_endpoints` is true (so meta
+    //   largest.user_key = upper_bound) and
+    //   tombstone.seq_ < meta.largest.seqno. So
+    //   tombstone_start < meta.largest < tombstone_end.
+    //
+    //   Suppose lower bound is comp_start_user_key and upper_bound is
+    //   next_table_min_key. The call it->Seek(*lower_bound) implies we have
+    //   tombstone_end_key.user_key > comp_start_user_key. So
+    //   tombstone_end_key.user_key > smallest_candidate.user_key. For
+    //   smallest_candidate to be > largest_candidate, we need
+    //   tombstone_start.user_key = largest_candidate.user_key = upper_bound =
+    //   next_table_min_key.user_key. This means `has_overlapping_endpoints` is
+    //   true (so meta.largest.user_key = upper_bound) and tombstone.seq_ <
+    //   meta.largest.seqno. So tombstone_start < meta.largest < tombstone_end.
   }
   return Status::OK();
 }

db/compaction/compaction_outputs.h

@@ -221,6 +221,13 @@ class CompactionOutputs {
     }
   }
 
+  // Updates states related to file cutting for TTL.
+  // Returns a boolean value indicating whether the current
+  // compaction output file should be cut before `internal_key`.
+  //
+  // @param internal_key the current key to be added to output.
+  bool UpdateFilesToCutForTTLStates(const Slice& internal_key);
+
   // update tracked grandparents information like grandparent index, if it's
   // in the gap between 2 grandparent files, accumulated grandparent files size
   // etc.

db/compaction/compaction_picker.cc

@@ -31,27 +31,15 @@ bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
                            size_t min_files_to_compact,
                            uint64_t max_compact_bytes_per_del_file,
                            uint64_t max_compaction_bytes,
-                           CompactionInputFiles* comp_inputs,
-                           SequenceNumber earliest_mem_seqno) {
-  // Do not pick ingested file when there is at least one memtable not flushed
-  // which of seqno is overlap with the sst.
+                           CompactionInputFiles* comp_inputs) {
   TEST_SYNC_POINT("FindIntraL0Compaction");
+
   size_t start = 0;
-  for (; start < level_files.size(); start++) {
-    if (level_files[start]->being_compacted) {
-      return false;
-    }
-    // If there is no data in memtable, the earliest sequence number would the
-    // largest sequence number in last memtable.
-    // Because all files are sorted in descending order by largest_seqno, so we
-    // only need to check the first one.
-    if (level_files[start]->fd.largest_seqno <= earliest_mem_seqno) {
-      break;
-    }
-  }
-  if (start >= level_files.size()) {
+
+  if (level_files.size() == 0 || level_files[start]->being_compacted) {
     return false;
   }
+
   size_t compact_bytes = static_cast<size_t>(level_files[start]->fd.file_size);
   size_t compact_bytes_per_del_file = std::numeric_limits<size_t>::max();
   // Compaction range will be [start, limit).
@@ -995,6 +983,7 @@ Status CompactionPicker::SanitizeCompactionInputFilesForAllLevels(
                                current_files[f].name +
                                " is currently being compacted.");
       }
+
       input_files->insert(TableFileNameToNumber(current_files[f].name));
     }
 
@@ -1137,7 +1126,11 @@ void CompactionPicker::RegisterCompaction(Compaction* c) {
          c->output_level() == 0 ||
          !FilesRangeOverlapWithCompaction(*c->inputs(), c->output_level(),
                                           c->GetPenultimateLevel()));
-  if (c->start_level() == 0 ||
+  // CompactionReason::kExternalSstIngestion's start level is just a placeholder
+  // number without actual meaning as file ingestion technically does not have
+  // an input level like other compactions
+  if ((c->start_level() == 0 &&
+       c->compaction_reason() != CompactionReason::kExternalSstIngestion) ||
       ioptions_.compaction_style == kCompactionStyleUniversal) {
     level0_compactions_in_progress_.insert(c);
   }

db/compaction/compaction_picker.h

@@ -51,14 +51,15 @@ class CompactionPicker {
   virtual ~CompactionPicker();
 
   // Pick level and inputs for a new compaction.
+  //
   // Returns nullptr if there is no compaction to be done.
   // Otherwise returns a pointer to a heap-allocated object that
   // describes the compaction.  Caller should delete the result.
-  virtual Compaction* PickCompaction(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
-      LogBuffer* log_buffer,
-      SequenceNumber earliest_memtable_seqno = kMaxSequenceNumber) = 0;
+  virtual Compaction* PickCompaction(const std::string& cf_name,
+                                     const MutableCFOptions& mutable_cf_options,
+                                     const MutableDBOptions& mutable_db_options,
+                                     VersionStorageInfo* vstorage,
+                                     LogBuffer* log_buffer) = 0;
 
   // Return a compaction object for compacting the range [begin,end] in
   // the specified level.  Returns nullptr if there is nothing in that
@@ -91,6 +92,7 @@ class CompactionPicker {
 // files.  If it's not possible to conver an invalid input_files
 // into a valid one by adding more files, the function will return a
 // non-ok status with specific reason.
+//
 #ifndef ROCKSDB_LITE
   Status SanitizeCompactionInputFiles(std::unordered_set<uint64_t>* input_files,
                                       const ColumnFamilyMetaData& cf_meta,
@@ -255,12 +257,11 @@ class NullCompactionPicker : public CompactionPicker {
   virtual ~NullCompactionPicker() {}
 
   // Always return "nullptr"
-  Compaction* PickCompaction(
-      const std::string& /*cf_name*/,
-      const MutableCFOptions& /*mutable_cf_options*/,
-      const MutableDBOptions& /*mutable_db_options*/,
-      VersionStorageInfo* /*vstorage*/, LogBuffer* /* log_buffer */,
-      SequenceNumber /* earliest_memtable_seqno */) override {
+  Compaction* PickCompaction(const std::string& /*cf_name*/,
+                             const MutableCFOptions& /*mutable_cf_options*/,
+                             const MutableDBOptions& /*mutable_db_options*/,
+                             VersionStorageInfo* /*vstorage*/,
+                             LogBuffer* /* log_buffer */) override {
     return nullptr;
   }
 
@@ -304,11 +305,11 @@ class NullCompactionPicker : public CompactionPicker {
 //                                        files. Cannot be nullptr.
 //
 // @return                                true iff compaction was found.
-bool FindIntraL0Compaction(
-    const std::vector<FileMetaData*>& level_files, size_t min_files_to_compact,
-    uint64_t max_compact_bytes_per_del_file, uint64_t max_compaction_bytes,
-    CompactionInputFiles* comp_inputs,
-    SequenceNumber earliest_mem_seqno = kMaxSequenceNumber);
+bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
+                           size_t min_files_to_compact,
+                           uint64_t max_compact_bytes_per_del_file,
+                           uint64_t max_compaction_bytes,
+                           CompactionInputFiles* comp_inputs);
 
 CompressionType GetCompressionType(const VersionStorageInfo* vstorage,
                                    const MutableCFOptions& mutable_cf_options,

db/compaction/compaction_picker_fifo.cc

@@ -232,7 +232,8 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
         break;
       }
     }
-  } else {
+  } else if (total_size >
+             mutable_cf_options.compaction_options_fifo.max_table_files_size) {
     // If the last level is non-L0, we actually don't know which file is
     // logically the oldest since the file creation time only represents
     // when this file was compacted to this level, which is independent
@@ -247,15 +248,27 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
       inputs[0].files.push_back(f);
       char tmp_fsize[16];
       AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize));
-      ROCKS_LOG_BUFFER(log_buffer,
-                       "[%s] FIFO compaction: picking file %" PRIu64
-                       " with size %s for deletion",
-                       cf_name.c_str(), f->fd.GetNumber(), tmp_fsize);
+      ROCKS_LOG_BUFFER(
+          log_buffer,
+          "[%s] FIFO compaction: picking file %" PRIu64
+          " with size %s for deletion under total size %" PRIu64
+          " vs max table files size %" PRIu64,
+          cf_name.c_str(), f->fd.GetNumber(), tmp_fsize, total_size,
+          mutable_cf_options.compaction_options_fifo.max_table_files_size);
+
       if (total_size <=
           mutable_cf_options.compaction_options_fifo.max_table_files_size) {
         break;
       }
     }
+  } else {
+    ROCKS_LOG_BUFFER(
+        log_buffer,
+        "[%s] FIFO compaction: nothing to do. Total size %" PRIu64
+        ", max size %" PRIu64 "\n",
+        cf_name.c_str(), total_size,
+        mutable_cf_options.compaction_options_fifo.max_table_files_size);
+    return nullptr;
   }
 
   Compaction* c = new Compaction(
@@ -389,7 +402,7 @@ Compaction* FIFOCompactionPicker::PickCompactionToWarm(
 Compaction* FIFOCompactionPicker::PickCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
-    LogBuffer* log_buffer, SequenceNumber /*earliest_memtable_seqno*/) {
+    LogBuffer* log_buffer) {
   Compaction* c = nullptr;
   if (mutable_cf_options.ttl > 0) {
     c = PickTTLCompaction(cf_name, mutable_cf_options, mutable_db_options,

db/compaction/compaction_picker_fifo.h

@@ -19,11 +19,11 @@ class FIFOCompactionPicker : public CompactionPicker {
                        const InternalKeyComparator* icmp)
       : CompactionPicker(ioptions, icmp) {}
 
-  virtual Compaction* PickCompaction(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      const MutableDBOptions& mutable_db_options, VersionStorageInfo* version,
-      LogBuffer* log_buffer,
-      SequenceNumber earliest_memtable_seqno = kMaxSequenceNumber) override;
+  virtual Compaction* PickCompaction(const std::string& cf_name,
+                                     const MutableCFOptions& mutable_cf_options,
+                                     const MutableDBOptions& mutable_db_options,
+                                     VersionStorageInfo* version,
+                                     LogBuffer* log_buffer) override;
 
   virtual Compaction* CompactRange(
       const std::string& cf_name, const MutableCFOptions& mutable_cf_options,

db/compaction/compaction_picker_level.cc

@@ -50,7 +50,6 @@ class LevelCompactionBuilder {
  public:
   LevelCompactionBuilder(const std::string& cf_name,
                          VersionStorageInfo* vstorage,
-                         SequenceNumber earliest_mem_seqno,
                          CompactionPicker* compaction_picker,
                          LogBuffer* log_buffer,
                          const MutableCFOptions& mutable_cf_options,
@@ -58,7 +57,6 @@ class LevelCompactionBuilder {
                          const MutableDBOptions& mutable_db_options)
       : cf_name_(cf_name),
         vstorage_(vstorage),
-        earliest_mem_seqno_(earliest_mem_seqno),
         compaction_picker_(compaction_picker),
         log_buffer_(log_buffer),
         mutable_cf_options_(mutable_cf_options),
@@ -122,7 +120,6 @@ class LevelCompactionBuilder {
 
   const std::string& cf_name_;
   VersionStorageInfo* vstorage_;
-  SequenceNumber earliest_mem_seqno_;
   CompactionPicker* compaction_picker_;
   LogBuffer* log_buffer_;
   int start_level_ = -1;
@@ -196,7 +193,10 @@ void LevelCompactionBuilder::SetupInitialFiles() {
       }
       output_level_ =
           (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1;
-      if (PickFileToCompact()) {
+      bool picked_file_to_compact = PickFileToCompact();
+      TEST_SYNC_POINT_CALLBACK("PostPickFileToCompact",
+                               &picked_file_to_compact);
+      if (picked_file_to_compact) {
         // found the compaction!
         if (start_level_ == 0) {
           // L0 score = `num L0 files` / `level0_file_num_compaction_trigger`
@@ -447,21 +447,21 @@ bool LevelCompactionBuilder::SetupOtherInputsIfNeeded() {
       compaction_inputs_.push_back(output_level_inputs_);
     }
 
+    // In some edge cases we could pick a compaction that will be compacting
+    // a key range that overlap with another running compaction, and both
+    // of them have the same output level. This could happen if
+    // (1) we are running a non-exclusive manual compaction
+    // (2) AddFile ingest a new file into the LSM tree
+    // We need to disallow this from happening.
+    if (compaction_picker_->FilesRangeOverlapWithCompaction(
+            compaction_inputs_, output_level_,
+            Compaction::EvaluatePenultimateLevel(
+                vstorage_, ioptions_, start_level_, output_level_))) {
+      // This compaction output could potentially conflict with the output
+      // of a currently running compaction, we cannot run it.
+      return false;
+    }
     if (!is_l0_trivial_move_) {
-      // In some edge cases we could pick a compaction that will be compacting
-      // a key range that overlap with another running compaction, and both
-      // of them have the same output level. This could happen if
-      // (1) we are running a non-exclusive manual compaction
-      // (2) AddFile ingest a new file into the LSM tree
-      // We need to disallow this from happening.
-      if (compaction_picker_->FilesRangeOverlapWithCompaction(
-              compaction_inputs_, output_level_,
-              Compaction::EvaluatePenultimateLevel(
-                  vstorage_, ioptions_, start_level_, output_level_))) {
-        // This compaction output could potentially conflict with the output
-        // of a currently running compaction, we cannot run it.
-        return false;
-      }
       compaction_picker_->GetGrandparents(vstorage_, start_level_inputs_,
                                           output_level_inputs_, &grandparents_);
     }
@@ -825,16 +825,16 @@ bool LevelCompactionBuilder::PickIntraL0Compaction() {
   return FindIntraL0Compaction(level_files, kMinFilesForIntraL0Compaction,
                                std::numeric_limits<uint64_t>::max(),
                                mutable_cf_options_.max_compaction_bytes,
-                               &start_level_inputs_, earliest_mem_seqno_);
+                               &start_level_inputs_);
 }
 }  // namespace
 
 Compaction* LevelCompactionPicker::PickCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
-    LogBuffer* log_buffer, SequenceNumber earliest_mem_seqno) {
-  LevelCompactionBuilder builder(cf_name, vstorage, earliest_mem_seqno, this,
-                                 log_buffer, mutable_cf_options, ioptions_,
+    LogBuffer* log_buffer) {
+  LevelCompactionBuilder builder(cf_name, vstorage, this, log_buffer,
+                                 mutable_cf_options, ioptions_,
                                  mutable_db_options);
   return builder.PickCompaction();
 }

db/compaction/compaction_picker_level.h

@@ -20,11 +20,11 @@ class LevelCompactionPicker : public CompactionPicker {
   LevelCompactionPicker(const ImmutableOptions& ioptions,
                         const InternalKeyComparator* icmp)
       : CompactionPicker(ioptions, icmp) {}
-  virtual Compaction* PickCompaction(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
-      LogBuffer* log_buffer,
-      SequenceNumber earliest_memtable_seqno = kMaxSequenceNumber) override;
+  virtual Compaction* PickCompaction(const std::string& cf_name,
+                                     const MutableCFOptions& mutable_cf_options,
+                                     const MutableDBOptions& mutable_db_options,
+                                     VersionStorageInfo* vstorage,
+                                     LogBuffer* log_buffer) override;
 
   virtual bool NeedsCompaction(
       const VersionStorageInfo* vstorage) const override;

db/compaction/compaction_picker_test.cc

@@ -77,8 +77,9 @@ class CompactionPickerTestBase : public testing::Test {
   void NewVersionStorage(int num_levels, CompactionStyle style) {
     DeleteVersionStorage();
     options_.num_levels = num_levels;
-    vstorage_.reset(new VersionStorageInfo(&icmp_, ucmp_, options_.num_levels,
-                                           style, nullptr, false));
+    vstorage_.reset(new VersionStorageInfo(
+        &icmp_, ucmp_, options_.num_levels, style, nullptr, false,
+        EpochNumberRequirement::kMustPresent));
     vstorage_->PrepareForVersionAppend(ioptions_, mutable_cf_options_);
   }
 
@@ -87,7 +88,7 @@ class CompactionPickerTestBase : public testing::Test {
   void AddVersionStorage() {
     temp_vstorage_.reset(new VersionStorageInfo(
         &icmp_, ucmp_, options_.num_levels, ioptions_.compaction_style,
-        vstorage_.get(), false));
+        vstorage_.get(), false, EpochNumberRequirement::kMustPresent));
   }
 
   void DeleteVersionStorage() {
@@ -105,7 +106,8 @@ class CompactionPickerTestBase : public testing::Test {
            size_t compensated_file_size = 0, bool marked_for_compact = false,
            Temperature temperature = Temperature::kUnknown,
            uint64_t oldest_ancestor_time = kUnknownOldestAncesterTime,
-           Slice ts_of_smallest = Slice(), Slice ts_of_largest = Slice()) {
+           Slice ts_of_smallest = Slice(), Slice ts_of_largest = Slice(),
+           uint64_t epoch_number = kUnknownEpochNumber) {
     assert(ts_of_smallest.size() == ucmp_->timestamp_size());
     assert(ts_of_largest.size() == ucmp_->timestamp_size());
 
@@ -145,8 +147,8 @@ class CompactionPickerTestBase : public testing::Test {
         file_number, path_id, file_size, smallest_ikey, largest_ikey,
         smallest_seq, largest_seq, marked_for_compact, temperature,
         kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
-        kUnknownFileCreationTime, kUnknownFileChecksum,
-        kUnknownFileChecksumFuncName, kNullUniqueId64x2);
+        kUnknownFileCreationTime, epoch_number, kUnknownFileChecksum,
+        kUnknownFileChecksumFuncName, kNullUniqueId64x2, 0);
     f->compensated_file_size =
         (compensated_file_size != 0) ? compensated_file_size : file_size;
     f->oldest_ancester_time = oldest_ancestor_time;
@@ -2871,39 +2873,6 @@ TEST_F(CompactionPickerTest, IntraL0MaxCompactionBytesHit) {
   ASSERT_EQ(0, compaction->output_level());
 }
 
-TEST_F(CompactionPickerTest, IntraL0ForEarliestSeqno) {
-  // Intra L0 compaction triggers only if there are at least
-  // level0_file_num_compaction_trigger + 2 L0 files.
-  mutable_cf_options_.level0_file_num_compaction_trigger = 3;
-  mutable_cf_options_.max_compaction_bytes = 999999u;
-  NewVersionStorage(6, kCompactionStyleLevel);
-
-  // 4 out of 6 L0 files will be picked for intra L0 compaction due to
-  // being_compact limit. And the latest one L0 will be skipped due to earliest
-  // seqno. The one L1 file spans entire L0 key range and is marked as being
-  // compacted to avoid L0->L1 compaction.
-  Add(1, 1U, "100", "350", 200000U, 0, 110, 111);
-  Add(0, 2U, "301", "350", 1U, 0, 108, 109);
-  Add(0, 3U, "251", "300", 1U, 0, 106, 107);
-  Add(0, 4U, "201", "250", 1U, 0, 104, 105);
-  Add(0, 5U, "151", "200", 1U, 0, 102, 103);
-  Add(0, 6U, "100", "150", 1U, 0, 100, 101);
-  Add(0, 7U, "100", "100", 1U, 0, 99, 100);
-  vstorage_->LevelFiles(0)[5]->being_compacted = true;
-  vstorage_->LevelFiles(1)[0]->being_compacted = true;
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, mutable_db_options_, vstorage_.get(),
-      &log_buffer_, 107));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_levels());
-  ASSERT_EQ(4U, compaction->num_input_files(0));
-  ASSERT_EQ(CompactionReason::kLevelL0FilesNum,
-            compaction->compaction_reason());
-  ASSERT_EQ(0, compaction->output_level());
-}
-
 #ifndef ROCKSDB_LITE
 TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
   const uint64_t kFileSize = 100000;
@@ -2916,9 +2885,23 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
   // should fail
   NewVersionStorage(5, kCompactionStyleUniversal);
 
-  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
-  Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450);
-  Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300);
+  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550, /*compensated_file_size*/ 0,
+      /*marked_for_compact*/ false, /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 3);
+  Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450,
+      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
+      /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 2);
+  Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300,
+      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
+      /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 1);
   Add(3, 5U, "010", "080", 8 * kFileSize, 0, 200, 251);
   Add(4, 3U, "301", "350", 8 * kFileSize, 0, 101, 150);
   Add(4, 6U, "501", "750", 8 * kFileSize, 0, 101, 150);
@@ -2940,7 +2923,11 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
 
   AddVersionStorage();
   // Simulate a flush and mark the file for compaction
-  Add(0, 7U, "150", "200", kFileSize, 0, 551, 600, 0, true);
+  Add(0, 7U, "150", "200", kFileSize, 0, 551, 600, 0, true,
+      /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 4);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction2(
@@ -2962,7 +2949,11 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap2) {
   NewVersionStorage(5, kCompactionStyleUniversal);
 
   // Mark file number 4 for compaction
-  Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300, 0, true);
+  Add(0, 4U, "260", "300", 4 * kFileSize, 0, 260, 300, 0, true,
+      /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 1);
   Add(3, 5U, "240", "290", 8 * kFileSize, 0, 201, 250);
   Add(4, 3U, "301", "350", 8 * kFileSize, 0, 101, 150);
   Add(4, 6U, "501", "750", 8 * kFileSize, 0, 101, 150);
@@ -2983,8 +2974,17 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap2) {
   ASSERT_EQ(1U, compaction->num_input_files(1));
 
   AddVersionStorage();
-  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
-  Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450);
+  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550, /*compensated_file_size*/ 0,
+      /*marked_for_compact*/ false, /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 3);
+  Add(0, 2U, "201", "250", 2 * kFileSize, 0, 401, 450,
+      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
+      /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 2);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction2(
@@ -3150,10 +3150,29 @@ TEST_F(CompactionPickerTest, UniversalMarkedL0Overlap2) {
   NewVersionStorage(1, kCompactionStyleUniversal);
 
   // Mark file number 5 for compaction
-  Add(0, 4U, "260", "300", 1 * kFileSize, 0, 260, 300);
-  Add(0, 5U, "240", "290", 2 * kFileSize, 0, 201, 250, 0, true);
-  Add(0, 3U, "301", "350", 4 * kFileSize, 0, 101, 150);
-  Add(0, 6U, "501", "750", 8 * kFileSize, 0, 50, 100);
+  Add(0, 4U, "260", "300", 1 * kFileSize, 0, 260, 300,
+      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
+      /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 4);
+  Add(0, 5U, "240", "290", 2 * kFileSize, 0, 201, 250, 0, true,
+      /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 3);
+  Add(0, 3U, "301", "350", 4 * kFileSize, 0, 101, 150,
+      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
+      /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 2);
+  Add(0, 6U, "501", "750", 8 * kFileSize, 0, 50, 100,
+      /*compensated_file_size*/ 0, /*marked_for_compact*/ false,
+      /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 1);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(
@@ -3173,8 +3192,18 @@ TEST_F(CompactionPickerTest, UniversalMarkedL0Overlap2) {
   ASSERT_TRUE(file_map_[6].first->being_compacted);
 
   AddVersionStorage();
-  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
-  Add(0, 2U, "201", "250", kFileSize, 0, 401, 450);
+  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550, /*compensated_file_size*/ 0,
+      /*marked_for_compact*/ false,
+      /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 6);
+  Add(0, 2U, "201", "250", kFileSize, 0, 401, 450, /*compensated_file_size*/ 0,
+      /*marked_for_compact*/ false,
+      /* temperature*/ Temperature::kUnknown,
+      /*oldest_ancestor_time*/ kUnknownOldestAncesterTime,
+      /*ts_of_smallest*/ Slice(), /*ts_of_largest*/ Slice(),
+      /*epoch_number*/ 5);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction2(

db/compaction/compaction_picker_universal.cc

@@ -293,7 +293,7 @@ bool UniversalCompactionPicker::NeedsCompaction(
 Compaction* UniversalCompactionPicker::PickCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
-    LogBuffer* log_buffer, SequenceNumber /* earliest_memtable_seqno */) {
+    LogBuffer* log_buffer) {
   UniversalCompactionBuilder builder(ioptions_, icmp_, cf_name,
                                      mutable_cf_options, mutable_db_options,
                                      vstorage, this, log_buffer);
@@ -400,6 +400,7 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
   if (!vstorage_->FilesMarkedForPeriodicCompaction().empty()) {
     // Always need to do a full compaction for periodic compaction.
     c = PickPeriodicCompaction();
+    TEST_SYNC_POINT_CALLBACK("PostPickPeriodicCompaction", c);
   }
 
   // Check for size amplification.
@@ -408,6 +409,7 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
           static_cast<size_t>(
               mutable_cf_options_.level0_file_num_compaction_trigger)) {
     if ((c = PickCompactionToReduceSizeAmp()) != nullptr) {
+      TEST_SYNC_POINT("PickCompactionToReduceSizeAmpReturnNonnullptr");
       ROCKS_LOG_BUFFER(log_buffer_, "[%s] Universal: compacting for size amp\n",
                        cf_name_.c_str());
     } else {
@@ -417,6 +419,7 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
           mutable_cf_options_.compaction_options_universal.size_ratio;
 
       if ((c = PickCompactionToReduceSortedRuns(ratio, UINT_MAX)) != nullptr) {
+        TEST_SYNC_POINT("PickCompactionToReduceSortedRunsReturnNonnullptr");
         ROCKS_LOG_BUFFER(log_buffer_,
                          "[%s] Universal: compacting for size ratio\n",
                          cf_name_.c_str());
@@ -457,6 +460,7 @@ Compaction* UniversalCompactionBuilder::PickCompaction() {
 
   if (c == nullptr) {
     if ((c = PickDeleteTriggeredCompaction()) != nullptr) {
+      TEST_SYNC_POINT("PickDeleteTriggeredCompactionReturnNonnullptr");
       ROCKS_LOG_BUFFER(log_buffer_,
                        "[%s] Universal: delete triggered compaction\n",
                        cf_name_.c_str());

db/compaction/compaction_picker_universal.h

@@ -18,11 +18,11 @@ class UniversalCompactionPicker : public CompactionPicker {
   UniversalCompactionPicker(const ImmutableOptions& ioptions,
                             const InternalKeyComparator* icmp)
       : CompactionPicker(ioptions, icmp) {}
-  virtual Compaction* PickCompaction(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      const MutableDBOptions& mutable_db_options, VersionStorageInfo* vstorage,
-      LogBuffer* log_buffer,
-      SequenceNumber earliest_memtable_seqno = kMaxSequenceNumber) override;
+  virtual Compaction* PickCompaction(const std::string& cf_name,
+                                     const MutableCFOptions& mutable_cf_options,
+                                     const MutableDBOptions& mutable_db_options,
+                                     VersionStorageInfo* vstorage,
+                                     LogBuffer* log_buffer) override;
   virtual int MaxOutputLevel() const override { return NumberLevels() - 1; }
 
   virtual bool NeedsCompaction(

db/compaction/compaction_service_job.cc

@@ -190,6 +190,7 @@ CompactionJob::ProcessKeyValueCompactionWithCompactionService(
     meta.largest.DecodeFrom(file.largest_internal_key);
     meta.oldest_ancester_time = file.oldest_ancester_time;
     meta.file_creation_time = file.file_creation_time;
+    meta.epoch_number = file.epoch_number;
     meta.marked_for_compaction = file.marked_for_compaction;
     meta.unique_id = file.unique_id;
 
@@ -333,8 +334,9 @@ Status CompactionServiceCompactionJob::Run() {
         MakeTableFileName(meta.fd.GetNumber()), meta.fd.smallest_seqno,
         meta.fd.largest_seqno, meta.smallest.Encode().ToString(),
         meta.largest.Encode().ToString(), meta.oldest_ancester_time,
-        meta.file_creation_time, output_file.validator.GetHash(),
-        meta.marked_for_compaction, meta.unique_id);
+        meta.file_creation_time, meta.epoch_number,
+        output_file.validator.GetHash(), meta.marked_for_compaction,
+        meta.unique_id);
   }
   InternalStats::CompactionStatsFull compaction_stats;
   sub_compact->AggregateCompactionStats(compaction_stats);
@@ -489,6 +491,10 @@ static std::unordered_map<std::string, OptionTypeInfo>
          {offsetof(struct CompactionServiceOutputFile, file_creation_time),
           OptionType::kUInt64T, OptionVerificationType::kNormal,
           OptionTypeFlags::kNone}},
+        {"epoch_number",
+         {offsetof(struct CompactionServiceOutputFile, epoch_number),
+          OptionType::kUInt64T, OptionVerificationType::kNormal,
+          OptionTypeFlags::kNone}},
         {"paranoid_hash",
          {offsetof(struct CompactionServiceOutputFile, paranoid_hash),
           OptionType::kUInt64T, OptionVerificationType::kNormal,

db/compaction/subcompaction_state.h

@@ -196,8 +196,11 @@ class SubcompactionState {
                               const CompactionFileCloseFunc& close_file_func) {
     // Call FinishCompactionOutputFile() even if status is not ok: it needs to
     // close the output file.
+    // CloseOutput() may open new compaction output files.
+    is_current_penultimate_level_ = true;
     Status s = penultimate_level_outputs_.CloseOutput(
         curr_status, open_file_func, close_file_func);
+    is_current_penultimate_level_ = false;
     s = compaction_outputs_.CloseOutput(s, open_file_func, close_file_func);
     return s;
   }

db/compaction/tiered_compaction_test.cc

@@ -663,8 +663,19 @@ TEST_P(TieredCompactionTest, LevelOutofBoundaryRangeDelete) {
   cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
   ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
 
-  ASSERT_EQ(GetSstSizeHelper(Temperature::kUnknown),
-            0);  // tombstone has no size, even it's in hot tier
+  // range tombstone is not in cold tier
+  ASSERT_GT(GetSstSizeHelper(Temperature::kUnknown), 0);
+  std::vector<std::vector<FileMetaData>> level_to_files;
+  dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
+                                  &level_to_files);
+  // range tombstone is in the penultimate level
+  const int penultimate_level = kNumLevels - 2;
+  ASSERT_EQ(level_to_files[penultimate_level].size(), 1);
+  ASSERT_EQ(level_to_files[penultimate_level][0].num_entries, 1);
+  ASSERT_EQ(level_to_files[penultimate_level][0].num_deletions, 1);
+  ASSERT_EQ(level_to_files[penultimate_level][0].temperature,
+            Temperature::kUnknown);
+
   ASSERT_GT(GetSstSizeHelper(Temperature::kCold), 0);
   ASSERT_EQ("0,1,10",
             FilesPerLevel());  // one file is at the penultimate level which
@@ -1240,7 +1251,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {
 
   // pass some time first, otherwise the first a few keys write time are going
   // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
       [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
 
   int sst_num = 0;
@@ -1248,7 +1259,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeManualCompaction) {
   for (; sst_num < kNumTrigger; sst_num++) {
     for (int i = 0; i < kNumKeys; i++) {
       ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
         mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
       });
     }
@@ -1302,7 +1313,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
 
   // pass some time first, otherwise the first a few keys write time are going
   // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
       [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
 
   int sst_num = 0;
@@ -1310,7 +1321,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
   for (; sst_num < kNumTrigger; sst_num++) {
     for (int i = 0; i < kNumKeys; i++) {
       ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
         mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
       });
     }
@@ -1344,7 +1355,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimeAutoCompaction) {
     for (int i = 0; i < kNumKeys; i++) {
       // the value needs to be big enough to trigger full compaction
       ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100)));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
         mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
       });
     }
@@ -1378,7 +1389,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {
 
   // pass some time first, otherwise the first a few keys write time are going
   // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
       [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
 
   int sst_num = 0;
@@ -1386,7 +1397,7 @@ TEST_F(PrecludeLastLevelTest, MigrationFromPreserveTimePartial) {
   for (; sst_num < kNumTrigger; sst_num++) {
     for (int i = 0; i < kNumKeys; i++) {
       ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
         mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
       });
     }
@@ -1452,13 +1463,13 @@ TEST_F(PrecludeLastLevelTest, SmallPrecludeTime) {
 
   Random rnd(301);
 
-  dbfull()->TEST_WaitForPeridicTaskRun([&] {
+  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
     mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(10) + 1));
   });
 
   for (int i = 0; i < kNumKeys; i++) {
     ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
-    dbfull()->TEST_WaitForPeridicTaskRun([&] {
+    dbfull()->TEST_WaitForPeriodicTaskRun([&] {
       mock_clock_->MockSleepForSeconds(static_cast<int>(rnd.Uniform(2)));
     });
   }
@@ -1505,7 +1516,7 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
 
   // pass some time first, otherwise the first a few keys write time are going
   // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
       [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
 
   int sst_num = 0;
@@ -1513,7 +1524,7 @@ TEST_F(PrecludeLastLevelTest, LastLevelOnlyCompactionPartial) {
   for (; sst_num < kNumTrigger; sst_num++) {
     for (int i = 0; i < kNumKeys; i++) {
       ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
         mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
       });
     }
@@ -1583,7 +1594,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
 
   // pass some time first, otherwise the first a few keys write time are going
   // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
       [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec)); });
 
   Random rnd(301);
@@ -1592,7 +1603,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
   for (; sst_num < kNumTrigger; sst_num++) {
     for (int i = 0; i < kNumKeys; i++) {
       ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), rnd.RandomString(100)));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
         mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
       });
     }
@@ -1685,7 +1696,7 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
     for (int i = 0; i < kNumKeys; i++) {
       // the value needs to be big enough to trigger full compaction
       ASSERT_OK(Put(Key(sst_num * (kNumKeys - 1) + i), "value"));
-      dbfull()->TEST_WaitForPeridicTaskRun([&] {
+      dbfull()->TEST_WaitForPeriodicTaskRun([&] {
         mock_clock_->MockSleepForSeconds(static_cast<int>(kKeyPerSec));
       });
     }
@@ -1710,6 +1721,133 @@ TEST_P(PrecludeLastLevelTestWithParms, LastLevelOnlyCompactionNoPreclude) {
   Close();
 }
 
+TEST_P(PrecludeLastLevelTestWithParms, PeriodicCompactionToPenultimateLevel) {
+  // Test the last level only periodic compaction should also be blocked by an
+  // ongoing compaction in penultimate level if tiered compaction is enabled
+  // otherwise, the periodic compaction should just run for the last level.
+  const int kNumTrigger = 4;
+  const int kNumLevels = 7;
+  const int kPenultimateLevel = kNumLevels - 2;
+  const int kKeyPerSec = 1;
+  const int kNumKeys = 100;
+
+  bool enable_preclude_last_level = GetParam();
+
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.preserve_internal_time_seconds = 20000;
+  options.env = mock_env_.get();
+  options.level0_file_num_compaction_trigger = kNumTrigger;
+  options.num_levels = kNumLevels;
+  options.ignore_max_compaction_bytes_for_input = false;
+  options.periodic_compaction_seconds = 10000;
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+
+  for (int i = 0; i < 3 * kNumKeys; i++) {
+    ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
+    dbfull()->TEST_WaitForPeriodicTaskRun(
+        [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
+  }
+  ASSERT_OK(Flush());
+  CompactRangeOptions cro;
+  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+
+  // make sure all data is compacted to the last level
+  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
+
+  // enable preclude feature
+  if (enable_preclude_last_level) {
+    options.preclude_last_level_data_seconds = 20000;
+  }
+  options.max_background_jobs = 8;
+  options.last_level_temperature = Temperature::kCold;
+  Reopen(options);
+
+  std::atomic_bool is_size_ratio_compaction_running = false;
+  std::atomic_bool verified_last_level_compaction = false;
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "CompactionJob::ProcessKeyValueCompaction()::Processing", [&](void* arg) {
+        auto compaction = static_cast<Compaction*>(arg);
+        if (compaction->output_level() == kPenultimateLevel) {
+          is_size_ratio_compaction_running = true;
+          TEST_SYNC_POINT(
+              "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+              "SizeRatioCompaction1");
+          TEST_SYNC_POINT(
+              "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+              "SizeRatioCompaction2");
+          is_size_ratio_compaction_running = false;
+        }
+      });
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) {
+        auto compaction = static_cast<Compaction*>(arg);
+
+        if (is_size_ratio_compaction_running) {
+          if (enable_preclude_last_level) {
+            ASSERT_TRUE(compaction == nullptr);
+          } else {
+            ASSERT_TRUE(compaction != nullptr);
+            ASSERT_EQ(compaction->compaction_reason(),
+                      CompactionReason::kPeriodicCompaction);
+            ASSERT_EQ(compaction->start_level(), kNumLevels - 1);
+          }
+          verified_last_level_compaction = true;
+        }
+        TEST_SYNC_POINT(
+            "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+            "AutoCompactionPicked");
+      });
+
+  SyncPoint::GetInstance()->LoadDependency({
+      {"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+       "SizeRatioCompaction1",
+       "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:DoneWrite"},
+      {"PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+       "AutoCompactionPicked",
+       "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:"
+       "SizeRatioCompaction2"},
+  });
+
+  auto stop_token =
+      dbfull()->TEST_write_controler().GetCompactionPressureToken();
+
+  for (int i = 0; i < kNumTrigger - 1; i++) {
+    for (int j = 0; j < kNumKeys; j++) {
+      ASSERT_OK(Put(Key(i * (kNumKeys - 1) + i), rnd.RandomString(10)));
+      dbfull()->TEST_WaitForPeriodicTaskRun(
+          [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
+    }
+    ASSERT_OK(Flush());
+  }
+
+  TEST_SYNC_POINT(
+      "PrecludeLastLevelTest::PeriodicCompactionToPenultimateLevel:DoneWrite");
+
+  // wait for periodic compaction time and flush to trigger the periodic
+  // compaction, which should be blocked by ongoing compaction in the
+  // penultimate level
+  mock_clock_->MockSleepForSeconds(10000);
+  for (int i = 0; i < 3 * kNumKeys; i++) {
+    ASSERT_OK(Put(Key(i), rnd.RandomString(10)));
+    dbfull()->TEST_WaitForPeriodicTaskRun(
+        [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
+  }
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(dbfull()->WaitForCompact(true));
+
+  stop_token.reset();
+
+  Close();
+}
+
 INSTANTIATE_TEST_CASE_P(PrecludeLastLevelTestWithParms,
                         PrecludeLastLevelTestWithParms, testing::Bool());
 
@@ -1770,14 +1908,14 @@ TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) {
 
   // pass some time first, otherwise the first a few keys write time are going
   // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
+  dbfull()->TEST_WaitForPeriodicTaskRun(
       [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
 
   Random rnd(301);
 
   for (int i = 0; i < 300; i++) {
     ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
-    dbfull()->TEST_WaitForPeridicTaskRun(
+    dbfull()->TEST_WaitForPeriodicTaskRun(
         [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
   }
   ASSERT_OK(Flush());
@@ -1858,155 +1996,145 @@ TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompaction) {
   Close();
 }
 
-struct TestPropertiesCollector : public TablePropertiesCollector {
-  Status AddUserKey(const Slice& key, const Slice& /*value*/,
-                    EntryType /*type*/, SequenceNumber /*seq*/,
-                    uint64_t /*file_size*/) override {
-    if (cmp->Compare(key, DBTestBase::Key(100)) == 0) {
-      has_key_100 = true;
-    }
-    if (cmp->Compare(key, DBTestBase::Key(200)) == 0) {
-      has_key_200 = true;
-    }
-
-    return Status::OK();
-  }
-
-  const char* Name() const override { return "TestTablePropertiesCollector"; }
-
-  UserCollectedProperties GetReadableProperties() const override {
-    UserCollectedProperties ret;
-    return ret;
-  }
-
-  Status Finish(UserCollectedProperties* /*properties*/) override {
-    // The LSM tree would be like:
-    // L5: [0,19] [20,39] [40,299]
-    // L6: [0,                299]
-    // the 3rd file @L5 has both 100 and 200, which will be marked for
-    // compaction
-    // Also avoid marking flushed SST for compaction, which won't have both 100
-    // and 200
-    if (has_key_100 && has_key_200) {
-      need_compact_ = true;
-    } else {
-      need_compact_ = false;
-    }
-    has_key_100 = false;
-    has_key_200 = false;
-    return Status::OK();
-  }
-
-  bool NeedCompact() const override { return need_compact_; }
-
-  const Comparator* cmp = BytewiseComparator();
-
- private:
-  bool has_key_100 = false;
-  bool has_key_200 = false;
-
-  bool need_compact_ = false;
-};
-
-class TestPropertiesCollectorFactory : public TablePropertiesCollectorFactory {
- public:
-  TablePropertiesCollector* CreateTablePropertiesCollector(
-      TablePropertiesCollectorFactory::Context /*context*/) override {
-    return new TestPropertiesCollector;
-  }
-  const char* Name() const override { return "TestTablePropertiesCollector"; }
-};
-
-TEST_F(PrecludeLastLevelTest, PartialPenultimateLevelCompactionWithRangeDel) {
-  const int kNumTrigger = 4;
+TEST_F(PrecludeLastLevelTest, RangeDelsCauseFileEndpointsToOverlap) {
   const int kNumLevels = 7;
-  const int kKeyPerSec = 10;
+  const int kSecondsPerKey = 10;
+  const int kNumFiles = 3;
+  const int kValueBytes = 4 << 10;
+  const int kFileBytes = 4 * kValueBytes;
+  // `kNumKeysPerFile == 5` is determined by the current file cutting heuristics
+  // for this choice of `kValueBytes` and `kFileBytes`.
+  const int kNumKeysPerFile = 5;
+  const int kNumKeys = kNumFiles * kNumKeysPerFile;
 
   Options options = CurrentOptions();
   options.compaction_style = kCompactionStyleUniversal;
   options.env = mock_env_.get();
-  options.level0_file_num_compaction_trigger = kNumTrigger;
-  options.preserve_internal_time_seconds = 10000;
+  options.last_level_temperature = Temperature::kCold;
+  options.preserve_internal_time_seconds = 600;
+  options.preclude_last_level_data_seconds = 1;
   options.num_levels = kNumLevels;
-  // set a small max_compaction_bytes to avoid input level expansion
-  options.max_compaction_bytes = 30000;
-  options.ignore_max_compaction_bytes_for_input = false;
+  options.target_file_size_base = kFileBytes;
   DestroyAndReopen(options);
 
   // pass some time first, otherwise the first a few keys write time are going
   // to be zero, and internally zero has special meaning: kUnknownSeqnoTime
-  dbfull()->TEST_WaitForPeridicTaskRun(
-      [&] { mock_clock_->MockSleepForSeconds(static_cast<int>(10)); });
+  dbfull()->TEST_WaitForPeriodicTaskRun([&] {
+    mock_clock_->MockSleepForSeconds(static_cast<int>(kSecondsPerKey));
+  });
 
-  Random rnd(301);
-
-  for (int i = 0; i < 300; i++) {
-    ASSERT_OK(Put(Key(i), rnd.RandomString(100)));
-    dbfull()->TEST_WaitForPeridicTaskRun(
-        [&] { mock_clock_->MockSleepForSeconds(kKeyPerSec); });
-  }
-  ASSERT_OK(Flush());
-  CompactRangeOptions cro;
-  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
-
-  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
-
-  // make sure all data is compacted to the last level
-  ASSERT_EQ("0,0,0,0,0,0,1", FilesPerLevel());
-
-  // Create 3 L5 files
-  auto factory = std::make_shared<ThreeRangesPartitionerFactory>();
-  options.sst_partitioner_factory = factory;
-
-  // the user defined properties_collector will mark the 3rd file for compaction
-  auto collector_factory = std::make_shared<TestPropertiesCollectorFactory>();
-  options.table_properties_collector_factories.resize(1);
-  options.table_properties_collector_factories[0] = collector_factory;
-  // enable tiered storage feature
-  options.preclude_last_level_data_seconds = 10000;
-  options.last_level_temperature = Temperature::kCold;
-  Reopen(options);
-
-  for (int i = 0; i < kNumTrigger - 2; i++) {
-    for (int j = 0; j < 100; j++) {
-      ASSERT_OK(Put(Key(i * 100 + j), rnd.RandomString(10)));
+  // Flush an L0 file with the following contents (new to old):
+  //
+  // Range deletions [4, 6) [7, 8) [9, 11)
+  // --- snap2 ---
+  // Key(0) .. Key(14)
+  // --- snap1 ---
+  // Key(3) .. Key(17)
+  const auto verify_db = [&]() {
+    for (int i = 0; i < kNumKeys; i++) {
+      std::string value;
+      auto s = db_->Get(ReadOptions(), Key(i), &value);
+      if (i == 4 || i == 5 || i == 7 || i == 9 || i == 10) {
+        ASSERT_TRUE(s.IsNotFound());
+      } else {
+        ASSERT_OK(s);
+      }
     }
-    ASSERT_OK(Flush());
+  };
+  Random rnd(301);
+  for (int i = 0; i < kNumKeys; i++) {
+    ASSERT_OK(Put(Key(i + 3), rnd.RandomString(kValueBytes)));
+    dbfull()->TEST_WaitForPeriodicTaskRun(
+        [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); });
   }
+  auto* snap1 = db_->GetSnapshot();
+  for (int i = 0; i < kNumKeys; i++) {
+    ASSERT_OK(Put(Key(i), rnd.RandomString(kValueBytes)));
+    dbfull()->TEST_WaitForPeriodicTaskRun(
+        [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); });
+  }
+  auto* snap2 = db_->GetSnapshot();
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
+                             Key(kNumKeysPerFile - 1),
+                             Key(kNumKeysPerFile + 1)));
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
+                             Key(kNumKeysPerFile + 2),
+                             Key(kNumKeysPerFile + 3)));
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
+                             Key(2 * kNumKeysPerFile - 1),
+                             Key(2 * kNumKeysPerFile + 1)));
+  ASSERT_OK(Flush());
+  dbfull()->TEST_WaitForPeriodicTaskRun(
+      [&] { mock_clock_->MockSleepForSeconds(kSecondsPerKey); });
+  verify_db();
 
-  // make sure there is one and only one compaction supports per-key placement
-  // but has the penultimate level output disabled.
+  // Count compactions supporting per-key placement
   std::atomic_int per_key_comp_num = 0;
   SyncPoint::GetInstance()->SetCallBack(
       "UniversalCompactionBuilder::PickCompaction:Return", [&](void* arg) {
         auto compaction = static_cast<Compaction*>(arg);
         if (compaction->SupportsPerKeyPlacement()) {
           ASSERT_EQ(compaction->GetPenultimateOutputRangeType(),
-                    Compaction::PenultimateOutputRangeType::kDisabled);
+                    Compaction::PenultimateOutputRangeType::kNonLastRange);
           per_key_comp_num++;
         }
       });
-
   SyncPoint::GetInstance()->EnableProcessing();
 
-  for (int j = 0; j < 100; j++) {
-    ASSERT_OK(Put(Key(200 + j), rnd.RandomString(10)));
-  }
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
-                             Key(32), Key(40)));
-  ASSERT_OK(Flush());
-
-  // Before the per-key placement compaction, the LSM tress should be like:
-  // L5: [0,19] [20,40] [40,299]
-  // L6: [0,                299]
-  // The 2nd file @L5 has the largest key 40 because of range del
+  // The `CompactRange()` writes the following files to L5.
+  //
+  //   [key000000#16,kTypeValue,
+  //    key000005#kMaxSequenceNumber,kTypeRangeDeletion]
+  //   [key000005#21,kTypeValue,
+  //    key000010#kMaxSequenceNumber,kTypeRangeDeletion]
+  //   [key000010#26,kTypeValue, key000014#30,kTypeValue]
+  //
+  // And it writes the following files to L6.
+  //
+  //   [key000003#1,kTypeValue, key000007#5,kTypeValue]
+  //   [key000008#6,kTypeValue, key000012#10,kTypeValue]
+  //   [key000013#11,kTypeValue, key000017#15,kTypeValue]
+  CompactRangeOptions cro;
+  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+  ASSERT_EQ("0,0,0,0,0,3,3", FilesPerLevel());
+  verify_db();
 
+  // Rewrite the middle file only. File endpoints should not change.
+  std::string begin_key_buf = Key(kNumKeysPerFile + 1),
+              end_key_buf = Key(kNumKeysPerFile + 2);
+  Slice begin_key(begin_key_buf), end_key(end_key_buf);
+  ASSERT_OK(db_->SuggestCompactRange(db_->DefaultColumnFamily(), &begin_key,
+                                     &end_key));
   ASSERT_OK(dbfull()->WaitForCompact(true));
+  ASSERT_EQ("0,0,0,0,0,3,3", FilesPerLevel());
+  ASSERT_EQ(1, per_key_comp_num);
+  verify_db();
 
-  ASSERT_EQ(per_key_comp_num, 1);
+  // Rewrite the middle file again after releasing snap2. Still file endpoints
+  // should not change.
+  db_->ReleaseSnapshot(snap2);
+  ASSERT_OK(db_->SuggestCompactRange(db_->DefaultColumnFamily(), &begin_key,
+                                     &end_key));
+  ASSERT_OK(dbfull()->WaitForCompact(true));
+  ASSERT_EQ("0,0,0,0,0,3,3", FilesPerLevel());
+  ASSERT_EQ(2, per_key_comp_num);
+  verify_db();
 
-  // the compaction won't move any data to the penultimate level
+  // Middle file once more after releasing snap1. This time the data in the
+  // middle L5 file can all be compacted to the last level.
+  db_->ReleaseSnapshot(snap1);
+  ASSERT_OK(db_->SuggestCompactRange(db_->DefaultColumnFamily(), &begin_key,
+                                     &end_key));
+  ASSERT_OK(dbfull()->WaitForCompact(true));
   ASSERT_EQ("0,0,0,0,0,2,3", FilesPerLevel());
+  ASSERT_EQ(3, per_key_comp_num);
+  verify_db();
+
+  // Finish off the penultimate level.
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+  ASSERT_EQ("0,0,0,0,0,0,3", FilesPerLevel());
+  verify_db();
 
   Close();
 }

db/corruption_test.cc

@@ -223,7 +223,8 @@ class CorruptionTest : public testing::Test {
     }
     ASSERT_TRUE(!fname.empty()) << filetype;
 
-    ASSERT_OK(test::CorruptFile(env_, fname, offset, bytes_to_corrupt));
+    ASSERT_OK(test::CorruptFile(env_, fname, offset, bytes_to_corrupt,
+                                /*verify_checksum*/ filetype == kTableFile));
   }
 
   // corrupts exactly one file at level `level`. if no file found at level,
@@ -518,6 +519,90 @@ TEST_F(CorruptionTest, TableFileIndexData) {
   ASSERT_TRUE(TryReopen().IsCorruption());
 }
 
+TEST_F(CorruptionTest, TableFileFooterMagic) {
+  Build(100);
+  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
+  ASSERT_OK(dbi->TEST_FlushMemTable());
+  Check(100, 100);
+  // Corrupt the whole footer
+  Corrupt(kTableFile, -100, 100);
+  Status s = TryReopen();
+  ASSERT_TRUE(s.IsCorruption());
+  // Contains useful message, and magic number should be the first thing
+  // reported as corrupt.
+  ASSERT_TRUE(s.ToString().find("magic number") != std::string::npos);
+  // with file name
+  ASSERT_TRUE(s.ToString().find(".sst") != std::string::npos);
+}
+
+TEST_F(CorruptionTest, TableFileFooterNotMagic) {
+  Build(100);
+  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
+  ASSERT_OK(dbi->TEST_FlushMemTable());
+  Check(100, 100);
+  // Corrupt footer except magic number
+  Corrupt(kTableFile, -100, 92);
+  Status s = TryReopen();
+  ASSERT_TRUE(s.IsCorruption());
+  // The next thing checked after magic number is format_version
+  ASSERT_TRUE(s.ToString().find("format_version") != std::string::npos);
+  // with file name
+  ASSERT_TRUE(s.ToString().find(".sst") != std::string::npos);
+}
+
+TEST_F(CorruptionTest, TableFileWrongSize) {
+  Build(100);
+  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
+  ASSERT_OK(dbi->TEST_FlushMemTable());
+  Check(100, 100);
+
+  // ********************************************
+  // Make the file bigger by appending to it
+  std::vector<LiveFileMetaData> metadata;
+  db_->GetLiveFilesMetaData(&metadata);
+  ASSERT_EQ(1U, metadata.size());
+  std::string filename = dbname_ + metadata[0].name;
+  const auto& fs = options_.env->GetFileSystem();
+  {
+    std::unique_ptr<FSWritableFile> f;
+    ASSERT_OK(fs->ReopenWritableFile(filename, FileOptions(), &f, nullptr));
+    ASSERT_OK(f->Append("blahblah", IOOptions(), nullptr));
+    ASSERT_OK(f->Close(IOOptions(), nullptr));
+  }
+
+  // DB actually accepts this without paranoid checks, relying on size
+  // recorded in manifest to locate the SST footer.
+  options_.paranoid_checks = false;
+  options_.skip_checking_sst_file_sizes_on_db_open = false;
+  Reopen();
+  Check(100, 100);
+
+  // But reports the issue with paranoid checks
+  options_.paranoid_checks = true;
+  Status s = TryReopen();
+  ASSERT_TRUE(s.IsCorruption());
+  ASSERT_TRUE(s.ToString().find("file size mismatch") != std::string::npos);
+
+  // ********************************************
+  // Make the file smaller with truncation.
+  // First leaving a partial footer, and then completely removing footer.
+  for (size_t bytes_lost : {8, 100}) {
+    ASSERT_OK(
+        test::TruncateFile(env_, filename, metadata[0].size - bytes_lost));
+
+    // Reported well with paranoid checks
+    options_.paranoid_checks = true;
+    s = TryReopen();
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(s.ToString().find("file size mismatch") != std::string::npos);
+
+    // Without paranoid checks, not reported until read
+    options_.paranoid_checks = false;
+    Reopen();
+    Check(0, 0);  // Missing data
+  }
+}
+
 TEST_F(CorruptionTest, MissingDescriptor) {
   Build(1000);
   RepairDB();

db/db_basic_test.cc

@@ -32,6 +32,9 @@
 
 namespace ROCKSDB_NAMESPACE {
 
+static bool enable_io_uring = true;
+extern "C" bool RocksDbIOUringEnable() { return enable_io_uring; }
+
 class DBBasicTest : public DBTestBase {
  public:
   DBBasicTest() : DBTestBase("db_basic_test", /*env_do_fsync=*/false) {}
@@ -2162,6 +2165,7 @@ class DBMultiGetAsyncIOTest : public DBBasicTest,
     options_.disable_auto_compactions = true;
     options_.statistics = statistics_;
     options_.table_factory.reset(NewBlockBasedTableFactory(bbto));
+    options_.env = Env::Default();
     Reopen(options_);
     int num_keys = 0;
 
@@ -2228,6 +2232,20 @@ class DBMultiGetAsyncIOTest : public DBBasicTest,
   const std::shared_ptr<Statistics>& statistics() { return statistics_; }
 
  protected:
+  void PrepareDBForTest() {
+#ifdef ROCKSDB_IOURING_PRESENT
+    Reopen(options_);
+#else   // ROCKSDB_IOURING_PRESENT
+    // Warm up the block cache so we don't need to use the IO uring
+    Iterator* iter = dbfull()->NewIterator(ReadOptions());
+    for (iter->SeekToFirst(); iter->Valid() && iter->status().ok();
+         iter->Next())
+      ;
+    EXPECT_OK(iter->status());
+    delete iter;
+#endif  // ROCKSDB_IOURING_PRESENT
+  }
+
   void ReopenDB() { Reopen(options_); }
 
  private:
@@ -2242,6 +2260,8 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL0) {
   std::vector<PinnableSlice> values(key_strs.size());
   std::vector<Status> statuses(key_strs.size());
 
+  PrepareDBForTest();
+
   ReadOptions ro;
   ro.async_io = true;
   ro.optimize_multiget_for_io = GetParam();
@@ -2260,6 +2280,7 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL0) {
   statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size);
 
   // With async IO, lookups will happen in parallel for each key
+#ifdef ROCKSDB_IOURING_PRESENT
   if (GetParam()) {
     ASSERT_EQ(multiget_io_batch_size.count, 1);
     ASSERT_EQ(multiget_io_batch_size.max, 3);
@@ -2269,6 +2290,11 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL0) {
     // L0 file
     ASSERT_EQ(multiget_io_batch_size.count, 3);
   }
+#else   // ROCKSDB_IOURING_PRESENT
+  if (GetParam()) {
+    ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 3);
+  }
+#endif  // ROCKSDB_IOURING_PRESENT
 }
 
 TEST_P(DBMultiGetAsyncIOTest, GetFromL1) {
@@ -2286,6 +2312,8 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1) {
   values.resize(keys.size());
   statuses.resize(keys.size());
 
+  PrepareDBForTest();
+
   ReadOptions ro;
   ro.async_io = true;
   ro.optimize_multiget_for_io = GetParam();
@@ -2299,6 +2327,7 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1) {
   ASSERT_EQ(values[1], "val_l1_" + std::to_string(54));
   ASSERT_EQ(values[2], "val_l1_" + std::to_string(102));
 
+#ifdef ROCKSDB_IOURING_PRESENT
   HistogramData multiget_io_batch_size;
 
   statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size);
@@ -2306,9 +2335,11 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1) {
   // A batch of 3 async IOs is expected, one for each overlapping file in L1
   ASSERT_EQ(multiget_io_batch_size.count, 1);
   ASSERT_EQ(multiget_io_batch_size.max, 3);
+#endif  // ROCKSDB_IOURING_PRESENT
   ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 3);
 }
 
+#ifdef ROCKSDB_IOURING_PRESENT
 TEST_P(DBMultiGetAsyncIOTest, GetFromL1Error) {
   std::vector<std::string> key_strs;
   std::vector<Slice> keys;
@@ -2324,9 +2355,9 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1Error) {
   values.resize(keys.size());
   statuses.resize(keys.size());
 
+  int count = 0;
   SyncPoint::GetInstance()->SetCallBack(
       "TableCache::GetTableReader:BeforeOpenFile", [&](void* status) {
-        static int count = 0;
         count++;
         // Fail the last table reader open, which is the 6th SST file
         // since 3 overlapping L0 files + 3 L1 files containing the keys
@@ -2349,7 +2380,7 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1Error) {
       });
   SyncPoint::GetInstance()->EnableProcessing();
 
-  ReopenDB();
+  PrepareDBForTest();
 
   ReadOptions ro;
   ro.async_io = true;
@@ -2371,6 +2402,7 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1Error) {
   ASSERT_EQ(multiget_io_batch_size.max, 2);
   ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 2);
 }
+#endif  // ROCKSDB_IOURING_PRESENT
 
 TEST_P(DBMultiGetAsyncIOTest, LastKeyInFile) {
   std::vector<std::string> key_strs;
@@ -2388,6 +2420,8 @@ TEST_P(DBMultiGetAsyncIOTest, LastKeyInFile) {
   values.resize(keys.size());
   statuses.resize(keys.size());
 
+  PrepareDBForTest();
+
   ReadOptions ro;
   ro.async_io = true;
   ro.optimize_multiget_for_io = GetParam();
@@ -2401,6 +2435,7 @@ TEST_P(DBMultiGetAsyncIOTest, LastKeyInFile) {
   ASSERT_EQ(values[1], "val_l1_" + std::to_string(54));
   ASSERT_EQ(values[2], "val_l1_" + std::to_string(102));
 
+#ifdef ROCKSDB_IOURING_PRESENT
   HistogramData multiget_io_batch_size;
 
   statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size);
@@ -2411,6 +2446,7 @@ TEST_P(DBMultiGetAsyncIOTest, LastKeyInFile) {
   // will lookup 2 files in parallel and issue 2 async reads
   ASSERT_EQ(multiget_io_batch_size.count, 2);
   ASSERT_EQ(multiget_io_batch_size.max, 2);
+#endif  // ROCKSDB_IOURING_PRESENT
 }
 
 TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2) {
@@ -2429,6 +2465,8 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2) {
   values.resize(keys.size());
   statuses.resize(keys.size());
 
+  PrepareDBForTest();
+
   ReadOptions ro;
   ro.async_io = true;
   ro.optimize_multiget_for_io = GetParam();
@@ -2442,6 +2480,7 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2) {
   ASSERT_EQ(values[1], "val_l2_" + std::to_string(56));
   ASSERT_EQ(values[2], "val_l1_" + std::to_string(102));
 
+#ifdef ROCKSDB_IOURING_PRESENT
   HistogramData multiget_io_batch_size;
 
   statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size);
@@ -2451,6 +2490,7 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2) {
   // Otherwise, the L2 lookup will happen after L1.
   ASSERT_EQ(multiget_io_batch_size.count, GetParam() ? 1 : 2);
   ASSERT_EQ(multiget_io_batch_size.max, GetParam() ? 3 : 2);
+#endif  // ROCKSDB_IOURING_PRESENT
 }
 
 TEST_P(DBMultiGetAsyncIOTest, GetFromL2WithRangeOverlapL0L1) {
@@ -2467,6 +2507,8 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL2WithRangeOverlapL0L1) {
   values.resize(keys.size());
   statuses.resize(keys.size());
 
+  PrepareDBForTest();
+
   ReadOptions ro;
   ro.async_io = true;
   ro.optimize_multiget_for_io = GetParam();
@@ -2482,6 +2524,7 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL2WithRangeOverlapL0L1) {
   ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 2);
 }
 
+#ifdef ROCKSDB_IOURING_PRESENT
 TEST_P(DBMultiGetAsyncIOTest, GetFromL2WithRangeDelInL1) {
   std::vector<std::string> key_strs;
   std::vector<Slice> keys;
@@ -2496,6 +2539,8 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL2WithRangeDelInL1) {
   values.resize(keys.size());
   statuses.resize(keys.size());
 
+  PrepareDBForTest();
+
   ReadOptions ro;
   ro.async_io = true;
   ro.optimize_multiget_for_io = GetParam();
@@ -2525,6 +2570,8 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2WithRangeDelInL1) {
   values.resize(keys.size());
   statuses.resize(keys.size());
 
+  PrepareDBForTest();
+
   ReadOptions ro;
   ro.async_io = true;
   ro.optimize_multiget_for_io = GetParam();
@@ -2539,6 +2586,45 @@ TEST_P(DBMultiGetAsyncIOTest, GetFromL1AndL2WithRangeDelInL1) {
   // Bloom filters in L0/L1 will avoid the coroutine calls in those levels
   ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 3);
 }
+#endif  // ROCKSDB_IOURING_PRESENT
+
+TEST_P(DBMultiGetAsyncIOTest, GetNoIOUring) {
+  std::vector<std::string> key_strs;
+  std::vector<Slice> keys;
+  std::vector<PinnableSlice> values;
+  std::vector<Status> statuses;
+
+  key_strs.push_back(Key(33));
+  key_strs.push_back(Key(54));
+  key_strs.push_back(Key(102));
+  keys.push_back(key_strs[0]);
+  keys.push_back(key_strs[1]);
+  keys.push_back(key_strs[2]);
+  values.resize(keys.size());
+  statuses.resize(keys.size());
+
+  enable_io_uring = false;
+  ReopenDB();
+
+  ReadOptions ro;
+  ro.async_io = true;
+  ro.optimize_multiget_for_io = GetParam();
+  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
+                     keys.data(), values.data(), statuses.data());
+  ASSERT_EQ(values.size(), 3);
+  ASSERT_EQ(statuses[0], Status::NotSupported());
+  ASSERT_EQ(statuses[1], Status::NotSupported());
+  ASSERT_EQ(statuses[2], Status::NotSupported());
+
+  HistogramData multiget_io_batch_size;
+
+  statistics()->histogramData(MULTIGET_IO_BATCH_SIZE, &multiget_io_batch_size);
+
+  // A batch of 3 async IOs is expected, one for each overlapping file in L1
+  ASSERT_EQ(multiget_io_batch_size.count, 1);
+  ASSERT_EQ(multiget_io_batch_size.max, 3);
+  ASSERT_EQ(statistics()->getTickerCount(MULTIGET_COROUTINE_COUNT), 3);
+}
 
 INSTANTIATE_TEST_CASE_P(DBMultiGetAsyncIOTest, DBMultiGetAsyncIOTest,
                         testing::Bool());
@@ -3537,24 +3623,27 @@ class DBBasicTestMultiGet : public DBTestBase {
 
     const char* Name() const override { return "MyBlockCache"; }
 
-    using Cache::Insert;
-    Status Insert(const Slice& key, void* value, size_t charge,
-                  void (*deleter)(const Slice& key, void* value),
+    Status Insert(const Slice& key, Cache::ObjectPtr value,
+                  const CacheItemHelper* helper, size_t charge,
                   Handle** handle = nullptr,
                   Priority priority = Priority::LOW) override {
       num_inserts_++;
-      return target_->Insert(key, value, charge, deleter, handle, priority);
+      return target_->Insert(key, value, helper, charge, handle, priority);
     }
 
-    using Cache::Lookup;
-    Handle* Lookup(const Slice& key, Statistics* stats = nullptr) override {
+    Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
+                   CreateContext* create_context,
+                   Priority priority = Priority::LOW, bool wait = true,
+                   Statistics* stats = nullptr) override {
       num_lookups_++;
-      Handle* handle = target_->Lookup(key, stats);
+      Handle* handle =
+          target_->Lookup(key, helper, create_context, priority, wait, stats);
       if (handle != nullptr) {
         num_found_++;
       }
       return handle;
     }
+
     int num_lookups() { return num_lookups_; }
 
     int num_found() { return num_found_; }

db/db_block_cache_test.cc

@@ -14,6 +14,7 @@
 #include "cache/cache_entry_roles.h"
 #include "cache/cache_key.h"
 #include "cache/lru_cache.h"
+#include "cache/typed_cache.h"
 #include "db/column_family.h"
 #include "db/db_impl/db_impl.h"
 #include "db/db_test_util.h"
@@ -365,9 +366,7 @@ class PersistentCacheFromCache : public PersistentCache {
     }
     std::unique_ptr<char[]> copy{new char[size]};
     std::copy_n(data, size, copy.get());
-    Status s = cache_->Insert(
-        key, copy.get(), size,
-        GetCacheEntryDeleterForRole<char[], CacheEntryRole::kMisc>());
+    Status s = cache_.Insert(key, copy.get(), size);
     if (s.ok()) {
       copy.release();
     }
@@ -376,13 +375,13 @@ class PersistentCacheFromCache : public PersistentCache {
 
   Status Lookup(const Slice& key, std::unique_ptr<char[]>* data,
                 size_t* size) override {
-    auto handle = cache_->Lookup(key);
+    auto handle = cache_.Lookup(key);
     if (handle) {
-      char* ptr = static_cast<char*>(cache_->Value(handle));
-      *size = cache_->GetCharge(handle);
+      char* ptr = cache_.Value(handle);
+      *size = cache_.get()->GetCharge(handle);
       data->reset(new char[*size]);
       std::copy_n(ptr, *size, data->get());
-      cache_->Release(handle);
+      cache_.Release(handle);
       return Status::OK();
     } else {
       return Status::NotFound();
@@ -395,10 +394,10 @@ class PersistentCacheFromCache : public PersistentCache {
 
   std::string GetPrintableOptions() const override { return ""; }
 
-  uint64_t NewId() override { return cache_->NewId(); }
+  uint64_t NewId() override { return cache_.get()->NewId(); }
 
  private:
-  std::shared_ptr<Cache> cache_;
+  BasicTypedSharedCacheInterface<char[], CacheEntryRole::kMisc> cache_;
   bool read_only_;
 };
 
@@ -406,8 +405,8 @@ class ReadOnlyCacheWrapper : public CacheWrapper {
   using CacheWrapper::CacheWrapper;
 
   using Cache::Insert;
-  Status Insert(const Slice& /*key*/, void* /*value*/, size_t /*charge*/,
-                void (*)(const Slice& key, void* value) /*deleter*/,
+  Status Insert(const Slice& /*key*/, Cache::ObjectPtr /*value*/,
+                const CacheItemHelper* /*helper*/, size_t /*charge*/,
                 Handle** /*handle*/, Priority /*priority*/) override {
     return Status::NotSupported();
   }
@@ -827,16 +826,15 @@ class MockCache : public LRUCache {
 
   using ShardedCache::Insert;
 
-  Status Insert(const Slice& key, void* value,
-                const Cache::CacheItemHelper* helper_cb, size_t charge,
+  Status Insert(const Slice& key, Cache::ObjectPtr value,
+                const Cache::CacheItemHelper* helper, size_t charge,
                 Handle** handle, Priority priority) override {
-    DeleterFn delete_cb = helper_cb->del_cb;
     if (priority == Priority::LOW) {
       low_pri_insert_count++;
     } else {
       high_pri_insert_count++;
     }
-    return LRUCache::Insert(key, value, charge, delete_cb, handle, priority);
+    return LRUCache::Insert(key, value, helper, charge, handle, priority);
   }
 };
 
@@ -916,7 +914,10 @@ class LookupLiarCache : public CacheWrapper {
       : CacheWrapper(std::move(target)) {}
 
   using Cache::Lookup;
-  Handle* Lookup(const Slice& key, Statistics* stats) override {
+  Handle* Lookup(const Slice& key, const CacheItemHelper* helper = nullptr,
+                 CreateContext* create_context = nullptr,
+                 Priority priority = Priority::LOW, bool wait = true,
+                 Statistics* stats = nullptr) override {
     if (nth_lookup_not_found_ == 1) {
       nth_lookup_not_found_ = 0;
       return nullptr;
@@ -924,7 +925,8 @@ class LookupLiarCache : public CacheWrapper {
     if (nth_lookup_not_found_ > 1) {
       --nth_lookup_not_found_;
     }
-    return CacheWrapper::Lookup(key, stats);
+    return CacheWrapper::Lookup(key, helper, create_context, priority, wait,
+                                stats);
   }
 
   // 1 == next lookup, 2 == after next, etc.
@@ -1275,12 +1277,11 @@ TEST_F(DBBlockCacheTest, CacheCompressionDict) {
 }
 
 static void ClearCache(Cache* cache) {
-  auto roles = CopyCacheDeleterRoleMap();
   std::deque<std::string> keys;
   Cache::ApplyToAllEntriesOptions opts;
-  auto callback = [&](const Slice& key, void* /*value*/, size_t /*charge*/,
-                      Cache::DeleterFn deleter) {
-    if (roles.find(deleter) == roles.end()) {
+  auto callback = [&](const Slice& key, Cache::ObjectPtr, size_t /*charge*/,
+                      const Cache::CacheItemHelper* helper) {
+    if (helper && helper->role == CacheEntryRole::kMisc) {
       // Keep the stats collector
       return;
     }
@@ -1450,14 +1451,13 @@ TEST_F(DBBlockCacheTest, CacheEntryRoleStats) {
       ClearCache(cache.get());
       Cache::Handle* h = nullptr;
       if (strcmp(cache->Name(), "LRUCache") == 0) {
-        ASSERT_OK(cache->Insert("Fill-it-up", nullptr, capacity + 1,
-                                GetNoopDeleterForRole<CacheEntryRole::kMisc>(),
-                                &h, Cache::Priority::HIGH));
+        ASSERT_OK(cache->Insert("Fill-it-up", nullptr, &kNoopCacheItemHelper,
+                                capacity + 1, &h, Cache::Priority::HIGH));
       } else {
         // For ClockCache we use a 16-byte key.
-        ASSERT_OK(cache->Insert("Fill-it-up-xxxxx", nullptr, capacity + 1,
-                                GetNoopDeleterForRole<CacheEntryRole::kMisc>(),
-                                &h, Cache::Priority::HIGH));
+        ASSERT_OK(cache->Insert("Fill-it-up-xxxxx", nullptr,
+                                &kNoopCacheItemHelper, capacity + 1, &h,
+                                Cache::Priority::HIGH));
       }
       ASSERT_GT(cache->GetUsage(), cache->GetCapacity());
       expected = {};
@@ -1548,7 +1548,7 @@ void DummyFillCache(Cache& cache, size_t entry_size,
     size_t charge = std::min(entry_size, capacity - my_usage);
     Cache::Handle* handle;
     Status st = cache.Insert(ck.WithOffset(my_usage).AsSlice(), fake_value,
-                             charge, /*deleter*/ nullptr, &handle);
+                             &kNoopCacheItemHelper, charge, &handle);
     ASSERT_OK(st);
     handles.emplace_back(&cache, handle);
     my_usage += charge;

db/db_bloom_filter_test.cc

@@ -1229,7 +1229,7 @@ TEST_P(ChargeFilterConstructionTestWithParam, Basic) {
      *
      *  The test is designed in a way such that the reservation for (p1 - b')
      *  will trigger at least another dummy entry insertion
-     *  (or equivelantly to saying, creating another peak).
+     *  (or equivalently to saying, creating another peak).
      *
      * kStandard128Ribbon + FullFilter +
      * detect_filter_construct_corruption
@@ -2618,8 +2618,7 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {
       BottommostLevelCompaction::kSkip;
   compact_options.change_level = true;
   compact_options.target_level = 7;
-  ASSERT_TRUE(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr)
-                  .IsNotSupported());
+  ASSERT_OK(db_->CompactRange(compact_options, handles_[1], nullptr, nullptr));
 
   ASSERT_EQ(trivial_move, 1);
   ASSERT_EQ(non_trivial_move, 0);

db/db_flush_test.cc

@@ -746,6 +746,64 @@ class TestFlushListener : public EventListener {
 };
 #endif  // !ROCKSDB_LITE
 
+// RocksDB lite does not support GetLiveFiles()
+#ifndef ROCKSDB_LITE
+TEST_F(DBFlushTest, FixFlushReasonRaceFromConcurrentFlushes) {
+  Options options = CurrentOptions();
+  options.atomic_flush = true;
+  options.disable_auto_compactions = true;
+  CreateAndReopenWithCF({"cf1"}, options);
+
+  for (int idx = 0; idx < 1; ++idx) {
+    ASSERT_OK(Put(0, Key(idx), std::string(1, 'v')));
+    ASSERT_OK(Put(1, Key(idx), std::string(1, 'v')));
+  }
+
+  // To coerce a manual flush happenning in the middle of GetLiveFiles's flush,
+  // we need to pause background flush thread and enable it later.
+  std::shared_ptr<test::SleepingBackgroundTask> sleeping_task =
+      std::make_shared<test::SleepingBackgroundTask>();
+  env_->SetBackgroundThreads(1, Env::HIGH);
+  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                 sleeping_task.get(), Env::Priority::HIGH);
+  sleeping_task->WaitUntilSleeping();
+
+  // Coerce a manual flush happenning in the middle of GetLiveFiles's flush
+  bool get_live_files_paused_at_sync_point = false;
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::AtomicFlushMemTables:AfterScheduleFlush", [&](void* /* arg */) {
+        if (get_live_files_paused_at_sync_point) {
+          // To prevent non-GetLiveFiles() flush from pausing at this sync point
+          return;
+        }
+        get_live_files_paused_at_sync_point = true;
+
+        FlushOptions fo;
+        fo.wait = false;
+        fo.allow_write_stall = true;
+        ASSERT_OK(dbfull()->Flush(fo));
+
+        // Resume background flush thread so GetLiveFiles() can finish
+        sleeping_task->WakeUp();
+        sleeping_task->WaitUntilDone();
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  std::vector<std::string> files;
+  uint64_t manifest_file_size;
+  // Before the fix, a race condition on default cf's flush reason due to
+  // concurrent GetLiveFiles's flush and manual flush will fail
+  // an internal assertion.
+  // After the fix, such race condition is fixed and there is no assertion
+  // failure.
+  ASSERT_OK(db_->GetLiveFiles(files, &manifest_file_size, /*flush*/ true));
+  ASSERT_TRUE(get_live_files_paused_at_sync_point);
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+}
+#endif  // !ROCKSDB_LITE
+
 TEST_F(DBFlushTest, MemPurgeBasic) {
   Options options = CurrentOptions();
 
@@ -2440,7 +2498,9 @@ TEST_P(DBAtomicFlushTest, ManualFlushUnder2PC) {
   options.atomic_flush = GetParam();
   // 64MB so that memtable flush won't be trigger by the small writes.
   options.write_buffer_size = (static_cast<size_t>(64) << 20);
-
+  auto flush_listener = std::make_shared<FlushCounterListener>();
+  flush_listener->expected_flush_reason = FlushReason::kManualFlush;
+  options.listeners.push_back(flush_listener);
   // Destroy the DB to recreate as a TransactionDB.
   Close();
   Destroy(options, true);
@@ -2507,7 +2567,6 @@ TEST_P(DBAtomicFlushTest, ManualFlushUnder2PC) {
     auto cfh = static_cast<ColumnFamilyHandleImpl*>(handles_[i]);
     ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed());
     ASSERT_TRUE(cfh->cfd()->mem()->IsEmpty());
-    ASSERT_EQ(cfh->cfd()->GetFlushReason(), FlushReason::kManualFlush);
   }
 
   // The recovered min log number with prepared data should be non-zero.
@@ -2520,13 +2579,15 @@ TEST_P(DBAtomicFlushTest, ManualFlushUnder2PC) {
   ASSERT_TRUE(db_impl->allow_2pc());
   ASSERT_NE(db_impl->MinLogNumberToKeep(), 0);
 }
-#endif  // ROCKSDB_LITE
 
 TEST_P(DBAtomicFlushTest, ManualAtomicFlush) {
   Options options = CurrentOptions();
   options.create_if_missing = true;
   options.atomic_flush = GetParam();
   options.write_buffer_size = (static_cast<size_t>(64) << 20);
+  auto flush_listener = std::make_shared<FlushCounterListener>();
+  flush_listener->expected_flush_reason = FlushReason::kManualFlush;
+  options.listeners.push_back(flush_listener);
 
   CreateAndReopenWithCF({"pikachu", "eevee"}, options);
   size_t num_cfs = handles_.size();
@@ -2551,11 +2612,11 @@ TEST_P(DBAtomicFlushTest, ManualAtomicFlush) {
 
   for (size_t i = 0; i != num_cfs; ++i) {
     auto cfh = static_cast<ColumnFamilyHandleImpl*>(handles_[i]);
-    ASSERT_EQ(cfh->cfd()->GetFlushReason(), FlushReason::kManualFlush);
     ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed());
     ASSERT_TRUE(cfh->cfd()->mem()->IsEmpty());
   }
 }
+#endif  // ROCKSDB_LITE
 
 TEST_P(DBAtomicFlushTest, PrecomputeMinLogNumberToKeepNon2PC) {
   Options options = CurrentOptions();

db/db_impl/db_impl.cc

@@ -604,7 +604,7 @@ Status DBImpl::CloseHelper() {
 
   while (!flush_queue_.empty()) {
     const FlushRequest& flush_req = PopFirstFromFlushQueue();
-    for (const auto& iter : flush_req) {
+    for (const auto& iter : flush_req.cfd_to_max_mem_id_to_persist) {
       iter.first->UnrefAndTryDelete();
     }
   }
@@ -1570,21 +1570,31 @@ Status DBImpl::ApplyWALToManifest(VersionEdit* synced_wals) {
 }
 
 Status DBImpl::LockWAL() {
-  log_write_mutex_.Lock();
-  auto cur_log_writer = logs_.back().writer;
-  IOStatus status = cur_log_writer->WriteBuffer();
-  if (!status.ok()) {
-    ROCKS_LOG_ERROR(immutable_db_options_.info_log, "WAL flush error %s",
-                    status.ToString().c_str());
-    // In case there is a fs error we should set it globally to prevent the
-    // future writes
-    WriteStatusCheck(status);
+  {
+    InstrumentedMutexLock lock(&mutex_);
+    WriteThread::Writer w;
+    write_thread_.EnterUnbatched(&w, &mutex_);
+    WriteThread::Writer nonmem_w;
+    if (two_write_queues_) {
+      nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
+    }
+
+    lock_wal_write_token_ = write_controller_.GetStopToken();
+
+    if (two_write_queues_) {
+      nonmem_write_thread_.ExitUnbatched(&nonmem_w);
+    }
+    write_thread_.ExitUnbatched(&w);
   }
-  return static_cast<Status>(status);
+  return FlushWAL(/*sync=*/false);
 }
 
 Status DBImpl::UnlockWAL() {
-  log_write_mutex_.Unlock();
+  {
+    InstrumentedMutexLock lock(&mutex_);
+    lock_wal_write_token_.reset();
+  }
+  bg_cv_.SignalAll();
   return Status::OK();
 }
 
@@ -5189,8 +5199,9 @@ Status DBImpl::IngestExternalFiles(
   for (const auto& arg : args) {
     auto* cfd = static_cast<ColumnFamilyHandleImpl*>(arg.column_family)->cfd();
     ingestion_jobs.emplace_back(versions_.get(), cfd, immutable_db_options_,
-                                file_options_, &snapshots_, arg.options,
-                                &directories_, &event_logger_, io_tracer_);
+                                mutable_db_options_, file_options_, &snapshots_,
+                                arg.options, &directories_, &event_logger_,
+                                io_tracer_);
   }
 
   // TODO(yanqin) maybe make jobs run in parallel
@@ -5318,10 +5329,12 @@ Status DBImpl::IngestExternalFiles(
     // Run ingestion jobs.
     if (status.ok()) {
       for (size_t i = 0; i != num_cfs; ++i) {
+        mutex_.AssertHeld();
         status = ingestion_jobs[i].Run();
         if (!status.ok()) {
           break;
         }
+        ingestion_jobs[i].RegisterRange();
       }
     }
     if (status.ok()) {
@@ -5377,6 +5390,10 @@ Status DBImpl::IngestExternalFiles(
       }
     }
 
+    for (auto& job : ingestion_jobs) {
+      job.UnregisterRange();
+    }
+
     if (status.ok()) {
       for (size_t i = 0; i != num_cfs; ++i) {
         auto* cfd =
@@ -5522,6 +5539,7 @@ Status DBImpl::CreateColumnFamilyWithImport(
 
       num_running_ingest_file_++;
       assert(!cfd->IsDropped());
+      mutex_.AssertHeld();
       status = import_job.Run();
 
       // Install job edit [Mutex will be unlocked here]
@@ -5747,13 +5765,6 @@ void DBImpl::NotifyOnExternalFileIngested(
   }
 }
 
-void DBImpl::WaitForIngestFile() {
-  mutex_.AssertHeld();
-  while (num_running_ingest_file_ > 0) {
-    bg_cv_.Wait();
-  }
-}
-
 Status DBImpl::StartTrace(const TraceOptions& trace_options,
                           std::unique_ptr<TraceWriter>&& trace_writer) {
   InstrumentedMutexLock lock(&trace_mutex_);

db/db_impl/db_impl.h

@@ -16,6 +16,7 @@
 #include <map>
 #include <set>
 #include <string>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -1161,7 +1162,7 @@ class DBImpl : public DB {
   int TEST_BGCompactionsAllowed() const;
   int TEST_BGFlushesAllowed() const;
   size_t TEST_GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;
-  void TEST_WaitForPeridicTaskRun(std::function<void()> callback) const;
+  void TEST_WaitForPeriodicTaskRun(std::function<void()> callback) const;
   SeqnoToTimeMapping TEST_GetSeqnoToTimeMapping() const;
   size_t TEST_EstimateInMemoryStatsHistorySize() const;
 
@@ -1383,7 +1384,7 @@ class DBImpl : public DB {
 
   void NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
                           const MutableCFOptions& mutable_cf_options,
-                          int job_id);
+                          int job_id, FlushReason flush_reason);
 
   void NotifyOnFlushCompleted(
       ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
@@ -1675,12 +1676,17 @@ class DBImpl : public DB {
   // Argument required by background flush thread.
   struct BGFlushArg {
     BGFlushArg()
-        : cfd_(nullptr), max_memtable_id_(0), superversion_context_(nullptr) {}
+        : cfd_(nullptr),
+          max_memtable_id_(0),
+          superversion_context_(nullptr),
+          flush_reason_(FlushReason::kOthers) {}
     BGFlushArg(ColumnFamilyData* cfd, uint64_t max_memtable_id,
-               SuperVersionContext* superversion_context)
+               SuperVersionContext* superversion_context,
+               FlushReason flush_reason)
         : cfd_(cfd),
           max_memtable_id_(max_memtable_id),
-          superversion_context_(superversion_context) {}
+          superversion_context_(superversion_context),
+          flush_reason_(flush_reason) {}
 
     // Column family to flush.
     ColumnFamilyData* cfd_;
@@ -1691,6 +1697,7 @@ class DBImpl : public DB {
     // installs a new superversion for the column family. This operation
     // requires a SuperVersionContext object (currently embedded in JobContext).
     SuperVersionContext* superversion_context_;
+    FlushReason flush_reason_;
   };
 
   // Argument passed to flush thread.
@@ -1819,7 +1826,7 @@ class DBImpl : public DB {
   // installs a new super version for the column family.
   Status FlushMemTableToOutputFile(
       ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
-      bool* madeProgress, JobContext* job_context,
+      bool* madeProgress, JobContext* job_context, FlushReason flush_reason,
       SuperVersionContext* superversion_context,
       std::vector<SequenceNumber>& snapshot_seqs,
       SequenceNumber earliest_write_conflict_snapshot,
@@ -1865,7 +1872,8 @@ class DBImpl : public DB {
 
   // num_bytes: for slowdown case, delay time is calculated based on
   //            `num_bytes` going through.
-  Status DelayWrite(uint64_t num_bytes, const WriteOptions& write_options);
+  Status DelayWrite(uint64_t num_bytes, WriteThread& write_thread,
+                    const WriteOptions& write_options);
 
   // Begin stalling of writes when memory usage increases beyond a certain
   // threshold.
@@ -2023,32 +2031,28 @@ class DBImpl : public DB {
                           const int output_level, int output_path_id,
                           JobContext* job_context, LogBuffer* log_buffer,
                           CompactionJobInfo* compaction_job_info);
-
-  // Wait for current IngestExternalFile() calls to finish.
-  // REQUIRES: mutex_ held
-  void WaitForIngestFile();
-#else
-  // IngestExternalFile is not supported in ROCKSDB_LITE so this function
-  // will be no-op
-  void WaitForIngestFile() {}
 #endif  // ROCKSDB_LITE
 
   ColumnFamilyData* GetColumnFamilyDataByName(const std::string& cf_name);
 
   void MaybeScheduleFlushOrCompaction();
 
-  // A flush request specifies the column families to flush as well as the
-  // largest memtable id to persist for each column family. Once all the
-  // memtables whose IDs are smaller than or equal to this per-column-family
-  // specified value, this flush request is considered to have completed its
-  // work of flushing this column family. After completing the work for all
-  // column families in this request, this flush is considered complete.
-  using FlushRequest = std::vector<std::pair<ColumnFamilyData*, uint64_t>>;
+  struct FlushRequest {
+    FlushReason flush_reason;
+    // A map from column family to flush to largest memtable id to persist for
+    // each column family. Once all the memtables whose IDs are smaller than or
+    // equal to this per-column-family specified value, this flush request is
+    // considered to have completed its work of flushing this column family.
+    // After completing the work for all column families in this request, this
+    // flush is considered complete.
+    std::unordered_map<ColumnFamilyData*, uint64_t>
+        cfd_to_max_mem_id_to_persist;
+  };
 
   void GenerateFlushRequest(const autovector<ColumnFamilyData*>& cfds,
-                            FlushRequest* req);
+                            FlushReason flush_reason, FlushRequest* req);
 
-  void SchedulePendingFlush(const FlushRequest& req, FlushReason flush_reason);
+  void SchedulePendingFlush(const FlushRequest& req);
 
   void SchedulePendingCompaction(ColumnFamilyData* cfd);
   void SchedulePendingPurge(std::string fname, std::string dir_to_sync,
@@ -2680,6 +2684,10 @@ class DBImpl : public DB {
   // seqno_time_mapping_ stores the sequence number to time mapping, it's not
   // thread safe, both read and write need db mutex hold.
   SeqnoToTimeMapping seqno_time_mapping_;
+
+  // stop write token that is acquired when LockWal() is called. Destructed
+  // when UnlockWal() is called.
+  std::unique_ptr<WriteControllerToken> lock_wal_write_token_;
 };
 
 class GetWithTimestampReadCallback : public ReadCallback {

db/db_impl/db_impl_compaction_flush.cc

@@ -155,7 +155,7 @@ IOStatus DBImpl::SyncClosedLogs(JobContext* job_context,
 
 Status DBImpl::FlushMemTableToOutputFile(
     ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
-    bool* made_progress, JobContext* job_context,
+    bool* made_progress, JobContext* job_context, FlushReason flush_reason,
     SuperVersionContext* superversion_context,
     std::vector<SequenceNumber>& snapshot_seqs,
     SequenceNumber earliest_write_conflict_snapshot,
@@ -215,7 +215,8 @@ Status DBImpl::FlushMemTableToOutputFile(
       dbname_, cfd, immutable_db_options_, mutable_cf_options, max_memtable_id,
       file_options_for_compaction_, versions_.get(), &mutex_, &shutting_down_,
       snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker,
-      job_context, log_buffer, directories_.GetDbDir(), GetDataDir(cfd, 0U),
+      job_context, flush_reason, log_buffer, directories_.GetDbDir(),
+      GetDataDir(cfd, 0U),
       GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
       &event_logger_, mutable_cf_options.report_bg_io_stats,
       true /* sync_output_directory */, true /* write_manifest */, thread_pri,
@@ -260,7 +261,8 @@ Status DBImpl::FlushMemTableToOutputFile(
 
 #ifndef ROCKSDB_LITE
   // may temporarily unlock and lock the mutex.
-  NotifyOnFlushBegin(cfd, &file_meta, mutable_cf_options, job_context->job_id);
+  NotifyOnFlushBegin(cfd, &file_meta, mutable_cf_options, job_context->job_id,
+                     flush_reason);
 #endif  // ROCKSDB_LITE
 
   bool switched_to_mempurge = false;
@@ -390,8 +392,9 @@ Status DBImpl::FlushMemTablesToOutputFiles(
   MutableCFOptions mutable_cf_options_copy = *cfd->GetLatestMutableCFOptions();
   SuperVersionContext* superversion_context =
       bg_flush_arg.superversion_context_;
+  FlushReason flush_reason = bg_flush_arg.flush_reason_;
   Status s = FlushMemTableToOutputFile(
-      cfd, mutable_cf_options_copy, made_progress, job_context,
+      cfd, mutable_cf_options_copy, made_progress, job_context, flush_reason,
       superversion_context, snapshot_seqs, earliest_write_conflict_snapshot,
       snapshot_checker, log_buffer, thread_pri);
   return s;
@@ -420,7 +423,9 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
   for (const auto cfd : cfds) {
     assert(cfd->imm()->NumNotFlushed() != 0);
     assert(cfd->imm()->IsFlushPending());
-    assert(cfd->GetFlushReason() == cfds[0]->GetFlushReason());
+  }
+  for (const auto bg_flush_arg : bg_flush_args) {
+    assert(bg_flush_arg.flush_reason_ == bg_flush_args[0].flush_reason_);
   }
 #endif /* !NDEBUG */
 
@@ -459,13 +464,15 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
     all_mutable_cf_options.emplace_back(*cfd->GetLatestMutableCFOptions());
     const MutableCFOptions& mutable_cf_options = all_mutable_cf_options.back();
     uint64_t max_memtable_id = bg_flush_args[i].max_memtable_id_;
+    FlushReason flush_reason = bg_flush_args[i].flush_reason_;
     jobs.emplace_back(new FlushJob(
         dbname_, cfd, immutable_db_options_, mutable_cf_options,
         max_memtable_id, file_options_for_compaction_, versions_.get(), &mutex_,
         &shutting_down_, snapshot_seqs, earliest_write_conflict_snapshot,
-        snapshot_checker, job_context, log_buffer, directories_.GetDbDir(),
-        data_dir, GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
-        stats_, &event_logger_, mutable_cf_options.report_bg_io_stats,
+        snapshot_checker, job_context, flush_reason, log_buffer,
+        directories_.GetDbDir(), data_dir,
+        GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
+        &event_logger_, mutable_cf_options.report_bg_io_stats,
         false /* sync_output_directory */, false /* write_manifest */,
         thread_pri, io_tracer_, seqno_time_mapping_, db_id_, db_session_id_,
         cfd->GetFullHistoryTsLow(), &blob_callback_));
@@ -483,8 +490,9 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
   for (int i = 0; i != num_cfs; ++i) {
     const MutableCFOptions& mutable_cf_options = all_mutable_cf_options.at(i);
     // may temporarily unlock and lock the mutex.
+    FlushReason flush_reason = bg_flush_args[i].flush_reason_;
     NotifyOnFlushBegin(cfds[i], &file_meta[i], mutable_cf_options,
-                       job_context->job_id);
+                       job_context->job_id, flush_reason);
   }
 #endif /* !ROCKSDB_LITE */
 
@@ -642,8 +650,9 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
 
     bool resuming_from_bg_err =
         error_handler_.IsDBStopped() ||
-        (cfds[0]->GetFlushReason() == FlushReason::kErrorRecovery ||
-         cfds[0]->GetFlushReason() == FlushReason::kErrorRecoveryRetryFlush);
+        (bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery ||
+         bg_flush_args[0].flush_reason_ ==
+             FlushReason::kErrorRecoveryRetryFlush);
     while ((!resuming_from_bg_err || error_handler_.GetRecoveryError().ok())) {
       std::pair<Status, bool> res = wait_to_install_func();
 
@@ -660,8 +669,9 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
 
       resuming_from_bg_err =
           error_handler_.IsDBStopped() ||
-          (cfds[0]->GetFlushReason() == FlushReason::kErrorRecovery ||
-           cfds[0]->GetFlushReason() == FlushReason::kErrorRecoveryRetryFlush);
+          (bg_flush_args[0].flush_reason_ == FlushReason::kErrorRecovery ||
+           bg_flush_args[0].flush_reason_ ==
+               FlushReason::kErrorRecoveryRetryFlush);
     }
 
     if (!resuming_from_bg_err) {
@@ -816,7 +826,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
 
 void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
                                 const MutableCFOptions& mutable_cf_options,
-                                int job_id) {
+                                int job_id, FlushReason flush_reason) {
 #ifndef ROCKSDB_LITE
   if (immutable_db_options_.listeners.size() == 0U) {
     return;
@@ -849,7 +859,7 @@ void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
     info.triggered_writes_stop = triggered_writes_stop;
     info.smallest_seqno = file_meta->fd.smallest_seqno;
     info.largest_seqno = file_meta->fd.largest_seqno;
-    info.flush_reason = cfd->GetFlushReason();
+    info.flush_reason = flush_reason;
     for (auto listener : immutable_db_options_.listeners) {
       listener->OnFlushBegin(this, info);
     }
@@ -862,6 +872,7 @@ void DBImpl::NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
   (void)file_meta;
   (void)mutable_cf_options;
   (void)job_id;
+  (void)flush_reason;
 #endif  // ROCKSDB_LITE
 }
 
@@ -1087,6 +1098,22 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
     {
       SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
       Version* current_version = super_version->current;
+
+      // Might need to query the partitioner
+      SstPartitionerFactory* partitioner_factory =
+          current_version->cfd()->ioptions()->sst_partitioner_factory.get();
+      std::unique_ptr<SstPartitioner> partitioner;
+      if (partitioner_factory && begin != nullptr && end != nullptr) {
+        SstPartitioner::Context context;
+        context.is_full_compaction = false;
+        context.is_manual_compaction = true;
+        context.output_level = /*unknown*/ -1;
+        // Small lies about compaction range
+        context.smallest_user_key = *begin;
+        context.largest_user_key = *end;
+        partitioner = partitioner_factory->CreatePartitioner(context);
+      }
+
       ReadOptions ro;
       ro.total_order_seek = true;
       bool overlap;
@@ -1094,14 +1121,50 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
            level < current_version->storage_info()->num_non_empty_levels();
            level++) {
         overlap = true;
+
+        // Whether to look at specific keys within files for overlap with
+        // compaction range, other than largest and smallest keys of the file
+        // known in Version metadata.
+        bool check_overlap_within_file = false;
         if (begin != nullptr && end != nullptr) {
+          // Typically checking overlap within files in this case
+          check_overlap_within_file = true;
+          // WART: Not known why we don't check within file in one-sided bound
+          // cases
+          if (partitioner) {
+            // Especially if the partitioner is new, the manual compaction
+            // might be used to enforce the partitioning. Checking overlap
+            // within files might miss cases where compaction is needed to
+            // partition the files, as in this example:
+            // * File has two keys "001" and "111"
+            // * Compaction range is ["011", "101")
+            // * Partition boundary at "100"
+            // In cases like this, file-level overlap with the compaction
+            // range is sufficient to force any partitioning that is needed
+            // within the compaction range.
+            //
+            // But if there's no partitioning boundary within the compaction
+            // range, we can be sure there's no need to fix partitioning
+            // within that range, thus safe to check overlap within file.
+            //
+            // Use a hypothetical trivial move query to check for partition
+            // boundary in range. (NOTE: in defiance of all conventions,
+            // `begin` and `end` here are both INCLUSIVE bounds, which makes
+            // this analogy to CanDoTrivialMove() accurate even when `end` is
+            // the first key in a partition.)
+            if (!partitioner->CanDoTrivialMove(*begin, *end)) {
+              check_overlap_within_file = false;
+            }
+          }
+        }
+        if (check_overlap_within_file) {
           Status status = current_version->OverlapWithLevelIterator(
               ro, file_options_, *begin, *end, level, &overlap);
           if (!status.ok()) {
-            overlap = current_version->storage_info()->OverlapInLevel(
-                level, begin, end);
+            check_overlap_within_file = false;
           }
-        } else {
+        }
+        if (!check_overlap_within_file) {
           overlap = current_version->storage_info()->OverlapInLevel(level,
                                                                     begin, end);
         }
@@ -1197,6 +1260,12 @@ Status DBImpl::CompactRangeInternal(const CompactRangeOptions& options,
 
     ROCKS_LOG_INFO(immutable_db_options_.info_log,
                    "[RefitLevel] waiting for background threads to stop");
+    // TODO(hx235): remove `Enable/DisableManualCompaction` and
+    // `Continue/PauseBackgroundWork` once we ensure registering RefitLevel()'s
+    // range is sufficient (if not, what else is needed) for avoiding range
+    // conflicts with other activities (e.g, compaction, flush) that are
+    // currently avoided by `Enable/DisableManualCompaction` and
+    // `Continue/PauseBackgroundWork`.
     DisableManualCompaction();
     s = PauseBackgroundWork();
     if (s.ok()) {
@@ -1261,13 +1330,6 @@ Status DBImpl::CompactFiles(const CompactionOptions& compact_options,
           const_cast<std::atomic<int>*>(&manual_compaction_paused_)));
   {
     InstrumentedMutexLock l(&mutex_);
-
-    // This call will unlock/lock the mutex to wait for current running
-    // IngestExternalFile() calls to finish.
-    WaitForIngestFile();
-
-    // We need to get current after `WaitForIngestFile`, because
-    // `IngestExternalFile` may add files that overlap with `input_file_names`
     auto* current = cfd->current();
     current->Ref();
 
@@ -1346,6 +1408,7 @@ Status DBImpl::CompactFilesImpl(
 
   Status s = cfd->compaction_picker()->SanitizeCompactionInputFiles(
       &input_set, cf_meta, output_level);
+  TEST_SYNC_POINT("DBImpl::CompactFilesImpl::PostSanitizeCompactionInputFiles");
   if (!s.ok()) {
     return s;
   }
@@ -1639,6 +1702,10 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
 
   InstrumentedMutexLock guard_lock(&mutex_);
 
+  auto* vstorage = cfd->current()->storage_info();
+  if (vstorage->LevelFiles(level).empty()) {
+    return Status::OK();
+  }
   // only allow one thread refitting
   if (refitting_level_) {
     ROCKS_LOG_INFO(immutable_db_options_.info_log,
@@ -1654,8 +1721,16 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
     to_level = FindMinimumEmptyLevelFitting(cfd, mutable_cf_options, level);
   }
 
-  auto* vstorage = cfd->current()->storage_info();
   if (to_level != level) {
+    std::vector<CompactionInputFiles> input(1);
+    input[0].level = level;
+    for (auto& f : vstorage->LevelFiles(level)) {
+      input[0].files.push_back(f);
+    }
+    InternalKey refit_level_smallest;
+    InternalKey refit_level_largest;
+    cfd->compaction_picker()->GetRange(input[0], &refit_level_smallest,
+                                       &refit_level_largest);
     if (to_level > level) {
       if (level == 0) {
         refitting_level_ = false;
@@ -1669,6 +1744,14 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
           return Status::NotSupported(
               "Levels between source and target are not empty for a move.");
         }
+        if (cfd->RangeOverlapWithCompaction(refit_level_smallest.user_key(),
+                                            refit_level_largest.user_key(),
+                                            l)) {
+          refitting_level_ = false;
+          return Status::NotSupported(
+              "Levels between source and target "
+              "will have some ongoing compaction's output.");
+        }
       }
     } else {
       // to_level < level
@@ -1679,22 +1762,51 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
           return Status::NotSupported(
               "Levels between source and target are not empty for a move.");
         }
+        if (cfd->RangeOverlapWithCompaction(refit_level_smallest.user_key(),
+                                            refit_level_largest.user_key(),
+                                            l)) {
+          refitting_level_ = false;
+          return Status::NotSupported(
+              "Levels between source and target "
+              "will have some ongoing compaction's output.");
+        }
       }
     }
     ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                     "[%s] Before refitting:\n%s", cfd->GetName().c_str(),
                     cfd->current()->DebugString().data());
 
+    std::unique_ptr<Compaction> c(new Compaction(
+        vstorage, *cfd->ioptions(), mutable_cf_options, mutable_db_options_,
+        {input}, to_level,
+        MaxFileSizeForLevel(
+            mutable_cf_options, to_level,
+            cfd->ioptions()
+                ->compaction_style) /* output file size limit, not applicable */
+        ,
+        LLONG_MAX /* max compaction bytes, not applicable */,
+        0 /* output path ID, not applicable */, mutable_cf_options.compression,
+        mutable_cf_options.compression_opts, Temperature::kUnknown,
+        0 /* max_subcompactions, not applicable */,
+        {} /* grandparents, not applicable */, false /* is manual */,
+        "" /* trim_ts */, -1 /* score, not applicable */,
+        false /* is deletion compaction, not applicable */,
+        false /* l0_files_might_overlap, not applicable */,
+        CompactionReason::kRefitLevel));
+    cfd->compaction_picker()->RegisterCompaction(c.get());
+    TEST_SYNC_POINT("DBImpl::ReFitLevel:PostRegisterCompaction");
     VersionEdit edit;
     edit.SetColumnFamily(cfd->GetID());
+
     for (const auto& f : vstorage->LevelFiles(level)) {
       edit.DeleteFile(level, f->fd.GetNumber());
       edit.AddFile(
           to_level, f->fd.GetNumber(), f->fd.GetPathId(), f->fd.GetFileSize(),
           f->smallest, f->largest, f->fd.smallest_seqno, f->fd.largest_seqno,
           f->marked_for_compaction, f->temperature, f->oldest_blob_file_number,
-          f->oldest_ancester_time, f->file_creation_time, f->file_checksum,
-          f->file_checksum_func_name, f->unique_id);
+          f->oldest_ancester_time, f->file_creation_time, f->epoch_number,
+          f->file_checksum, f->file_checksum_func_name, f->unique_id,
+          f->compensated_range_deletion_size);
     }
     ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                     "[%s] Apply version edit:\n%s", cfd->GetName().c_str(),
@@ -1703,6 +1815,9 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
     Status status = versions_->LogAndApply(cfd, mutable_cf_options, &edit,
                                            &mutex_, directories_.GetDbDir());
 
+    cfd->compaction_picker()->UnregisterCompaction(c.get());
+    c.reset();
+
     InstallSuperVersionAndScheduleWork(cfd, &sv_context, mutable_cf_options);
 
     ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "[%s] LogAndApply: %s\n",
@@ -1919,11 +2034,19 @@ Status DBImpl::RunManualCompaction(
                manual.begin, manual.end, &manual.manual_end, &manual_conflict,
                max_file_num_to_ignore, trim_ts)) == nullptr &&
           manual_conflict))) {
-      // exclusive manual compactions should not see a conflict during
-      // CompactRange
-      assert(!exclusive || !manual_conflict);
-      // Running either this or some other manual compaction
-      bg_cv_.Wait();
+      if (!scheduled) {
+        // There is a conflicting compaction
+        if (manual_compaction_paused_ > 0 || manual.canceled == true) {
+          // Stop waiting since it was canceled. Pretend the error came from
+          // compaction so the below cleanup/error handling code can process it.
+          manual.done = true;
+          manual.status =
+              Status::Incomplete(Status::SubCode::kManualCompactionPaused);
+        }
+      }
+      if (!manual.done) {
+        bg_cv_.Wait();
+      }
       if (manual_compaction_paused_ > 0 && scheduled && !unscheduled) {
         assert(thread_pool_priority != Env::Priority::TOTAL);
         // unschedule all manual compactions
@@ -2001,16 +2124,17 @@ Status DBImpl::RunManualCompaction(
 }
 
 void DBImpl::GenerateFlushRequest(const autovector<ColumnFamilyData*>& cfds,
-                                  FlushRequest* req) {
+                                  FlushReason flush_reason, FlushRequest* req) {
   assert(req != nullptr);
-  req->reserve(cfds.size());
+  req->flush_reason = flush_reason;
+  req->cfd_to_max_mem_id_to_persist.reserve(cfds.size());
   for (const auto cfd : cfds) {
     if (nullptr == cfd) {
       // cfd may be null, see DBImpl::ScheduleFlushes
       continue;
     }
     uint64_t max_memtable_id = cfd->imm()->GetLatestMemTableID();
-    req->emplace_back(cfd, max_memtable_id);
+    req->cfd_to_max_mem_id_to_persist.emplace(cfd, max_memtable_id);
   }
 }
 
@@ -2068,7 +2192,7 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
     if (s.ok()) {
       if (cfd->imm()->NumNotFlushed() != 0 || !cfd->mem()->IsEmpty() ||
           !cached_recoverable_state_empty_.load()) {
-        FlushRequest req{{cfd, flush_memtable_id}};
+        FlushRequest req{flush_reason, {{cfd, flush_memtable_id}}};
         flush_reqs.emplace_back(std::move(req));
         memtable_ids_to_wait.emplace_back(cfd->imm()->GetLatestMemTableID());
       }
@@ -2096,10 +2220,10 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
                            "to avoid holding old logs",
                            cfd->GetName().c_str());
             s = SwitchMemtable(cfd_stats, &context);
-            FlushRequest req{{cfd_stats, flush_memtable_id}};
+            FlushRequest req{flush_reason, {{cfd_stats, flush_memtable_id}}};
             flush_reqs.emplace_back(std::move(req));
             memtable_ids_to_wait.emplace_back(
-                cfd->imm()->GetLatestMemTableID());
+                cfd_stats->imm()->GetLatestMemTableID());
           }
         }
       }
@@ -2107,8 +2231,9 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
 
     if (s.ok() && !flush_reqs.empty()) {
       for (const auto& req : flush_reqs) {
-        assert(req.size() == 1);
-        ColumnFamilyData* loop_cfd = req[0].first;
+        assert(req.cfd_to_max_mem_id_to_persist.size() == 1);
+        ColumnFamilyData* loop_cfd =
+            req.cfd_to_max_mem_id_to_persist.begin()->first;
         loop_cfd->imm()->FlushRequested();
       }
       // If the caller wants to wait for this flush to complete, it indicates
@@ -2117,13 +2242,14 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
       // Therefore, we increase the cfd's ref count.
       if (flush_options.wait) {
         for (const auto& req : flush_reqs) {
-          assert(req.size() == 1);
-          ColumnFamilyData* loop_cfd = req[0].first;
+          assert(req.cfd_to_max_mem_id_to_persist.size() == 1);
+          ColumnFamilyData* loop_cfd =
+              req.cfd_to_max_mem_id_to_persist.begin()->first;
           loop_cfd->Ref();
         }
       }
       for (const auto& req : flush_reqs) {
-        SchedulePendingFlush(req, flush_reason);
+        SchedulePendingFlush(req);
       }
       MaybeScheduleFlushOrCompaction();
     }
@@ -2142,8 +2268,8 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
     autovector<const uint64_t*> flush_memtable_ids;
     assert(flush_reqs.size() == memtable_ids_to_wait.size());
     for (size_t i = 0; i < flush_reqs.size(); ++i) {
-      assert(flush_reqs[i].size() == 1);
-      cfds.push_back(flush_reqs[i][0].first);
+      assert(flush_reqs[i].cfd_to_max_mem_id_to_persist.size() == 1);
+      cfds.push_back(flush_reqs[i].cfd_to_max_mem_id_to_persist.begin()->first);
       flush_memtable_ids.push_back(&(memtable_ids_to_wait[i]));
     }
     s = WaitForFlushMemTables(
@@ -2240,8 +2366,8 @@ Status DBImpl::AtomicFlushMemTables(
           cfd->Ref();
         }
       }
-      GenerateFlushRequest(cfds, &flush_req);
-      SchedulePendingFlush(flush_req, flush_reason);
+      GenerateFlushRequest(cfds, flush_reason, &flush_req);
+      SchedulePendingFlush(flush_req);
       MaybeScheduleFlushOrCompaction();
     }
 
@@ -2256,7 +2382,7 @@ Status DBImpl::AtomicFlushMemTables(
   TEST_SYNC_POINT("DBImpl::AtomicFlushMemTables:BeforeWaitForBgFlush");
   if (s.ok() && flush_options.wait) {
     autovector<const uint64_t*> flush_memtable_ids;
-    for (auto& iter : flush_req) {
+    for (auto& iter : flush_req.cfd_to_max_mem_id_to_persist) {
       flush_memtable_ids.push_back(&(iter.second));
     }
     s = WaitForFlushMemTables(
@@ -2603,9 +2729,9 @@ DBImpl::FlushRequest DBImpl::PopFirstFromFlushQueue() {
   FlushRequest flush_req = flush_queue_.front();
   flush_queue_.pop_front();
   if (!immutable_db_options_.atomic_flush) {
-    assert(flush_req.size() == 1);
+    assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1);
   }
-  for (const auto& elem : flush_req) {
+  for (const auto& elem : flush_req.cfd_to_max_mem_id_to_persist) {
     if (!immutable_db_options_.atomic_flush) {
       ColumnFamilyData* cfd = elem.first;
       assert(cfd);
@@ -2613,7 +2739,6 @@ DBImpl::FlushRequest DBImpl::PopFirstFromFlushQueue() {
       cfd->set_queued_for_flush(false);
     }
   }
-  // TODO: need to unset flush reason?
   return flush_req;
 }
 
@@ -2643,31 +2768,29 @@ ColumnFamilyData* DBImpl::PickCompactionFromQueue(
   return cfd;
 }
 
-void DBImpl::SchedulePendingFlush(const FlushRequest& flush_req,
-                                  FlushReason flush_reason) {
+void DBImpl::SchedulePendingFlush(const FlushRequest& flush_req) {
   mutex_.AssertHeld();
-  if (flush_req.empty()) {
+  if (flush_req.cfd_to_max_mem_id_to_persist.empty()) {
     return;
   }
   if (!immutable_db_options_.atomic_flush) {
     // For the non-atomic flush case, we never schedule multiple column
     // families in the same flush request.
-    assert(flush_req.size() == 1);
-    ColumnFamilyData* cfd = flush_req[0].first;
+    assert(flush_req.cfd_to_max_mem_id_to_persist.size() == 1);
+    ColumnFamilyData* cfd =
+        flush_req.cfd_to_max_mem_id_to_persist.begin()->first;
     assert(cfd);
 
     if (!cfd->queued_for_flush() && cfd->imm()->IsFlushPending()) {
       cfd->Ref();
       cfd->set_queued_for_flush(true);
-      cfd->SetFlushReason(flush_reason);
       ++unscheduled_flushes_;
       flush_queue_.push_back(flush_req);
     }
   } else {
-    for (auto& iter : flush_req) {
+    for (auto& iter : flush_req.cfd_to_max_mem_id_to_persist) {
       ColumnFamilyData* cfd = iter.first;
       cfd->Ref();
-      cfd->SetFlushReason(flush_reason);
     }
     ++unscheduled_flushes_;
     flush_queue_.push_back(flush_req);
@@ -2799,10 +2922,12 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
   while (!flush_queue_.empty()) {
     // This cfd is already referenced
     const FlushRequest& flush_req = PopFirstFromFlushQueue();
+    FlushReason flush_reason = flush_req.flush_reason;
     superversion_contexts.clear();
-    superversion_contexts.reserve(flush_req.size());
+    superversion_contexts.reserve(
+        flush_req.cfd_to_max_mem_id_to_persist.size());
 
-    for (const auto& iter : flush_req) {
+    for (const auto& iter : flush_req.cfd_to_max_mem_id_to_persist) {
       ColumnFamilyData* cfd = iter.first;
       if (cfd->GetMempurgeUsed()) {
         // If imm() contains silent memtables (e.g.: because
@@ -2818,7 +2943,7 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
       }
       superversion_contexts.emplace_back(SuperVersionContext(true));
       bg_flush_args.emplace_back(cfd, iter.second,
-                                 &(superversion_contexts.back()));
+                                 &(superversion_contexts.back()), flush_reason);
     }
     if (!bg_flush_args.empty()) {
       break;
@@ -2842,9 +2967,14 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
     status = FlushMemTablesToOutputFiles(bg_flush_args, made_progress,
                                          job_context, log_buffer, thread_pri);
     TEST_SYNC_POINT("DBImpl::BackgroundFlush:BeforeFlush");
-    // All the CFDs in the FlushReq must have the same flush reason, so just
-    // grab the first one
-    *reason = bg_flush_args[0].cfd_->GetFlushReason();
+// All the CFD/bg_flush_arg in the FlushReq must have the same flush reason, so
+// just grab the first one
+#ifndef NDEBUG
+    for (const auto bg_flush_arg : bg_flush_args) {
+      assert(bg_flush_arg.flush_reason_ == bg_flush_args[0].flush_reason_);
+    }
+#endif /* !NDEBUG */
+    *reason = bg_flush_args[0].flush_reason_;
     for (auto& arg : bg_flush_args) {
       ColumnFamilyData* cfd = arg.cfd_;
       if (cfd->UnrefAndTryDelete()) {
@@ -2950,10 +3080,6 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
   {
     InstrumentedMutexLock l(&mutex_);
 
-    // This call will unlock/lock the mutex to wait for current running
-    // IngestExternalFile() calls to finish.
-    WaitForIngestFile();
-
     num_running_compactions_++;
 
     std::unique_ptr<std::list<uint64_t>::iterator>
@@ -3334,8 +3460,9 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
             f->fd.GetFileSize(), f->smallest, f->largest, f->fd.smallest_seqno,
             f->fd.largest_seqno, f->marked_for_compaction, f->temperature,
             f->oldest_blob_file_number, f->oldest_ancester_time,
-            f->file_creation_time, f->file_checksum, f->file_checksum_func_name,
-            f->unique_id);
+            f->file_creation_time, f->epoch_number, f->file_checksum,
+            f->file_checksum_func_name, f->unique_id,
+            f->compensated_range_deletion_size);
 
         ROCKS_LOG_BUFFER(
             log_buffer,
@@ -3594,11 +3721,6 @@ void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) {
 }
 
 bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) {
-  if (num_running_ingest_file_ > 0) {
-    // We need to wait for other IngestExternalFile() calls to finish
-    // before running a manual compaction.
-    return true;
-  }
   if (m->exclusive) {
     return (bg_bottom_compaction_scheduled_ > 0 ||
             bg_compaction_scheduled_ > 0);

db/db_impl/db_impl_debug.cc

@@ -290,7 +290,7 @@ size_t DBImpl::TEST_GetWalPreallocateBlockSize(
 }
 
 #ifndef ROCKSDB_LITE
-void DBImpl::TEST_WaitForPeridicTaskRun(std::function<void()> callback) const {
+void DBImpl::TEST_WaitForPeriodicTaskRun(std::function<void()> callback) const {
   periodic_task_scheduler_.TEST_WaitForRun(callback);
 }
 

db/db_impl/db_impl_experimental.cc

@@ -136,8 +136,9 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
                    f->fd.smallest_seqno, f->fd.largest_seqno,
                    f->marked_for_compaction, f->temperature,
                    f->oldest_blob_file_number, f->oldest_ancester_time,
-                   f->file_creation_time, f->file_checksum,
-                   f->file_checksum_func_name, f->unique_id);
+                   f->file_creation_time, f->epoch_number, f->file_checksum,
+                   f->file_checksum_func_name, f->unique_id,
+                   f->compensated_range_deletion_size);
     }
 
     status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),

db/db_impl/db_impl_files.cc

@@ -315,6 +315,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
     }
     log_write_mutex_.Unlock();
     mutex_.Unlock();
+    TEST_SYNC_POINT_CALLBACK("FindObsoleteFiles::PostMutexUnlock", nullptr);
     log_write_mutex_.Lock();
     while (!logs_.empty() && logs_.front().number < min_log_number) {
       auto& log = logs_.front();
@@ -360,6 +361,8 @@ void DBImpl::DeleteObsoleteFileImpl(int job_id, const std::string& fname,
   }
   TEST_SYNC_POINT_CALLBACK("DBImpl::DeleteObsoleteFileImpl:AfterDeletion",
                            &file_deletion_status);
+  TEST_SYNC_POINT_CALLBACK("DBImpl::DeleteObsoleteFileImpl:AfterDeletion2",
+                           const_cast<std::string*>(&fname));
   if (file_deletion_status.ok()) {
     ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                     "[JOB %d] Delete %s type=%d #%" PRIu64 " -- %s\n", job_id,

db/db_impl/db_impl_open.cc

@@ -1515,7 +1515,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
         .PermitUncheckedError();  // ignore error
     const uint64_t current_time = static_cast<uint64_t>(_current_time);
     meta.oldest_ancester_time = current_time;
-
+    meta.epoch_number = cfd->NewEpochNumber();
     {
       auto write_hint = cfd->CalculateSSTWriteHint(0);
       mutex_.Unlock();
@@ -1550,6 +1550,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
           0 /* file_creation_time */, db_id_, db_session_id_,
           0 /* target_file_size */, meta.fd.GetNumber());
       SeqnoToTimeMapping empty_seqno_time_mapping;
+      Version* version = cfd->current();
+      version->Ref();
       s = BuildTable(
           dbname_, versions_.get(), immutable_db_options_, tboptions,
           file_options_for_compaction_, cfd->table_cache(), iter.get(),
@@ -1559,7 +1561,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
           io_tracer_, BlobFileCreationReason::kRecovery,
           empty_seqno_time_mapping, &event_logger_, job_id, Env::IO_HIGH,
           nullptr /* table_properties */, write_hint,
-          nullptr /*full_history_ts_low*/, &blob_callback_);
+          nullptr /*full_history_ts_low*/, &blob_callback_, version);
+      version->Unref();
       LogFlush(immutable_db_options_.info_log);
       ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                       "[%s] [WriteLevel0TableForRecovery]"
@@ -1588,8 +1591,9 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
                   meta.fd.smallest_seqno, meta.fd.largest_seqno,
                   meta.marked_for_compaction, meta.temperature,
                   meta.oldest_blob_file_number, meta.oldest_ancester_time,
-                  meta.file_creation_time, meta.file_checksum,
-                  meta.file_checksum_func_name, meta.unique_id);
+                  meta.file_creation_time, meta.epoch_number,
+                  meta.file_checksum, meta.file_checksum_func_name,
+                  meta.unique_id, meta.compensated_range_deletion_size);
 
     for (const auto& blob : blob_file_additions) {
       edit->AddBlobFile(blob);

db/db_impl/db_impl_write.cc

@@ -924,6 +924,16 @@ Status DBImpl::WriteImplWALOnly(
       write_thread->ExitAsBatchGroupLeader(write_group, status);
       return status;
     }
+  } else {
+    InstrumentedMutexLock lock(&mutex_);
+    Status status =
+        DelayWrite(/*num_bytes=*/0ull, *write_thread, write_options);
+    if (!status.ok()) {
+      WriteThread::WriteGroup write_group;
+      write_thread->EnterAsBatchGroupLeader(&w, &write_group);
+      write_thread->ExitAsBatchGroupLeader(write_group, status);
+      return status;
+    }
   }
 
   WriteThread::WriteGroup write_group;
@@ -1192,7 +1202,7 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
     // might happen for smaller writes but larger writes can go through.
     // Can optimize it if it is an issue.
     InstrumentedMutexLock l(&mutex_);
-    status = DelayWrite(last_batch_group_size_, write_options);
+    status = DelayWrite(last_batch_group_size_, write_thread_, write_options);
     PERF_TIMER_START(write_pre_and_post_process_time);
   }
 
@@ -1644,14 +1654,14 @@ Status DBImpl::SwitchWAL(WriteContext* write_context) {
       cfd->imm()->FlushRequested();
       if (!immutable_db_options_.atomic_flush) {
         FlushRequest flush_req;
-        GenerateFlushRequest({cfd}, &flush_req);
-        SchedulePendingFlush(flush_req, FlushReason::kWalFull);
+        GenerateFlushRequest({cfd}, FlushReason::kWalFull, &flush_req);
+        SchedulePendingFlush(flush_req);
       }
     }
     if (immutable_db_options_.atomic_flush) {
       FlushRequest flush_req;
-      GenerateFlushRequest(cfds, &flush_req);
-      SchedulePendingFlush(flush_req, FlushReason::kWalFull);
+      GenerateFlushRequest(cfds, FlushReason::kWalFull, &flush_req);
+      SchedulePendingFlush(flush_req);
     }
     MaybeScheduleFlushOrCompaction();
   }
@@ -1735,14 +1745,15 @@ Status DBImpl::HandleWriteBufferManagerFlush(WriteContext* write_context) {
       cfd->imm()->FlushRequested();
       if (!immutable_db_options_.atomic_flush) {
         FlushRequest flush_req;
-        GenerateFlushRequest({cfd}, &flush_req);
-        SchedulePendingFlush(flush_req, FlushReason::kWriteBufferManager);
+        GenerateFlushRequest({cfd}, FlushReason::kWriteBufferManager,
+                             &flush_req);
+        SchedulePendingFlush(flush_req);
       }
     }
     if (immutable_db_options_.atomic_flush) {
       FlushRequest flush_req;
-      GenerateFlushRequest(cfds, &flush_req);
-      SchedulePendingFlush(flush_req, FlushReason::kWriteBufferManager);
+      GenerateFlushRequest(cfds, FlushReason::kWriteBufferManager, &flush_req);
+      SchedulePendingFlush(flush_req);
     }
     MaybeScheduleFlushOrCompaction();
   }
@@ -1759,16 +1770,25 @@ uint64_t DBImpl::GetMaxTotalWalSize() const {
 }
 
 // REQUIRES: mutex_ is held
-// REQUIRES: this thread is currently at the front of the writer queue
-Status DBImpl::DelayWrite(uint64_t num_bytes,
+// REQUIRES: this thread is currently at the leader for write_thread
+Status DBImpl::DelayWrite(uint64_t num_bytes, WriteThread& write_thread,
                           const WriteOptions& write_options) {
+  mutex_.AssertHeld();
   uint64_t time_delayed = 0;
   bool delayed = false;
   {
     StopWatch sw(immutable_db_options_.clock, stats_, WRITE_STALL,
                  &time_delayed);
-    uint64_t delay =
-        write_controller_.GetDelay(immutable_db_options_.clock, num_bytes);
+    // To avoid parallel timed delays (bad throttling), only support them
+    // on the primary write queue.
+    uint64_t delay;
+    if (&write_thread == &write_thread_) {
+      delay =
+          write_controller_.GetDelay(immutable_db_options_.clock, num_bytes);
+    } else {
+      assert(num_bytes == 0);
+      delay = 0;
+    }
     TEST_SYNC_POINT("DBImpl::DelayWrite:Start");
     if (delay > 0) {
       if (write_options.no_slowdown) {
@@ -1776,9 +1796,9 @@ Status DBImpl::DelayWrite(uint64_t num_bytes,
       }
       TEST_SYNC_POINT("DBImpl::DelayWrite:Sleep");
 
-      // Notify write_thread_ about the stall so it can setup a barrier and
+      // Notify write_thread about the stall so it can setup a barrier and
       // fail any pending writers with no_slowdown
-      write_thread_.BeginWriteStall();
+      write_thread.BeginWriteStall();
       mutex_.Unlock();
       TEST_SYNC_POINT("DBImpl::DelayWrite:BeginWriteStallDone");
       // We will delay the write until we have slept for `delay` microseconds
@@ -1798,7 +1818,7 @@ Status DBImpl::DelayWrite(uint64_t num_bytes,
         immutable_db_options_.clock->SleepForMicroseconds(kDelayInterval);
       }
       mutex_.Lock();
-      write_thread_.EndWriteStall();
+      write_thread.EndWriteStall();
     }
 
     // Don't wait if there's a background error, even if its a soft error. We
@@ -1812,12 +1832,12 @@ Status DBImpl::DelayWrite(uint64_t num_bytes,
       }
       delayed = true;
 
-      // Notify write_thread_ about the stall so it can setup a barrier and
+      // Notify write_thread about the stall so it can setup a barrier and
       // fail any pending writers with no_slowdown
-      write_thread_.BeginWriteStall();
+      write_thread.BeginWriteStall();
       TEST_SYNC_POINT("DBImpl::DelayWrite:Wait");
       bg_cv_.Wait();
-      write_thread_.EndWriteStall();
+      write_thread.EndWriteStall();
     }
   }
   assert(!delayed || !write_options.no_slowdown);
@@ -1998,13 +2018,13 @@ Status DBImpl::ScheduleFlushes(WriteContext* context) {
     if (immutable_db_options_.atomic_flush) {
       AssignAtomicFlushSeq(cfds);
       FlushRequest flush_req;
-      GenerateFlushRequest(cfds, &flush_req);
-      SchedulePendingFlush(flush_req, FlushReason::kWriteBufferFull);
+      GenerateFlushRequest(cfds, FlushReason::kWriteBufferFull, &flush_req);
+      SchedulePendingFlush(flush_req);
     } else {
       for (auto* cfd : cfds) {
         FlushRequest flush_req;
-        GenerateFlushRequest({cfd}, &flush_req);
-        SchedulePendingFlush(flush_req, FlushReason::kWriteBufferFull);
+        GenerateFlushRequest({cfd}, FlushReason::kWriteBufferFull, &flush_req);
+        SchedulePendingFlush(flush_req);
       }
     }
     MaybeScheduleFlushOrCompaction();

db/db_iter.cc

@@ -1247,10 +1247,13 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
 }
 
 bool DBIter::Merge(const Slice* val, const Slice& user_key) {
+  // `op_failure_scope` (an output parameter) is not provided (set to nullptr)
+  // since a failure must be propagated regardless of its value.
   Status s = MergeHelper::TimedFullMerge(
       merge_operator_, user_key, val, merge_context_.GetOperands(),
       &saved_value_, logger_, statistics_, clock_, &pinned_value_,
-      /* update_num_ops_stats */ true);
+      /* update_num_ops_stats */ true,
+      /* op_failure_scope */ nullptr);
   if (!s.ok()) {
     valid_ = false;
     status_ = s;
@@ -1265,10 +1268,13 @@ bool DBIter::Merge(const Slice* val, const Slice& user_key) {
 }
 
 bool DBIter::MergeEntity(const Slice& entity, const Slice& user_key) {
+  // `op_failure_scope` (an output parameter) is not provided (set to nullptr)
+  // since a failure must be propagated regardless of its value.
   Status s = MergeHelper::TimedFullMergeWithEntity(
       merge_operator_, user_key, entity, merge_context_.GetOperands(),
       &saved_value_, logger_, statistics_, clock_,
-      /* update_num_ops_stats */ true);
+      /* update_num_ops_stats */ true,
+      /* op_failure_scope */ nullptr);
   if (!s.ok()) {
     valid_ = false;
     status_ = s;

db/db_merge_operand_test.cc

@@ -439,6 +439,48 @@ TEST_F(DBMergeOperandTest, GetMergeOperandsLargeResultOptimization) {
   }
 }
 
+TEST_F(DBMergeOperandTest, GetMergeOperandsBaseDeletionInImmMem) {
+  // In this test, "k1" has a MERGE in a mutable memtable on top of a base
+  // DELETE in an immutable memtable.
+  Options opts = CurrentOptions();
+  opts.max_write_buffer_number = 10;
+  opts.min_write_buffer_number_to_merge = 10;
+  opts.merge_operator = MergeOperators::CreateDeprecatedPutOperator();
+  Reopen(opts);
+
+  ASSERT_OK(Put("k1", "val"));
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put("k0", "val"));
+  ASSERT_OK(Delete("k1"));
+  ASSERT_OK(Put("k2", "val"));
+  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
+  ASSERT_OK(Merge("k1", "val"));
+
+  {
+    std::vector<PinnableSlice> values(2);
+
+    GetMergeOperandsOptions merge_operands_info;
+    merge_operands_info.expected_max_number_of_operands =
+        static_cast<int>(values.size());
+
+    std::string key = "k1", from_db;
+    int number_of_operands = 0;
+    ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                    key, values.data(), &merge_operands_info,
+                                    &number_of_operands));
+    ASSERT_EQ(1, number_of_operands);
+    from_db = values[0].ToString();
+    ASSERT_EQ("val", from_db);
+  }
+
+  {
+    std::string val;
+    ASSERT_OK(db_->Get(ReadOptions(), "k1", &val));
+    ASSERT_EQ("val", val);
+  }
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_merge_operator_test.cc

@@ -202,6 +202,161 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnIteration) {
   VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}});
 }
 
+#ifndef ROCKSDB_LITE
+
+TEST_F(DBMergeOperatorTest, MergeOperatorFailsWithMustMerge) {
+  // This is like a mini-stress test dedicated to `OpFailureScope::kMustMerge`.
+  // Some or most of it might be deleted upon adding that option to the actual
+  // stress test.
+  //
+  // "k0" and "k2" are stable (uncorrupted) keys before and after a corrupted
+  // key ("k1"). The outer loop (`i`) varies which write (`j`) to "k1" triggers
+  // the corruption. Inside that loop there are three cases:
+  //
+  // - Case 1: pure `Merge()`s
+  // - Case 2: `Merge()`s on top of a `Put()`
+  // - Case 3: `Merge()`s on top of a `Delete()`
+  //
+  // For each case we test query results before flush, after flush, and after
+  // compaction, as well as cleanup after deletion+compaction. The queries
+  // expect "k0" and "k2" to always be readable. "k1" is expected to be readable
+  // only by APIs that do not require merging, such as `GetMergeOperands()`.
+  const int kNumOperands = 3;
+  Options options;
+  options.merge_operator.reset(new TestPutOperator());
+  options.env = env_;
+  Reopen(options);
+
+  for (int i = 0; i < kNumOperands; ++i) {
+    auto check_query = [&]() {
+      {
+        std::string value;
+        ASSERT_OK(db_->Get(ReadOptions(), "k0", &value));
+        ASSERT_TRUE(db_->Get(ReadOptions(), "k1", &value).IsCorruption());
+        ASSERT_OK(db_->Get(ReadOptions(), "k2", &value));
+      }
+
+      {
+        std::unique_ptr<Iterator> iter;
+        iter.reset(db_->NewIterator(ReadOptions()));
+        iter->SeekToFirst();
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_EQ("k0", iter->key());
+        iter->Next();
+        ASSERT_TRUE(iter->status().IsCorruption());
+
+        iter->SeekToLast();
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_EQ("k2", iter->key());
+        iter->Prev();
+        ASSERT_TRUE(iter->status().IsCorruption());
+
+        iter->Seek("k2");
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_EQ("k2", iter->key());
+      }
+
+      std::vector<PinnableSlice> values(kNumOperands);
+      GetMergeOperandsOptions merge_operands_info;
+      merge_operands_info.expected_max_number_of_operands = kNumOperands;
+      int num_operands_found = 0;
+      ASSERT_OK(db_->GetMergeOperands(ReadOptions(), db_->DefaultColumnFamily(),
+                                      "k1", values.data(), &merge_operands_info,
+                                      &num_operands_found));
+      ASSERT_EQ(kNumOperands, num_operands_found);
+      for (int j = 0; j < num_operands_found; ++j) {
+        if (i == j) {
+          ASSERT_EQ(values[j], "corrupted_must_merge");
+        } else {
+          ASSERT_EQ(values[j], "ok");
+        }
+      }
+    };
+
+    ASSERT_OK(Put("k0", "val"));
+    ASSERT_OK(Put("k2", "val"));
+
+    // Case 1
+    for (int j = 0; j < kNumOperands; ++j) {
+      if (j == i) {
+        ASSERT_OK(Merge("k1", "corrupted_must_merge"));
+      } else {
+        ASSERT_OK(Merge("k1", "ok"));
+      }
+    }
+    check_query();
+    ASSERT_OK(Flush());
+    check_query();
+    {
+      CompactRangeOptions cro;
+      cro.bottommost_level_compaction =
+          BottommostLevelCompaction::kForceOptimized;
+      ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+    }
+    check_query();
+
+    // Case 2
+    for (int j = 0; j < kNumOperands; ++j) {
+      Slice val;
+      if (j == i) {
+        val = "corrupted_must_merge";
+      } else {
+        val = "ok";
+      }
+      if (j == 0) {
+        ASSERT_OK(Put("k1", val));
+      } else {
+        ASSERT_OK(Merge("k1", val));
+      }
+    }
+    check_query();
+    ASSERT_OK(Flush());
+    check_query();
+    {
+      CompactRangeOptions cro;
+      cro.bottommost_level_compaction =
+          BottommostLevelCompaction::kForceOptimized;
+      ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+    }
+    check_query();
+
+    // Case 3
+    ASSERT_OK(Delete("k1"));
+    for (int j = 0; j < kNumOperands; ++j) {
+      if (i == j) {
+        ASSERT_OK(Merge("k1", "corrupted_must_merge"));
+      } else {
+        ASSERT_OK(Merge("k1", "ok"));
+      }
+    }
+    check_query();
+    ASSERT_OK(Flush());
+    check_query();
+    {
+      CompactRangeOptions cro;
+      cro.bottommost_level_compaction =
+          BottommostLevelCompaction::kForceOptimized;
+      ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+    }
+    check_query();
+
+    // Verify obsolete data removal still happens
+    ASSERT_OK(Delete("k0"));
+    ASSERT_OK(Delete("k1"));
+    ASSERT_OK(Delete("k2"));
+    ASSERT_EQ("NOT_FOUND", Get("k0"));
+    ASSERT_EQ("NOT_FOUND", Get("k1"));
+    ASSERT_EQ("NOT_FOUND", Get("k2"));
+    CompactRangeOptions cro;
+    cro.bottommost_level_compaction =
+        BottommostLevelCompaction::kForceOptimized;
+    ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+    ASSERT_EQ("", FilesPerLevel());
+  }
+}
+
+#endif  // ROCKSDB_LITE
+
 class MergeOperatorPinningTest : public DBMergeOperatorTest,
                                  public testing::WithParamInterface<bool> {
  public:

db/db_properties_test.cc

@@ -1848,8 +1848,8 @@ TEST_F(DBPropertiesTest, BlobCacheProperties) {
 
   // Insert unpinned blob to the cache and check size.
   constexpr size_t kSize1 = 70;
-  ASSERT_OK(blob_cache->Insert("blob1", nullptr /*value*/, kSize1,
-                               nullptr /*deleter*/));
+  ASSERT_OK(blob_cache->Insert("blob1", nullptr /*value*/,
+                               &kNoopCacheItemHelper, kSize1));
   ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheCapacity, &value));
   ASSERT_EQ(kCapacity, value);
   ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheUsage, &value));
@@ -1861,8 +1861,8 @@ TEST_F(DBPropertiesTest, BlobCacheProperties) {
   // Insert pinned blob to the cache and check size.
   constexpr size_t kSize2 = 60;
   Cache::Handle* blob2 = nullptr;
-  ASSERT_OK(blob_cache->Insert("blob2", nullptr /*value*/, kSize2,
-                               nullptr /*deleter*/, &blob2));
+  ASSERT_OK(blob_cache->Insert("blob2", nullptr /*value*/,
+                               &kNoopCacheItemHelper, kSize2, &blob2));
   ASSERT_NE(nullptr, blob2);
   ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheCapacity, &value));
   ASSERT_EQ(kCapacity, value);
@@ -1876,8 +1876,8 @@ TEST_F(DBPropertiesTest, BlobCacheProperties) {
   // Insert another pinned blob to make the cache over-sized.
   constexpr size_t kSize3 = 80;
   Cache::Handle* blob3 = nullptr;
-  ASSERT_OK(blob_cache->Insert("blob3", nullptr /*value*/, kSize3,
-                               nullptr /*deleter*/, &blob3));
+  ASSERT_OK(blob_cache->Insert("blob3", nullptr /*value*/,
+                               &kNoopCacheItemHelper, kSize3, &blob3));
   ASSERT_NE(nullptr, blob3);
   ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlobCacheCapacity, &value));
   ASSERT_EQ(kCapacity, value);
@@ -1956,8 +1956,8 @@ TEST_F(DBPropertiesTest, BlockCacheProperties) {
 
   // Insert unpinned item to the cache and check size.
   constexpr size_t kSize1 = 50;
-  ASSERT_OK(block_cache->Insert("item1", nullptr /*value*/, kSize1,
-                                nullptr /*deleter*/));
+  ASSERT_OK(block_cache->Insert("item1", nullptr /*value*/,
+                                &kNoopCacheItemHelper, kSize1));
   ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
   ASSERT_EQ(kCapacity, value);
   ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheUsage, &value));
@@ -1969,8 +1969,8 @@ TEST_F(DBPropertiesTest, BlockCacheProperties) {
   // Insert pinned item to the cache and check size.
   constexpr size_t kSize2 = 30;
   Cache::Handle* item2 = nullptr;
-  ASSERT_OK(block_cache->Insert("item2", nullptr /*value*/, kSize2,
-                                nullptr /*deleter*/, &item2));
+  ASSERT_OK(block_cache->Insert("item2", nullptr /*value*/,
+                                &kNoopCacheItemHelper, kSize2, &item2));
   ASSERT_NE(nullptr, item2);
   ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
   ASSERT_EQ(kCapacity, value);
@@ -1983,8 +1983,8 @@ TEST_F(DBPropertiesTest, BlockCacheProperties) {
   // Insert another pinned item to make the cache over-sized.
   constexpr size_t kSize3 = 80;
   Cache::Handle* item3 = nullptr;
-  ASSERT_OK(block_cache->Insert("item3", nullptr /*value*/, kSize3,
-                                nullptr /*deleter*/, &item3));
+  ASSERT_OK(block_cache->Insert("item3", nullptr /*value*/,
+                                &kNoopCacheItemHelper, kSize3, &item3));
   ASSERT_NE(nullptr, item2);
   ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBlockCacheCapacity, &value));
   ASSERT_EQ(kCapacity, value);

db/db_range_del_test.cc

@@ -479,7 +479,10 @@ TEST_F(DBRangeDelTest, ValidUniversalSubcompactionBoundaries) {
       std::vector<std::string> values;
       // Write 100KB (100 values, each 1K)
       for (int k = 0; k < kNumPerFile; k++) {
-        values.push_back(rnd.RandomString(990));
+        // For the highest level, use smaller value size such that it does not
+        // prematurely cause auto compaction due to range tombstone adding
+        // additional compensated file size
+        values.push_back(rnd.RandomString((i == kNumLevels - 2) ? 600 : 990));
         ASSERT_OK(Put(Key(j * kNumPerFile + k), values[k]));
       }
       // put extra key to trigger flush
@@ -492,7 +495,13 @@ TEST_F(DBRangeDelTest, ValidUniversalSubcompactionBoundaries) {
     }
     ASSERT_OK(dbfull()->TEST_WaitForCompact());
     ASSERT_EQ(NumTableFilesAtLevel(0), 0);
-    ASSERT_GT(NumTableFilesAtLevel(kNumLevels - 1 - i), kFilesPerLevel - 1);
+    if (i == kNumLevels - 2) {
+      // For the highest level, value size is smaller (see Put() above),
+      // so output file number is smaller.
+      ASSERT_GT(NumTableFilesAtLevel(kNumLevels - 1 - i), kFilesPerLevel - 2);
+    } else {
+      ASSERT_GT(NumTableFilesAtLevel(kNumLevels - 1 - i), kFilesPerLevel - 1);
+    }
   }
   // Now L1-L3 are full, when we compact L1->L2 we should see (1) subcompactions
   // happen since input level > 0; (2) range deletions are not dropped since
@@ -2796,6 +2805,225 @@ TEST_F(DBRangeDelTest, RangeTombstoneRespectIterateUpperBound) {
   ASSERT_OK(iter->status());
 }
 
+TEST_F(DBRangeDelTest, RangetombesoneCompensateFilesize) {
+  Options opts = CurrentOptions();
+  opts.disable_auto_compactions = true;
+  DestroyAndReopen(opts);
+
+  std::vector<std::string> values;
+  Random rnd(301);
+  // file in L2
+  values.push_back(rnd.RandomString(1 << 10));
+  ASSERT_OK(Put("a", values.back()));
+  values.push_back(rnd.RandomString(1 << 10));
+  ASSERT_OK(Put("b", values.back()));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(2);
+  uint64_t l2_size = 0;
+  ASSERT_OK(Size("a", "c", 0 /* cf */, &l2_size));
+  ASSERT_GT(l2_size, 0);
+  // file in L1
+  values.push_back(rnd.RandomString(1 << 10));
+  ASSERT_OK(Put("d", values.back()));
+  values.push_back(rnd.RandomString(1 << 10));
+  ASSERT_OK(Put("e", values.back()));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1);
+  uint64_t l1_size = 0;
+  ASSERT_OK(Size("d", "f", 0 /* cf */, &l1_size));
+  ASSERT_GT(l1_size, 0);
+
+  ASSERT_OK(
+      db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "f"));
+  ASSERT_OK(Flush());
+  // Range deletion compensated size computed during flush time
+  std::vector<std::vector<FileMetaData>> level_to_files;
+  dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
+                                  &level_to_files);
+  ASSERT_EQ(level_to_files[0].size(), 1);
+  ASSERT_EQ(level_to_files[0][0].compensated_range_deletion_size,
+            l1_size + l2_size);
+  ASSERT_EQ(level_to_files[1].size(), 1);
+  ASSERT_EQ(level_to_files[1][0].compensated_range_deletion_size, 0);
+  ASSERT_EQ(level_to_files[2].size(), 1);
+  ASSERT_EQ(level_to_files[2][0].compensated_range_deletion_size, 0);
+
+  // Range deletion compensated size computed during compaction time
+  ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr,
+                                        true /* disallow_trivial_move */));
+  ASSERT_EQ(NumTableFilesAtLevel(0), 0);
+  ASSERT_EQ(NumTableFilesAtLevel(1), 1);
+  ASSERT_EQ(NumTableFilesAtLevel(2), 1);
+  dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
+                                  &level_to_files);
+  ASSERT_EQ(level_to_files[1].size(), 1);
+  ASSERT_EQ(level_to_files[1][0].compensated_range_deletion_size, l2_size);
+  ASSERT_EQ(level_to_files[2].size(), 1);
+  ASSERT_EQ(level_to_files[2][0].compensated_range_deletion_size, 0);
+}
+
+TEST_F(DBRangeDelTest, RangetombesoneCompensateFilesizePersistDuringReopen) {
+  Options opts = CurrentOptions();
+  opts.disable_auto_compactions = true;
+  DestroyAndReopen(opts);
+
+  std::vector<std::string> values;
+  Random rnd(301);
+  values.push_back(rnd.RandomString(1 << 10));
+  ASSERT_OK(Put("a", values.back()));
+  values.push_back(rnd.RandomString(1 << 10));
+  ASSERT_OK(Put("b", values.back()));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(2);
+
+  ASSERT_OK(
+      db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "c"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1);
+
+  ASSERT_OK(
+      db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "z"));
+  ASSERT_OK(Flush());
+
+  std::vector<std::vector<FileMetaData>> level_to_files;
+  dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
+                                  &level_to_files);
+  ASSERT_EQ(level_to_files[0].size(), 1);
+  ASSERT_EQ(level_to_files[1].size(), 1);
+  ASSERT_EQ(level_to_files[2].size(), 1);
+  uint64_t l2_size = level_to_files[2][0].fd.GetFileSize();
+  uint64_t l1_size = level_to_files[1][0].fd.GetFileSize();
+  ASSERT_GT(l2_size, 0);
+  ASSERT_GT(l1_size, 0);
+  ASSERT_EQ(level_to_files[0][0].compensated_range_deletion_size,
+            l1_size + l2_size);
+  ASSERT_EQ(level_to_files[1][0].compensated_range_deletion_size, l2_size);
+
+  Reopen(opts);
+  dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
+                                  &level_to_files);
+  ASSERT_EQ(level_to_files[0].size(), 1);
+  ASSERT_EQ(level_to_files[0][0].compensated_range_deletion_size,
+            l1_size + l2_size);
+  ASSERT_EQ(level_to_files[1].size(), 1);
+  ASSERT_EQ(level_to_files[1][0].compensated_range_deletion_size, l2_size);
+}
+
+TEST_F(DBRangeDelTest, SingleKeyFile) {
+  // Test for a bug fix where a range tombstone could be added
+  // to an SST file while is not within the file's key range.
+  // Create 3 files in L0 and then L1 where all keys have the same user key
+  // `Key(2)`. The middle file will contain Key(2)@6 and Key(2)@5. Before fix,
+  // the range tombstone [Key(2), Key(5))@2 would be added to this file during
+  // compaction, but it is not in this file's key range.
+  Options opts = CurrentOptions();
+  opts.disable_auto_compactions = true;
+  opts.target_file_size_base = 1 << 10;
+  opts.level_compaction_dynamic_file_size = false;
+  DestroyAndReopen(opts);
+
+  // prevent range tombstone drop
+  std::vector<const Snapshot*> snapshots;
+  snapshots.push_back(db_->GetSnapshot());
+
+  // write a key to bottommost file so the compactions below
+  // are not bottommost compactions and will calculate
+  // compensated range tombstone size. Before bug fix, an assert would fail
+  // during this process.
+  Random rnd(301);
+  ASSERT_OK(Put(Key(2), rnd.RandomString(8 << 10)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(6);
+
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(2),
+                             Key(5)));
+  snapshots.push_back(db_->GetSnapshot());
+  std::vector<std::string> values;
+
+  values.push_back(rnd.RandomString(8 << 10));
+  ASSERT_OK(Put(Key(2), rnd.RandomString(8 << 10)));
+  snapshots.push_back(db_->GetSnapshot());
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put(Key(2), rnd.RandomString(8 << 10)));
+  snapshots.push_back(db_->GetSnapshot());
+  ASSERT_OK(Flush());
+
+  ASSERT_OK(Put(Key(2), rnd.RandomString(8 << 10)));
+  snapshots.push_back(db_->GetSnapshot());
+  ASSERT_OK(Flush());
+
+  ASSERT_EQ(NumTableFilesAtLevel(0), 3);
+  CompactRangeOptions co;
+  co.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+
+  ASSERT_OK(dbfull()->RunManualCompaction(
+      static_cast_with_check<ColumnFamilyHandleImpl>(db_->DefaultColumnFamily())
+          ->cfd(),
+      0, 1, co, nullptr, nullptr, true, true,
+      std::numeric_limits<uint64_t>::max() /*max_file_num_to_ignore*/,
+      "" /*trim_ts*/));
+
+  for (const auto s : snapshots) {
+    db_->ReleaseSnapshot(s);
+  }
+}
+
+TEST_F(DBRangeDelTest, DoubleCountRangeTombstoneCompensatedSize) {
+  // Test for a bug fix if a file has multiple range tombstones
+  // with same start and end key but with different sequence numbers,
+  // we should only calculate compensated range tombstone size
+  // for one of them.
+  Options opts = CurrentOptions();
+  opts.disable_auto_compactions = true;
+  DestroyAndReopen(opts);
+
+  std::vector<std::string> values;
+  Random rnd(301);
+  // file in L2
+  ASSERT_OK(Put(Key(1), rnd.RandomString(1 << 10)));
+  ASSERT_OK(Put(Key(2), rnd.RandomString(1 << 10)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(2);
+  uint64_t l2_size = 0;
+  ASSERT_OK(Size(Key(1), Key(3), 0 /* cf */, &l2_size));
+  ASSERT_GT(l2_size, 0);
+
+  // file in L1
+  ASSERT_OK(Put(Key(3), rnd.RandomString(1 << 10)));
+  ASSERT_OK(Put(Key(4), rnd.RandomString(1 << 10)));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1);
+  uint64_t l1_size = 0;
+  ASSERT_OK(Size(Key(3), Key(5), 0 /* cf */, &l1_size));
+  ASSERT_GT(l1_size, 0);
+
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(1),
+                             Key(5)));
+  // so that the range tombstone above is not dropped
+  const Snapshot* snapshot = db_->GetSnapshot();
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(1),
+                             Key(5)));
+  ASSERT_OK(Flush());
+  // Range deletion compensated size computed during flush time
+  std::vector<std::vector<FileMetaData>> level_to_files;
+  dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
+                                  &level_to_files);
+  ASSERT_EQ(level_to_files[0].size(), 1);
+  // instead of 2 * (l1_size + l2_size)
+  ASSERT_EQ(level_to_files[0][0].compensated_range_deletion_size,
+            l1_size + l2_size);
+
+  // Range deletion compensated size computed during compaction time
+  ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr, nullptr,
+                                        true /* disallow_trivial_move */));
+  dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
+                                  &level_to_files);
+  ASSERT_EQ(level_to_files[1].size(), 1);
+  ASSERT_EQ(level_to_files[1][0].compensated_range_deletion_size, l2_size);
+  db_->ReleaseSnapshot(snapshot);
+}
+
 #endif  // ROCKSDB_LITE
 
 }  // namespace ROCKSDB_NAMESPACE

db/db_test.cc

@@ -1203,6 +1203,8 @@ void CheckColumnFamilyMeta(
                 file_meta_from_files.file_creation_time);
       ASSERT_GE(file_meta_from_cf.file_creation_time, start_time);
       ASSERT_LE(file_meta_from_cf.file_creation_time, end_time);
+      ASSERT_EQ(file_meta_from_cf.epoch_number,
+                file_meta_from_files.epoch_number);
       ASSERT_GE(file_meta_from_cf.oldest_ancester_time, start_time);
       ASSERT_LE(file_meta_from_cf.oldest_ancester_time, end_time);
       // More from FileStorageInfo
@@ -1253,6 +1255,7 @@ void CheckLiveFilesMeta(
     ASSERT_EQ(meta.largestkey, expected_meta.largest.user_key().ToString());
     ASSERT_EQ(meta.oldest_blob_file_number,
               expected_meta.oldest_blob_file_number);
+    ASSERT_EQ(meta.epoch_number, expected_meta.epoch_number);
 
     // More from FileStorageInfo
     ASSERT_EQ(meta.file_type, kTableFile);

db/db_test2.cc

@@ -14,6 +14,7 @@
 
 #include "db/db_test_util.h"
 #include "db/read_callback.h"
+#include "db/version_edit.h"
 #include "options/options_helper.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
@@ -33,6 +34,18 @@ namespace ROCKSDB_NAMESPACE {
 class DBTest2 : public DBTestBase {
  public:
   DBTest2() : DBTestBase("db_test2", /*env_do_fsync=*/true) {}
+  std::vector<FileMetaData*> GetLevelFileMetadatas(int level, int cf = 0) {
+    VersionSet* const versions = dbfull()->GetVersionSet();
+    assert(versions);
+    ColumnFamilyData* const cfd =
+        versions->GetColumnFamilySet()->GetColumnFamily(cf);
+    assert(cfd);
+    Version* const current = cfd->current();
+    assert(current);
+    VersionStorageInfo* const storage_info = current->storage_info();
+    assert(storage_info);
+    return storage_info->LevelFiles(level);
+  }
 };
 
 #ifndef ROCKSDB_LITE
@@ -4523,7 +4536,7 @@ TEST_F(DBTest2, TraceAndReplay) {
   ASSERT_OK(replayer->Prepare());
   // Replay using 1 thread, 1x speed.
   ASSERT_OK(replayer->Replay(ReplayOptions(1, 1.0), res_cb));
-  ASSERT_GT(res_handler.GetAvgLatency(), 0.0);
+  ASSERT_GE(res_handler.GetAvgLatency(), 0.0);
   ASSERT_EQ(res_handler.GetNumWrites(), 8);
   ASSERT_EQ(res_handler.GetNumGets(), 3);
   ASSERT_EQ(res_handler.GetNumIterSeeks(), 2);
@@ -4549,7 +4562,7 @@ TEST_F(DBTest2, TraceAndReplay) {
   // Re-replay using 2 threads, 2x speed.
   ASSERT_OK(replayer->Prepare());
   ASSERT_OK(replayer->Replay(ReplayOptions(2, 2.0), res_cb));
-  ASSERT_GT(res_handler.GetAvgLatency(), 0.0);
+  ASSERT_GE(res_handler.GetAvgLatency(), 0.0);
   ASSERT_EQ(res_handler.GetNumWrites(), 8);
   ASSERT_EQ(res_handler.GetNumGets(), 3);
   ASSERT_EQ(res_handler.GetNumIterSeeks(), 2);
@@ -4559,7 +4572,7 @@ TEST_F(DBTest2, TraceAndReplay) {
   // Re-replay using 2 threads, 1/2 speed.
   ASSERT_OK(replayer->Prepare());
   ASSERT_OK(replayer->Replay(ReplayOptions(2, 0.5), res_cb));
-  ASSERT_GT(res_handler.GetAvgLatency(), 0.0);
+  ASSERT_GE(res_handler.GetAvgLatency(), 0.0);
   ASSERT_EQ(res_handler.GetNumWrites(), 8);
   ASSERT_EQ(res_handler.GetNumGets(), 3);
   ASSERT_EQ(res_handler.GetNumIterSeeks(), 2);
@@ -4757,7 +4770,7 @@ TEST_F(DBTest2, TraceAndManualReplay) {
     // end, or Prepare() was not called.
     ASSERT_TRUE(s.IsIncomplete());
     ASSERT_TRUE(replayer->Next(nullptr).IsIncomplete());
-    ASSERT_GT(res_handler.GetAvgLatency(), 0.0);
+    ASSERT_GE(res_handler.GetAvgLatency(), 0.0);
     ASSERT_EQ(res_handler.GetNumWrites(), 9);
     ASSERT_EQ(res_handler.GetNumGets(), 3);
     ASSERT_EQ(res_handler.GetNumIterSeeks(), 8);
@@ -4791,7 +4804,7 @@ TEST_F(DBTest2, TraceAndManualReplay) {
   ASSERT_EQ("write1", value);
   ASSERT_OK(db2->Get(ro, handles[0], "trace-record-write2", &value));
   ASSERT_EQ("write2", value);
-  ASSERT_GT(res_handler.GetAvgLatency(), 0.0);
+  ASSERT_GE(res_handler.GetAvgLatency(), 0.0);
   ASSERT_EQ(res_handler.GetNumWrites(), 1);
   ASSERT_EQ(res_handler.GetNumGets(), 0);
   ASSERT_EQ(res_handler.GetNumIterSeeks(), 0);
@@ -4816,7 +4829,7 @@ TEST_F(DBTest2, TraceAndManualReplay) {
   record.reset(new GetQueryTraceRecord(invalid_cf_id, "whatever", fake_ts++));
   ASSERT_TRUE(replayer->Execute(record, &result).IsCorruption());
   ASSERT_TRUE(result == nullptr);
-  ASSERT_GT(res_handler.GetAvgLatency(), 0.0);
+  ASSERT_GE(res_handler.GetAvgLatency(), 0.0);
   ASSERT_EQ(res_handler.GetNumWrites(), 0);
   ASSERT_EQ(res_handler.GetNumGets(), 2);
   ASSERT_EQ(res_handler.GetNumIterSeeks(), 0);
@@ -4845,7 +4858,7 @@ TEST_F(DBTest2, TraceAndManualReplay) {
     ASSERT_TRUE(replayer->Execute(record, &result).IsCorruption());
     ASSERT_TRUE(result == nullptr);
   }
-  ASSERT_GT(res_handler.GetAvgLatency(), 0.0);
+  ASSERT_GE(res_handler.GetAvgLatency(), 0.0);
   ASSERT_EQ(res_handler.GetNumWrites(), 0);
   ASSERT_EQ(res_handler.GetNumGets(), 0);
   ASSERT_EQ(res_handler.GetNumIterSeeks(), 4);  // Seek x 2 in two iterations
@@ -4900,7 +4913,7 @@ TEST_F(DBTest2, TraceAndManualReplay) {
       std::vector<std::string>({"a"}), fake_ts++));
   ASSERT_TRUE(replayer->Execute(record, &result).IsInvalidArgument());
   ASSERT_TRUE(result == nullptr);
-  ASSERT_GT(res_handler.GetAvgLatency(), 0.0);
+  ASSERT_GE(res_handler.GetAvgLatency(), 0.0);
   ASSERT_EQ(res_handler.GetNumWrites(), 0);
   ASSERT_EQ(res_handler.GetNumGets(), 0);
   ASSERT_EQ(res_handler.GetNumIterSeeks(), 0);
@@ -7325,6 +7338,218 @@ TEST_F(DBTest2, PointInTimeRecoveryWithSyncFailureInCFCreation) {
   ReopenWithColumnFamilies({"default", "test1", "test2"}, options);
 }
 
+#ifndef ROCKSDB_LITE
+TEST_F(DBTest2, SortL0FilesByEpochNumber) {
+  Options options = CurrentOptions();
+  options.num_levels = 1;
+  options.compaction_style = kCompactionStyleUniversal;
+  DestroyAndReopen(options);
+
+  // Set up L0 files to be sorted by their epoch_number
+  ASSERT_OK(Put("key1", "seq1"));
+
+  SstFileWriter sst_file_writer{EnvOptions(), options};
+  std::string external_file1 = dbname_ + "/test_files1.sst";
+  std::string external_file2 = dbname_ + "/test_files2.sst";
+  ASSERT_OK(sst_file_writer.Open(external_file1));
+  ASSERT_OK(sst_file_writer.Put("key2", "seq0"));
+  ASSERT_OK(sst_file_writer.Finish());
+  ASSERT_OK(sst_file_writer.Open(external_file2));
+  ASSERT_OK(sst_file_writer.Put("key3", "seq0"));
+  ASSERT_OK(sst_file_writer.Finish());
+
+  ASSERT_OK(Put("key4", "seq2"));
+  ASSERT_OK(Flush());
+
+  auto* handle = db_->DefaultColumnFamily();
+  ASSERT_OK(db_->IngestExternalFile(handle, {external_file1, external_file2},
+                                    IngestExternalFileOptions()));
+
+  // To verify L0 files are sorted by epoch_number in descending order
+  // instead of largest_seqno
+  std::vector<FileMetaData*> level0_files = GetLevelFileMetadatas(0 /* level*/);
+  ASSERT_EQ(level0_files.size(), 3);
+
+  EXPECT_EQ(level0_files[0]->epoch_number, 3);
+  EXPECT_EQ(level0_files[0]->fd.largest_seqno, 0);
+  ASSERT_EQ(level0_files[0]->num_entries, 1);
+  ASSERT_TRUE(level0_files[0]->largest.user_key() == Slice("key3"));
+
+  EXPECT_EQ(level0_files[1]->epoch_number, 2);
+  EXPECT_EQ(level0_files[1]->fd.largest_seqno, 0);
+  ASSERT_EQ(level0_files[1]->num_entries, 1);
+  ASSERT_TRUE(level0_files[1]->largest.user_key() == Slice("key2"));
+
+  EXPECT_EQ(level0_files[2]->epoch_number, 1);
+  EXPECT_EQ(level0_files[2]->fd.largest_seqno, 2);
+  ASSERT_EQ(level0_files[2]->num_entries, 2);
+  ASSERT_TRUE(level0_files[2]->largest.user_key() == Slice("key4"));
+  ASSERT_TRUE(level0_files[2]->smallest.user_key() == Slice("key1"));
+
+  // To verify compacted file is assigned with the minimum epoch_number
+  // among input files'
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+
+  level0_files = GetLevelFileMetadatas(0 /* level*/);
+  ASSERT_EQ(level0_files.size(), 1);
+  EXPECT_EQ(level0_files[0]->epoch_number, 1);
+  ASSERT_EQ(level0_files[0]->num_entries, 4);
+  ASSERT_TRUE(level0_files[0]->largest.user_key() == Slice("key4"));
+  ASSERT_TRUE(level0_files[0]->smallest.user_key() == Slice("key1"));
+}
+
+TEST_F(DBTest2, SameEpochNumberAfterCompactRangeChangeLevel) {
+  Options options = CurrentOptions();
+  options.num_levels = 7;
+  options.compaction_style = CompactionStyle::kCompactionStyleLevel;
+  options.disable_auto_compactions = true;
+  DestroyAndReopen(options);
+
+  // Set up the file in L1 to be moved to L0 in later step of CompactRange()
+  ASSERT_OK(Put("key1", "seq1"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(1, 0);
+  std::vector<FileMetaData*> level0_files = GetLevelFileMetadatas(0 /* level*/);
+  ASSERT_EQ(level0_files.size(), 0);
+  std::vector<FileMetaData*> level1_files = GetLevelFileMetadatas(1 /* level*/);
+  ASSERT_EQ(level1_files.size(), 1);
+  std::vector<FileMetaData*> level2_files = GetLevelFileMetadatas(2 /* level*/);
+  ASSERT_EQ(level2_files.size(), 0);
+
+  ASSERT_EQ(level1_files[0]->epoch_number, 1);
+
+  // To verify CompactRange() moving file to L0 still keeps the file's
+  // epoch_number
+  CompactRangeOptions croptions;
+  croptions.change_level = true;
+  croptions.target_level = 0;
+  ASSERT_OK(db_->CompactRange(croptions, nullptr, nullptr));
+  level0_files = GetLevelFileMetadatas(0 /* level*/);
+  level1_files = GetLevelFileMetadatas(1 /* level*/);
+  ASSERT_EQ(level0_files.size(), 1);
+  ASSERT_EQ(level1_files.size(), 0);
+
+  EXPECT_EQ(level0_files[0]->epoch_number, 1);
+
+  ASSERT_EQ(level0_files[0]->num_entries, 1);
+  ASSERT_TRUE(level0_files[0]->largest.user_key() == Slice("key1"));
+}
+
+TEST_F(DBTest2, RecoverEpochNumber) {
+  for (bool allow_ingest_behind : {true, false}) {
+    Options options = CurrentOptions();
+    options.allow_ingest_behind = allow_ingest_behind;
+    options.num_levels = 7;
+    options.compaction_style = kCompactionStyleLevel;
+    options.disable_auto_compactions = true;
+    DestroyAndReopen(options);
+    CreateAndReopenWithCF({"cf1"}, options);
+    VersionSet* versions = dbfull()->GetVersionSet();
+    assert(versions);
+    const ColumnFamilyData* default_cf =
+        versions->GetColumnFamilySet()->GetDefault();
+    const ColumnFamilyData* cf1 =
+        versions->GetColumnFamilySet()->GetColumnFamily("cf1");
+
+    // Set up files in default CF to recover in later step
+    ASSERT_OK(Put("key1", "epoch1"));
+    ASSERT_OK(Flush());
+    MoveFilesToLevel(1 /* level*/, 0 /* cf*/);
+    ASSERT_OK(Put("key2", "epoch2"));
+    ASSERT_OK(Flush());
+
+    std::vector<FileMetaData*> level0_files =
+        GetLevelFileMetadatas(0 /* level*/);
+    ASSERT_EQ(level0_files.size(), 1);
+    ASSERT_EQ(level0_files[0]->epoch_number,
+              allow_ingest_behind
+                  ? 2 + kReservedEpochNumberForFileIngestedBehind
+                  : 2);
+    ASSERT_EQ(level0_files[0]->num_entries, 1);
+    ASSERT_TRUE(level0_files[0]->largest.user_key() == Slice("key2"));
+
+    std::vector<FileMetaData*> level1_files =
+        GetLevelFileMetadatas(1 /* level*/);
+    ASSERT_EQ(level1_files.size(), 1);
+    ASSERT_EQ(level1_files[0]->epoch_number,
+              allow_ingest_behind
+                  ? 1 + kReservedEpochNumberForFileIngestedBehind
+                  : 1);
+    ASSERT_EQ(level1_files[0]->num_entries, 1);
+    ASSERT_TRUE(level1_files[0]->largest.user_key() == Slice("key1"));
+
+    // Set up files in cf1 to recover in later step
+    ASSERT_OK(Put(1 /* cf */, "cf1_key1", "epoch1"));
+    ASSERT_OK(Flush(1 /* cf */));
+
+    std::vector<FileMetaData*> level0_files_cf1 =
+        GetLevelFileMetadatas(0 /* level*/, 1 /* cf*/);
+    ASSERT_EQ(level0_files_cf1.size(), 1);
+    ASSERT_EQ(level0_files_cf1[0]->epoch_number,
+              allow_ingest_behind
+                  ? 1 + kReservedEpochNumberForFileIngestedBehind
+                  : 1);
+    ASSERT_EQ(level0_files_cf1[0]->num_entries, 1);
+    ASSERT_TRUE(level0_files_cf1[0]->largest.user_key() == Slice("cf1_key1"));
+
+    ASSERT_EQ(default_cf->GetNextEpochNumber(),
+              allow_ingest_behind
+                  ? 3 + kReservedEpochNumberForFileIngestedBehind
+                  : 3);
+    ASSERT_EQ(cf1->GetNextEpochNumber(),
+              allow_ingest_behind
+                  ? 2 + kReservedEpochNumberForFileIngestedBehind
+                  : 2);
+
+    // To verify epoch_number of files of different levels/CFs are
+    // persisted and recovered correctly
+    ReopenWithColumnFamilies({"default", "cf1"}, options);
+    versions = dbfull()->GetVersionSet();
+    assert(versions);
+    default_cf = versions->GetColumnFamilySet()->GetDefault();
+    cf1 = versions->GetColumnFamilySet()->GetColumnFamily("cf1");
+
+    level0_files = GetLevelFileMetadatas(0 /* level*/);
+    ASSERT_EQ(level0_files.size(), 1);
+    EXPECT_EQ(level0_files[0]->epoch_number,
+              allow_ingest_behind
+                  ? 2 + kReservedEpochNumberForFileIngestedBehind
+                  : 2);
+    ASSERT_EQ(level0_files[0]->num_entries, 1);
+    ASSERT_TRUE(level0_files[0]->largest.user_key() == Slice("key2"));
+
+    level1_files = GetLevelFileMetadatas(1 /* level*/);
+    ASSERT_EQ(level1_files.size(), 1);
+    EXPECT_EQ(level1_files[0]->epoch_number,
+              allow_ingest_behind
+                  ? 1 + kReservedEpochNumberForFileIngestedBehind
+                  : 1);
+    ASSERT_EQ(level1_files[0]->num_entries, 1);
+    ASSERT_TRUE(level1_files[0]->largest.user_key() == Slice("key1"));
+
+    level0_files_cf1 = GetLevelFileMetadatas(0 /* level*/, 1 /* cf*/);
+    ASSERT_EQ(level0_files_cf1.size(), 1);
+    EXPECT_EQ(level0_files_cf1[0]->epoch_number,
+              allow_ingest_behind
+                  ? 1 + kReservedEpochNumberForFileIngestedBehind
+                  : 1);
+    ASSERT_EQ(level0_files_cf1[0]->num_entries, 1);
+    ASSERT_TRUE(level0_files_cf1[0]->largest.user_key() == Slice("cf1_key1"));
+
+    // To verify next epoch number is recovered correctly
+    EXPECT_EQ(default_cf->GetNextEpochNumber(),
+              allow_ingest_behind
+                  ? 3 + kReservedEpochNumberForFileIngestedBehind
+                  : 3);
+    EXPECT_EQ(cf1->GetNextEpochNumber(),
+              allow_ingest_behind
+                  ? 2 + kReservedEpochNumberForFileIngestedBehind
+                  : 2);
+  }
+}
+
+#endif  // ROCKSDB_LITE
+
 TEST_F(DBTest2, RenameDirectory) {
   Options options = CurrentOptions();
   DestroyAndReopen(options);

db/db_test_util.cc

@@ -1723,12 +1723,13 @@ TargetCacheChargeTrackingCache<R>::TargetCacheChargeTrackingCache(
       cache_charge_increments_sum_(0) {}
 
 template <CacheEntryRole R>
-Status TargetCacheChargeTrackingCache<R>::Insert(
-    const Slice& key, void* value, size_t charge,
-    void (*deleter)(const Slice& key, void* value), Handle** handle,
-    Priority priority) {
-  Status s = target_->Insert(key, value, charge, deleter, handle, priority);
-  if (deleter == kNoopDeleter) {
+Status TargetCacheChargeTrackingCache<R>::Insert(const Slice& key,
+                                                 ObjectPtr value,
+                                                 const CacheItemHelper* helper,
+                                                 size_t charge, Handle** handle,
+                                                 Priority priority) {
+  Status s = target_->Insert(key, value, helper, charge, handle, priority);
+  if (helper == kCrmHelper) {
     if (last_peak_tracked_) {
       cache_charge_peak_ = 0;
       cache_charge_increment_ = 0;
@@ -1747,8 +1748,8 @@ Status TargetCacheChargeTrackingCache<R>::Insert(
 template <CacheEntryRole R>
 bool TargetCacheChargeTrackingCache<R>::Release(Handle* handle,
                                                 bool erase_if_last_ref) {
-  auto deleter = GetDeleter(handle);
-  if (deleter == kNoopDeleter) {
+  auto helper = GetCacheItemHelper(handle);
+  if (helper == kCrmHelper) {
     if (!last_peak_tracked_) {
       cache_charge_peaks_.push_back(cache_charge_peak_);
       cache_charge_increments_sum_ += cache_charge_increment_;
@@ -1761,8 +1762,8 @@ bool TargetCacheChargeTrackingCache<R>::Release(Handle* handle,
 }
 
 template <CacheEntryRole R>
-const Cache::DeleterFn TargetCacheChargeTrackingCache<R>::kNoopDeleter =
-    CacheReservationManagerImpl<R>::TEST_GetNoopDeleterForRole();
+const Cache::CacheItemHelper* TargetCacheChargeTrackingCache<R>::kCrmHelper =
+    CacheReservationManagerImpl<R>::TEST_GetCacheItemHelperForRole();
 
 template class TargetCacheChargeTrackingCache<
     CacheEntryRole::kFilterConstruction>;

db/db_test_util.h

@@ -49,6 +49,9 @@
 #include "util/string_util.h"
 #include "utilities/merge_operators.h"
 
+// In case defined by Windows headers
+#undef small
+
 namespace ROCKSDB_NAMESPACE {
 class MockEnv;
 
@@ -870,17 +873,34 @@ class FlushCounterListener : public EventListener {
 #endif
 
 // A test merge operator mimics put but also fails if one of merge operands is
-// "corrupted".
+// "corrupted", "corrupted_try_merge", or "corrupted_must_merge".
 class TestPutOperator : public MergeOperator {
  public:
   virtual bool FullMergeV2(const MergeOperationInput& merge_in,
                            MergeOperationOutput* merge_out) const override {
+    static const std::map<std::string, MergeOperator::OpFailureScope>
+        bad_operand_to_op_failure_scope = {
+            {"corrupted", MergeOperator::OpFailureScope::kDefault},
+            {"corrupted_try_merge", MergeOperator::OpFailureScope::kTryMerge},
+            {"corrupted_must_merge",
+             MergeOperator::OpFailureScope::kMustMerge}};
+    auto check_operand =
+        [](Slice operand_val,
+           MergeOperator::OpFailureScope* op_failure_scope) -> bool {
+      auto iter = bad_operand_to_op_failure_scope.find(operand_val.ToString());
+      if (iter != bad_operand_to_op_failure_scope.end()) {
+        *op_failure_scope = iter->second;
+        return false;
+      }
+      return true;
+    };
     if (merge_in.existing_value != nullptr &&
-        *(merge_in.existing_value) == "corrupted") {
+        !check_operand(*merge_in.existing_value,
+                       &merge_out->op_failure_scope)) {
       return false;
     }
     for (auto value : merge_in.operand_list) {
-      if (value == "corrupted") {
+      if (!check_operand(value, &merge_out->op_failure_scope)) {
         return false;
       }
     }
@@ -900,17 +920,18 @@ class CacheWrapper : public Cache {
 
   const char* Name() const override { return target_->Name(); }
 
-  using Cache::Insert;
-  Status Insert(const Slice& key, void* value, size_t charge,
-                void (*deleter)(const Slice& key, void* value),
+  Status Insert(const Slice& key, ObjectPtr value,
+                const CacheItemHelper* helper, size_t charge,
                 Handle** handle = nullptr,
                 Priority priority = Priority::LOW) override {
-    return target_->Insert(key, value, charge, deleter, handle, priority);
+    return target_->Insert(key, value, helper, charge, handle, priority);
   }
 
-  using Cache::Lookup;
-  Handle* Lookup(const Slice& key, Statistics* stats = nullptr) override {
-    return target_->Lookup(key, stats);
+  Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
+                 CreateContext* create_context,
+                 Priority priority = Priority::LOW, bool wait = true,
+                 Statistics* stats = nullptr) override {
+    return target_->Lookup(key, helper, create_context, priority, wait, stats);
   }
 
   bool Ref(Handle* handle) override { return target_->Ref(handle); }
@@ -920,7 +941,7 @@ class CacheWrapper : public Cache {
     return target_->Release(handle, erase_if_last_ref);
   }
 
-  void* Value(Handle* handle) override { return target_->Value(handle); }
+  ObjectPtr Value(Handle* handle) override { return target_->Value(handle); }
 
   void Erase(const Slice& key) override { target_->Erase(key); }
   uint64_t NewId() override { return target_->NewId(); }
@@ -949,18 +970,13 @@ class CacheWrapper : public Cache {
     return target_->GetCharge(handle);
   }
 
-  DeleterFn GetDeleter(Handle* handle) const override {
-    return target_->GetDeleter(handle);
-  }
-
-  void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                              bool thread_safe) override {
-    target_->ApplyToAllCacheEntries(callback, thread_safe);
+  const CacheItemHelper* GetCacheItemHelper(Handle* handle) const override {
+    return target_->GetCacheItemHelper(handle);
   }
 
   void ApplyToAllEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
+      const std::function<void(const Slice& key, ObjectPtr value, size_t charge,
+                               const CacheItemHelper* helper)>& callback,
       const ApplyToAllEntriesOptions& opts) override {
     target_->ApplyToAllEntries(callback, opts);
   }
@@ -988,9 +1004,8 @@ class TargetCacheChargeTrackingCache : public CacheWrapper {
  public:
   explicit TargetCacheChargeTrackingCache(std::shared_ptr<Cache> target);
 
-  using Cache::Insert;
-  Status Insert(const Slice& key, void* value, size_t charge,
-                void (*deleter)(const Slice& key, void* value),
+  Status Insert(const Slice& key, ObjectPtr value,
+                const CacheItemHelper* helper, size_t charge,
                 Handle** handle = nullptr,
                 Priority priority = Priority::LOW) override;
 
@@ -1006,7 +1021,7 @@ class TargetCacheChargeTrackingCache : public CacheWrapper {
   }
 
  private:
-  static const Cache::DeleterFn kNoopDeleter;
+  static const Cache::CacheItemHelper* kCrmHelper;
 
   std::size_t cur_cache_charge_;
   std::size_t cache_charge_peak_;

db/db_wal_test.cc

@@ -610,6 +610,52 @@ TEST_F(DBWALTest, WALWithChecksumHandoff) {
 #endif  // ROCKSDB_ASSERT_STATUS_CHECKED
 }
 
+#ifndef ROCKSDB_LITE
+TEST_F(DBWALTest, LockWal) {
+  do {
+    Options options = CurrentOptions();
+    options.create_if_missing = true;
+    DestroyAndReopen(options);
+    SyncPoint::GetInstance()->DisableProcessing();
+    SyncPoint::GetInstance()->LoadDependency(
+        {{"DBWALTest::LockWal:AfterGetSortedWal",
+          "DBWALTest::LockWal:BeforeFlush:1"}});
+    SyncPoint::GetInstance()->EnableProcessing();
+
+    ASSERT_OK(Put("foo", "v"));
+    ASSERT_OK(Put("bar", "v"));
+    port::Thread worker([&]() {
+      TEST_SYNC_POINT("DBWALTest::LockWal:BeforeFlush:1");
+      Status tmp_s = db_->Flush(FlushOptions());
+      ASSERT_OK(tmp_s);
+    });
+
+    ASSERT_OK(db_->LockWAL());
+    // Verify writes are stopped
+    WriteOptions wopts;
+    wopts.no_slowdown = true;
+    Status s = db_->Put(wopts, "foo", "dontcare");
+    ASSERT_TRUE(s.IsIncomplete());
+    {
+      VectorLogPtr wals;
+      ASSERT_OK(db_->GetSortedWalFiles(wals));
+      ASSERT_FALSE(wals.empty());
+    }
+    TEST_SYNC_POINT("DBWALTest::LockWal:AfterGetSortedWal");
+    FlushOptions flush_opts;
+    flush_opts.wait = false;
+    s = db_->Flush(flush_opts);
+    ASSERT_TRUE(s.IsTryAgain());
+    ASSERT_OK(db_->UnlockWAL());
+    ASSERT_OK(db_->Put(WriteOptions(), "foo", "dontcare"));
+
+    worker.join();
+
+    SyncPoint::GetInstance()->DisableProcessing();
+  } while (ChangeWalOptions());
+}
+#endif  //! ROCKSDB_LITE
+
 class DBRecoveryTestBlobError
     : public DBWALTest,
       public testing::WithParamInterface<std::string> {
@@ -1599,6 +1645,89 @@ TEST_F(DBWALTest, RaceInstallFlushResultsWithWalObsoletion) {
   delete db1;
 }
 
+TEST_F(DBWALTest, FixSyncWalOnObseletedWalWithNewManifestCausingMissingWAL) {
+  Options options = CurrentOptions();
+  options.track_and_verify_wals_in_manifest = true;
+  DestroyAndReopen(options);
+
+  // Accumulate memtable m1 and create the 1st wal (i.e, 4.log)
+  ASSERT_OK(Put(Key(1), ""));
+  ASSERT_OK(Put(Key(2), ""));
+  ASSERT_OK(Put(Key(3), ""));
+
+  const std::string wal_file_path = db_->GetName() + "/000004.log";
+
+  // Coerce the following sequence of events:
+  // (1) Flush() marks 4.log to be obsoleted, 8.log to be the latest (i.e,
+  // active) log and release the lock
+  // (2) SyncWAL() proceeds with the lock. It
+  // creates a new manifest and syncs all the inactive wals before the latest
+  // (i.e, active log), which is 4.log. Note that SyncWAL() is not aware of the
+  // fact that 4.log has marked as to be obseleted. Prior to the fix, such wal
+  // sync will then add a WAL addition record of 4.log to the new manifest
+  // without any special treatment.
+  // (3) BackgroundFlush() will eventually purge 4.log.
+  bool wal_synced = false;
+  SyncPoint::GetInstance()->SetCallBack(
+      "FindObsoleteFiles::PostMutexUnlock", [&](void*) {
+        ASSERT_OK(env_->FileExists(wal_file_path));
+
+        SyncPoint::GetInstance()->SetCallBack(
+            "VersionSet::ProcessManifestWrites:"
+            "PostDecidingCreateNewManifestOrNot",
+            [&](void* arg) {
+              bool* new_descriptor_log = (bool*)arg;
+              *new_descriptor_log = true;
+            });
+
+        ASSERT_OK(db_->SyncWAL());
+        wal_synced = true;
+      });
+
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::DeleteObsoleteFileImpl:AfterDeletion2", [&](void* arg) {
+        std::string* file_name = (std::string*)arg;
+        if (*file_name == wal_file_path) {
+          TEST_SYNC_POINT(
+              "DBWALTest::"
+              "FixSyncWalOnObseletedWalWithNewManifestCausingMissingWAL::"
+              "PostDeleteWAL");
+        }
+      });
+
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::BackgroundCallFlush:FilesFound",
+        "PreConfrimObsoletedWALSynced"},
+       {"DBWALTest::FixSyncWalOnObseletedWalWithNewManifestCausingMissingWAL::"
+        "PostDeleteWAL",
+        "PreConfrimWALDeleted"}});
+
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(Flush());
+
+  TEST_SYNC_POINT("PreConfrimObsoletedWALSynced");
+  ASSERT_TRUE(wal_synced);
+
+  TEST_SYNC_POINT("PreConfrimWALDeleted");
+  // BackgroundFlush() purged 4.log
+  // because the memtable associated with the WAL was flushed and new WAL was
+  // created (i.e, 8.log)
+  ASSERT_TRUE(env_->FileExists(wal_file_path).IsNotFound());
+
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+
+  // To verify the corruption of "Missing WAL with log number: 4" under
+  // `options.track_and_verify_wals_in_manifest = true` is fixed.
+  //
+  // Before the fix, `db_->SyncWAL()` will sync and record WAL addtion of the
+  // obseleted WAL 4.log in a new manifest without any special treament.
+  // This will result in missing-wal corruption in DB::Reopen().
+  Status s = TryReopen(options);
+  EXPECT_OK(s);
+}
+
 // Test scope:
 // - We expect to open data store under all circumstances
 // - We expect only data upto the point where the first error was encountered

db/db_with_timestamp_basic_test.cc

@@ -3870,6 +3870,34 @@ TEST_F(DBBasicTestWithTimestamp, MergeAfterDeletion) {
 
   Close();
 }
+
+TEST_F(DBBasicTestWithTimestamp, RangeTombstoneApproximateSize) {
+  // Test code path for calculating range tombstone compensated size
+  // during flush and compaction.
+  Options options = CurrentOptions();
+  const size_t kTimestampSize = Timestamp(0, 0).size();
+  TestComparator test_cmp(kTimestampSize);
+  options.comparator = &test_cmp;
+  DestroyAndReopen(options);
+  // So that the compaction below is non-bottommost and will calcualte
+  // compensated range tombstone size.
+  ASSERT_OK(db_->Put(WriteOptions(), Key(1), Timestamp(1, 0), "val"));
+  ASSERT_OK(Flush());
+  MoveFilesToLevel(5);
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
+                             Key(1), Timestamp(1, 0)));
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(1),
+                             Key(2), Timestamp(2, 0)));
+  ASSERT_OK(Flush());
+  ASSERT_OK(dbfull()->RunManualCompaction(
+      static_cast_with_check<ColumnFamilyHandleImpl>(db_->DefaultColumnFamily())
+          ->cfd(),
+      0 /* input_level */, 1 /* output_level */, CompactRangeOptions(),
+      nullptr /* begin */, nullptr /* end */, true /* exclusive */,
+      true /* disallow_trivial_move */,
+      std::numeric_limits<uint64_t>::max() /* max_file_num_to_ignore */,
+      "" /*trim_ts*/));
+}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/db_with_timestamp_compaction_test.cc

@@ -323,6 +323,27 @@ TEST_F(TimestampCompatibleCompactionTest, CompactFilesRangeCheckL1) {
               static_cast<int>(compaction_job_info.input_files.size()));
   }
 }
+
+TEST_F(TimestampCompatibleCompactionTest, EmptyCompactionOutput) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.comparator = test::BytewiseComparatorWithU64TsWrapper();
+  DestroyAndReopen(options);
+
+  std::string ts_str = Timestamp(1);
+  WriteOptions wopts;
+  ASSERT_OK(
+      db_->DeleteRange(wopts, db_->DefaultColumnFamily(), "k1", "k3", ts_str));
+  ASSERT_OK(Flush());
+
+  ts_str = Timestamp(3);
+  Slice ts = ts_str;
+  CompactRangeOptions cro;
+  // range tombstone will be dropped during compaction
+  cro.full_history_ts_low = &ts;
+  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+  ASSERT_OK(db_->CompactRange(cro, nullptr, nullptr));
+}
 #endif  // !ROCKSDB_LITE
 
 }  // namespace ROCKSDB_NAMESPACE

db/dbformat.cc

@@ -150,6 +150,31 @@ int InternalKeyComparator::Compare(const ParsedInternalKey& a,
   return r;
 }
 
+int InternalKeyComparator::Compare(const Slice& a,
+                                   const ParsedInternalKey& b) const {
+  // Order by:
+  //    increasing user key (according to user-supplied comparator)
+  //    decreasing sequence number
+  //    decreasing type (though sequence# should be enough to disambiguate)
+  int r = user_comparator_.Compare(ExtractUserKey(a), b.user_key);
+  if (r == 0) {
+    const uint64_t anum =
+        DecodeFixed64(a.data() + a.size() - kNumInternalBytes);
+    const uint64_t bnum = (b.sequence << 8) | b.type;
+    if (anum > bnum) {
+      r = -1;
+    } else if (anum < bnum) {
+      r = +1;
+    }
+  }
+  return r;
+}
+
+int InternalKeyComparator::Compare(const ParsedInternalKey& a,
+                                   const Slice& b) const {
+  return -Compare(b, a);
+}
+
 LookupKey::LookupKey(const Slice& _user_key, SequenceNumber s,
                      const Slice* ts) {
   size_t usize = _user_key.size();

db/dbformat.h

@@ -283,6 +283,8 @@ class InternalKeyComparator
 
   int Compare(const InternalKey& a, const InternalKey& b) const;
   int Compare(const ParsedInternalKey& a, const ParsedInternalKey& b) const;
+  int Compare(const Slice& a, const ParsedInternalKey& b) const;
+  int Compare(const ParsedInternalKey& a, const Slice& b) const;
   // In this `Compare()` overload, the sequence numbers provided in
   // `a_global_seqno` and `b_global_seqno` override the sequence numbers in `a`
   // and `b`, respectively. To disable sequence number override(s), provide the

db/experimental.cc

@@ -112,8 +112,9 @@ Status UpdateManifestForFilesState(
                   lf->smallest, lf->largest, lf->fd.smallest_seqno,
                   lf->fd.largest_seqno, lf->marked_for_compaction, temp,
                   lf->oldest_blob_file_number, lf->oldest_ancester_time,
-                  lf->file_creation_time, lf->file_checksum,
-                  lf->file_checksum_func_name, lf->unique_id);
+                  lf->file_creation_time, lf->epoch_number, lf->file_checksum,
+                  lf->file_checksum_func_name, lf->unique_id,
+                  lf->compensated_range_deletion_size);
             }
           }
         } else {

db/external_sst_file_basic_test.cc

@@ -694,6 +694,7 @@ TEST_P(ExternalSSTFileBasicTest, IngestFileWithGlobalSeqnoPickedSeqno) {
   bool verify_checksums_before_ingest = std::get<1>(GetParam());
   do {
     Options options = CurrentOptions();
+    options.disable_auto_compactions = true;
     DestroyAndReopen(options);
     std::map<std::string, std::string> true_data;
 
@@ -800,6 +801,7 @@ TEST_P(ExternalSSTFileBasicTest, IngestFileWithMultipleValueType) {
   bool verify_checksums_before_ingest = std::get<1>(GetParam());
   do {
     Options options = CurrentOptions();
+    options.disable_auto_compactions = true;
     options.merge_operator.reset(new TestPutOperator());
     DestroyAndReopen(options);
     std::map<std::string, std::string> true_data;
@@ -927,6 +929,7 @@ TEST_P(ExternalSSTFileBasicTest, IngestFileWithMixedValueType) {
   bool verify_checksums_before_ingest = std::get<1>(GetParam());
   do {
     Options options = CurrentOptions();
+    options.disable_auto_compactions = true;
     options.merge_operator.reset(new TestPutOperator());
     DestroyAndReopen(options);
     std::map<std::string, std::string> true_data;

db/external_sst_file_ingestion_job.cc

@@ -469,14 +469,90 @@ Status ExternalSstFileIngestionJob::Run() {
         f.fd.GetNumber(), f.fd.GetPathId(), f.fd.GetFileSize(),
         f.smallest_internal_key, f.largest_internal_key, f.assigned_seqno,
         f.assigned_seqno, false, f.file_temperature, kInvalidBlobFileNumber,
-        oldest_ancester_time, current_time, f.file_checksum,
-        f.file_checksum_func_name, f.unique_id);
+        oldest_ancester_time, current_time,
+        ingestion_options_.ingest_behind
+            ? kReservedEpochNumberForFileIngestedBehind
+            : cfd_->NewEpochNumber(),
+        f.file_checksum, f.file_checksum_func_name, f.unique_id, 0);
     f_metadata.temperature = f.file_temperature;
     edit_.AddFile(f.picked_level, f_metadata);
   }
+
+  CreateEquivalentFileIngestingCompactions();
   return status;
 }
 
+void ExternalSstFileIngestionJob::CreateEquivalentFileIngestingCompactions() {
+  // A map from output level to input of compactions equivalent to this
+  // ingestion job.
+  // TODO: simplify below logic to creating compaction per ingested file
+  // instead of per output level, once we figure out how to treat ingested files
+  // with adjacent range deletion tombstones to same output level in the same
+  // job as non-overlapping compactions.
+  std::map<int, CompactionInputFiles>
+      output_level_to_file_ingesting_compaction_input;
+
+  for (const auto& pair : edit_.GetNewFiles()) {
+    int output_level = pair.first;
+    const FileMetaData& f_metadata = pair.second;
+
+    CompactionInputFiles& input =
+        output_level_to_file_ingesting_compaction_input[output_level];
+    if (input.files.empty()) {
+      // Treat the source level of ingested files to be level 0
+      input.level = 0;
+    }
+
+    compaction_input_metdatas_.push_back(new FileMetaData(f_metadata));
+    input.files.push_back(compaction_input_metdatas_.back());
+  }
+
+  for (const auto& pair : output_level_to_file_ingesting_compaction_input) {
+    int output_level = pair.first;
+    const CompactionInputFiles& input = pair.second;
+
+    const auto& mutable_cf_options = *(cfd_->GetLatestMutableCFOptions());
+    file_ingesting_compactions_.push_back(new Compaction(
+        cfd_->current()->storage_info(), *cfd_->ioptions(), mutable_cf_options,
+        mutable_db_options_, {input}, output_level,
+        MaxFileSizeForLevel(
+            mutable_cf_options, output_level,
+            cfd_->ioptions()->compaction_style) /* output file size
+            limit,
+                                                 * not applicable
+                                                 */
+        ,
+        LLONG_MAX /* max compaction bytes, not applicable */,
+        0 /* output path ID, not applicable */, mutable_cf_options.compression,
+        mutable_cf_options.compression_opts, Temperature::kUnknown,
+        0 /* max_subcompaction, not applicable */,
+        {} /* grandparents, not applicable */, false /* is manual */,
+        "" /* trim_ts */, -1 /* score, not applicable */,
+        false /* is deletion compaction, not applicable */,
+        files_overlap_ /* l0_files_might_overlap, not applicable */,
+        CompactionReason::kExternalSstIngestion));
+  }
+}
+
+void ExternalSstFileIngestionJob::RegisterRange() {
+  for (const auto& c : file_ingesting_compactions_) {
+    cfd_->compaction_picker()->RegisterCompaction(c);
+  }
+}
+
+void ExternalSstFileIngestionJob::UnregisterRange() {
+  for (const auto& c : file_ingesting_compactions_) {
+    cfd_->compaction_picker()->UnregisterCompaction(c);
+    delete c;
+  }
+  file_ingesting_compactions_.clear();
+
+  for (const auto& f : compaction_input_metdatas_) {
+    delete f;
+  }
+  compaction_input_metdatas_.clear();
+}
+
 void ExternalSstFileIngestionJob::UpdateStats() {
   // Update internal stats for new ingested files
   uint64_t total_keys = 0;
@@ -795,8 +871,16 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
     if (lvl > 0 && lvl < vstorage->base_level()) {
       continue;
     }
-
-    if (vstorage->NumLevelFiles(lvl) > 0) {
+    if (cfd_->RangeOverlapWithCompaction(
+            file_to_ingest->smallest_internal_key.user_key(),
+            file_to_ingest->largest_internal_key.user_key(), lvl)) {
+      // We must use L0 or any level higher than `lvl` to be able to overwrite
+      // the compaction output keys that we overlap with in this level, We also
+      // need to assign this file a seqno to overwrite the compaction output
+      // keys in level `lvl`
+      overlap_with_db = true;
+      break;
+    } else if (vstorage->NumLevelFiles(lvl) > 0) {
       bool overlap_with_level = false;
       status = sv->current->OverlapWithLevelIterator(
           ro, env_options_, file_to_ingest->smallest_internal_key.user_key(),
@@ -853,6 +937,7 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
       target_level < cfd_->NumberLevels() - 1) {
     status = Status::TryAgain(
         "Files cannot be ingested to Lmax. Please make sure key range of Lmax "
+        "and ongoing compaction's output to Lmax"
         "does not overlap with files to ingest.");
     return status;
   }
@@ -870,7 +955,7 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
 Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
     IngestedFileInfo* file_to_ingest) {
   auto* vstorage = cfd_->current()->storage_info();
-  // first check if new files fit in the bottommost level
+  // First, check if new files fit in the bottommost level
   int bottom_lvl = cfd_->NumberLevels() - 1;
   if (!IngestedFileFitInLevel(file_to_ingest, bottom_lvl)) {
     return Status::InvalidArgument(
@@ -878,7 +963,7 @@ Status ExternalSstFileIngestionJob::CheckLevelForIngestedBehindFile(
         "at the bottommost level!");
   }
 
-  // second check if despite allow_ingest_behind=true we still have 0 seqnums
+  // Second, check if despite allow_ingest_behind=true we still have 0 seqnums
   // at some upper level
   for (int lvl = 0; lvl < cfd_->NumberLevels() - 1; lvl++) {
     for (auto file : vstorage->LevelFiles(lvl)) {
@@ -994,14 +1079,8 @@ bool ExternalSstFileIngestionJob::IngestedFileFitInLevel(
     // add it to this level
     return false;
   }
-  if (cfd_->RangeOverlapWithCompaction(file_smallest_user_key,
-                                       file_largest_user_key, level)) {
-    // File overlap with a running compaction output that will be stored
-    // in this level, we cannot add this file to this level
-    return false;
-  }
 
-  // File did not overlap with level files, our compaction output
+  // File did not overlap with level files, nor compaction output
   return true;
 }
 

db/external_sst_file_ingestion_job.h

@@ -11,6 +11,7 @@
 #include "db/column_family.h"
 #include "db/internal_stats.h"
 #include "db/snapshot_impl.h"
+#include "db/version_edit.h"
 #include "env/file_system_tracer.h"
 #include "logging/event_logger.h"
 #include "options/db_options.h"
@@ -78,7 +79,8 @@ class ExternalSstFileIngestionJob {
  public:
   ExternalSstFileIngestionJob(
       VersionSet* versions, ColumnFamilyData* cfd,
-      const ImmutableDBOptions& db_options, const EnvOptions& env_options,
+      const ImmutableDBOptions& db_options,
+      const MutableDBOptions& mutable_db_options, const EnvOptions& env_options,
       SnapshotList* db_snapshots,
       const IngestExternalFileOptions& ingestion_options,
       Directories* directories, EventLogger* event_logger,
@@ -88,6 +90,7 @@ class ExternalSstFileIngestionJob {
         versions_(versions),
         cfd_(cfd),
         db_options_(db_options),
+        mutable_db_options_(mutable_db_options),
         env_options_(env_options),
         db_snapshots_(db_snapshots),
         ingestion_options_(ingestion_options),
@@ -99,6 +102,17 @@ class ExternalSstFileIngestionJob {
     assert(directories != nullptr);
   }
 
+  ~ExternalSstFileIngestionJob() {
+    for (const auto& c : file_ingesting_compactions_) {
+      cfd_->compaction_picker()->UnregisterCompaction(c);
+      delete c;
+    }
+
+    for (const auto& f : compaction_input_metdatas_) {
+      delete f;
+    }
+  }
+
   // Prepare the job by copying external files into the DB.
   Status Prepare(const std::vector<std::string>& external_files_paths,
                  const std::vector<std::string>& files_checksums,
@@ -120,6 +134,15 @@ class ExternalSstFileIngestionJob {
   // REQUIRES: Mutex held
   Status Run();
 
+  // Register key range involved in this ingestion job
+  // to prevent key range conflict with other ongoing compaction/file ingestion
+  // REQUIRES: Mutex held
+  void RegisterRange();
+
+  // Unregister key range registered for this ingestion job
+  // REQUIRES: Mutex held
+  void UnregisterRange();
+
   // Update column family stats.
   // REQUIRES: Mutex held
   void UpdateStats();
@@ -175,11 +198,17 @@ class ExternalSstFileIngestionJob {
   template <typename TWritableFile>
   Status SyncIngestedFile(TWritableFile* file);
 
+  // Create equivalent `Compaction` objects to this file ingestion job
+  // , which will be used to check range conflict with other ongoing
+  // compactions.
+  void CreateEquivalentFileIngestingCompactions();
+
   SystemClock* clock_;
   FileSystemPtr fs_;
   VersionSet* versions_;
   ColumnFamilyData* cfd_;
   const ImmutableDBOptions& db_options_;
+  const MutableDBOptions& mutable_db_options_;
   const EnvOptions& env_options_;
   SnapshotList* db_snapshots_;
   autovector<IngestedFileInfo> files_to_ingest_;
@@ -196,6 +225,14 @@ class ExternalSstFileIngestionJob {
   // file_checksum_gen_factory is set, DB will generate checksum each file.
   bool need_generate_file_checksum_{true};
   std::shared_ptr<IOTracer> io_tracer_;
+
+  // Below are variables used in (un)registering range for this ingestion job
+  //
+  // FileMetaData used in inputs of compactions equivalent to this ingestion
+  // job
+  std::vector<FileMetaData*> compaction_input_metdatas_;
+  // Compactions equivalent to this ingestion job
+  std::vector<Compaction*> file_ingesting_compactions_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

db/external_sst_file_test.cc

@@ -973,7 +973,7 @@ TEST_F(ExternalSSTFileTest, MultiThreaded) {
 
   do {
     Options options = CurrentOptions();
-
+    options.disable_auto_compactions = true;
     std::atomic<int> thread_num(0);
     std::function<void()> write_file_func = [&]() {
       int file_idx = thread_num.fetch_add(1);
@@ -1249,8 +1249,9 @@ TEST_P(ExternalSSTFileTest, PickedLevel) {
 
   // This file overlaps with file 0 (L3), file 1 (L2) and the
   // output of compaction going to L1
-  ASSERT_OK(GenerateAndAddExternalFile(options, {4, 7}, -1, false, false, true,
-                                       false, false, &true_data));
+  ASSERT_OK(GenerateAndAddExternalFile(options, {4, 7}, -1,
+                                       true /* allow_global_seqno */, false,
+                                       true, false, false, &true_data));
   EXPECT_EQ(FilesPerLevel(), "5,0,1,1");
 
   // This file does not overlap with any file or with the running compaction
@@ -1270,106 +1271,6 @@ TEST_P(ExternalSSTFileTest, PickedLevel) {
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
 
-TEST_F(ExternalSSTFileTest, PickedLevelBug) {
-  env_->skip_fsync_ = true;
-  Options options = CurrentOptions();
-  options.disable_auto_compactions = false;
-  options.level0_file_num_compaction_trigger = 3;
-  options.num_levels = 2;
-  DestroyAndReopen(options);
-
-  std::vector<int> file_keys;
-
-  // file #1 in L0
-  file_keys = {0, 5, 7};
-  for (int k : file_keys) {
-    ASSERT_OK(Put(Key(k), Key(k)));
-  }
-  ASSERT_OK(Flush());
-
-  // file #2 in L0
-  file_keys = {4, 6, 8, 9};
-  for (int k : file_keys) {
-    ASSERT_OK(Put(Key(k), Key(k)));
-  }
-  ASSERT_OK(Flush());
-
-  // We have 2 overlapping files in L0
-  EXPECT_EQ(FilesPerLevel(), "2");
-
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
-      {{"DBImpl::IngestExternalFile:AfterIncIngestFileCounter",
-        "ExternalSSTFileTest::PickedLevelBug:0"},
-       {"ExternalSSTFileTest::PickedLevelBug:1", "DBImpl::AddFile:MutexUnlock"},
-       {"ExternalSSTFileTest::PickedLevelBug:2",
-        "DBImpl::RunManualCompaction:0"},
-       {"ExternalSSTFileTest::PickedLevelBug:3",
-        "DBImpl::RunManualCompaction:1"}});
-
-  std::atomic<bool> bg_compact_started(false);
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BackgroundCompaction:Start",
-      [&](void* /*arg*/) { bg_compact_started.store(true); });
-
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
-
-  Status bg_compact_status;
-  Status bg_addfile_status;
-
-  {
-    // While writing the MANIFEST start a thread that will ask for compaction
-    ThreadGuard bg_compact(port::Thread([&]() {
-      bg_compact_status =
-          db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-    }));
-    TEST_SYNC_POINT("ExternalSSTFileTest::PickedLevelBug:2");
-
-    // Start a thread that will ingest a new file
-    ThreadGuard bg_addfile(port::Thread([&]() {
-      file_keys = {1, 2, 3};
-      bg_addfile_status = GenerateAndAddExternalFile(options, file_keys, 1);
-    }));
-
-    // Wait for AddFile to start picking levels and writing MANIFEST
-    TEST_SYNC_POINT("ExternalSSTFileTest::PickedLevelBug:0");
-
-    TEST_SYNC_POINT("ExternalSSTFileTest::PickedLevelBug:3");
-
-    // We need to verify that no compactions can run while AddFile is
-    // ingesting the files into the levels it find suitable. So we will
-    // wait for 2 seconds to give a chance for compactions to run during
-    // this period, and then make sure that no compactions where able to run
-    env_->SleepForMicroseconds(1000000 * 2);
-    bool bg_compact_started_tmp = bg_compact_started.load();
-
-    // Hold AddFile from finishing writing the MANIFEST
-    TEST_SYNC_POINT("ExternalSSTFileTest::PickedLevelBug:1");
-
-    // check the status at the end, so even if the ASSERT fails the threads
-    // could be joined and return.
-    ASSERT_FALSE(bg_compact_started_tmp);
-  }
-
-  ASSERT_OK(bg_addfile_status);
-  ASSERT_OK(bg_compact_status);
-
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-
-  int total_keys = 0;
-  Iterator* iter = db_->NewIterator(ReadOptions());
-  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    ASSERT_OK(iter->status());
-    total_keys++;
-  }
-  ASSERT_EQ(total_keys, 10);
-
-  delete iter;
-
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-}
-
 TEST_F(ExternalSSTFileTest, IngestNonExistingFile) {
   Options options = CurrentOptions();
   DestroyAndReopen(options);
@@ -1420,7 +1321,8 @@ TEST_F(ExternalSSTFileTest, CompactDuringAddFileRandom) {
   int range_id = 0;
   std::vector<int> file_keys;
   std::function<void()> bg_addfile = [&]() {
-    ASSERT_OK(GenerateAndAddExternalFile(options, file_keys, range_id));
+    ASSERT_OK(GenerateAndAddExternalFile(options, file_keys, range_id,
+                                         true /* allow_global_seqno */));
   };
 
   const int num_of_ranges = 1000;
@@ -1503,8 +1405,9 @@ TEST_F(ExternalSSTFileTest, PickedLevelDynamic) {
 
   // This file overlaps with the output of the compaction (going to L3)
   // so the file will be added to L0 since L3 is the base level
-  ASSERT_OK(GenerateAndAddExternalFile(options, {31, 32, 33, 34}, -1, false,
-                                       false, true, false, false, &true_data));
+  ASSERT_OK(GenerateAndAddExternalFile(options, {31, 32, 33, 34}, -1,
+                                       true /* allow_global_seqno */, false,
+                                       true, false, false, &true_data));
   EXPECT_EQ(FilesPerLevel(), "5");
 
   // This file does not overlap with the current running compactiong
@@ -1642,14 +1545,15 @@ TEST_F(ExternalSSTFileTest, AddFileTrivialMoveBug) {
 
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
       "CompactionJob::Run():Start", [&](void* /*arg*/) {
-        // fit in L3 but will overlap with compaction so will be added
-        // to L2 but a compaction will trivially move it to L3
-        // and break LSM consistency
+        // Fit in L3 but will overlap with the compaction output so will be
+        // added to L2. Prior to the fix, a compaction will then trivially move
+        // this file to L3 and break LSM consistency
         static std::atomic<bool> called = {false};
         if (!called) {
           called = true;
           ASSERT_OK(dbfull()->SetOptions({{"max_bytes_for_level_base", "1"}}));
-          ASSERT_OK(GenerateAndAddExternalFile(options, {15, 16}, 7));
+          ASSERT_OK(GenerateAndAddExternalFile(options, {15, 16}, 7,
+                                               true /* allow_global_seqno */));
         }
       });
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();

db/flush_job.cc

@@ -23,6 +23,7 @@
 #include "db/memtable_list.h"
 #include "db/merge_context.h"
 #include "db/range_tombstone_fragmenter.h"
+#include "db/version_edit.h"
 #include "db/version_set.h"
 #include "file/file_util.h"
 #include "file/filename.h"
@@ -90,7 +91,7 @@ FlushJob::FlushJob(
     std::vector<SequenceNumber> existing_snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
     SnapshotChecker* snapshot_checker, JobContext* job_context,
-    LogBuffer* log_buffer, FSDirectory* db_directory,
+    FlushReason flush_reason, LogBuffer* log_buffer, FSDirectory* db_directory,
     FSDirectory* output_file_directory, CompressionType output_compression,
     Statistics* stats, EventLogger* event_logger, bool measure_io_stats,
     const bool sync_output_directory, const bool write_manifest,
@@ -113,6 +114,7 @@ FlushJob::FlushJob(
       earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
       snapshot_checker_(snapshot_checker),
       job_context_(job_context),
+      flush_reason_(flush_reason),
       log_buffer_(log_buffer),
       db_directory_(db_directory),
       output_file_directory_(output_file_directory),
@@ -199,6 +201,7 @@ void FlushJob::PickMemTable() {
 
   // path 0 for level 0 file.
   meta_.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0);
+  meta_.epoch_number = cfd_->NewEpochNumber();
 
   base_ = cfd_->current();
   base_->Ref();  // it is likely that we do not need this reference
@@ -243,9 +246,8 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker, FileMetaData* file_meta,
   }
   Status mempurge_s = Status::NotFound("No MemPurge.");
   if ((mempurge_threshold > 0.0) &&
-      (cfd_->GetFlushReason() == FlushReason::kWriteBufferFull) &&
-      (!mems_.empty()) && MemPurgeDecider(mempurge_threshold) &&
-      !(db_options_.atomic_flush)) {
+      (flush_reason_ == FlushReason::kWriteBufferFull) && (!mems_.empty()) &&
+      MemPurgeDecider(mempurge_threshold) && !(db_options_.atomic_flush)) {
     cfd_->SetMempurgeUsed();
     mempurge_s = MemPurge();
     if (!mempurge_s.ok()) {
@@ -876,7 +878,7 @@ Status FlushJob::WriteLevel0Table() {
                          << total_num_deletes << "total_data_size"
                          << total_data_size << "memory_usage"
                          << total_memory_usage << "flush_reason"
-                         << GetFlushReasonString(cfd_->GetFlushReason());
+                         << GetFlushReasonString(flush_reason_);
 
     {
       ScopedArenaIterator iter(
@@ -939,7 +941,7 @@ Status FlushJob::WriteLevel0Table() {
           cfd_->internal_stats(), &io_s, io_tracer_,
           BlobFileCreationReason::kFlush, seqno_to_time_mapping_, event_logger_,
           job_context_->job_id, io_priority, &table_properties_, write_hint,
-          full_history_ts_low, blob_callback_, &num_input_entries,
+          full_history_ts_low, blob_callback_, base_, &num_input_entries,
           &memtable_payload_bytes, &memtable_garbage_bytes);
       // TODO: Cleanup io_status in BuildTable and table builders
       assert(!s.ok() || io_s.ok());
@@ -999,9 +1001,9 @@ Status FlushJob::WriteLevel0Table() {
                    meta_.fd.smallest_seqno, meta_.fd.largest_seqno,
                    meta_.marked_for_compaction, meta_.temperature,
                    meta_.oldest_blob_file_number, meta_.oldest_ancester_time,
-                   meta_.file_creation_time, meta_.file_checksum,
-                   meta_.file_checksum_func_name, meta_.unique_id);
-
+                   meta_.file_creation_time, meta_.epoch_number,
+                   meta_.file_checksum, meta_.file_checksum_func_name,
+                   meta_.unique_id, meta_.compensated_range_deletion_size);
     edit_->SetBlobFileAdditions(std::move(blob_file_additions));
   }
 #ifndef ROCKSDB_LITE
@@ -1074,7 +1076,7 @@ std::unique_ptr<FlushJobInfo> FlushJob::GetFlushJobInfo() const {
   info->smallest_seqno = meta_.fd.smallest_seqno;
   info->largest_seqno = meta_.fd.largest_seqno;
   info->table_properties = table_properties_;
-  info->flush_reason = cfd_->GetFlushReason();
+  info->flush_reason = flush_reason_;
   info->blob_compression_type = mutable_cf_options_.blob_compression_type;
 
   // Update BlobFilesInfo.

db/flush_job.h

@@ -67,8 +67,8 @@ class FlushJob {
            std::vector<SequenceNumber> existing_snapshots,
            SequenceNumber earliest_write_conflict_snapshot,
            SnapshotChecker* snapshot_checker, JobContext* job_context,
-           LogBuffer* log_buffer, FSDirectory* db_directory,
-           FSDirectory* output_file_directory,
+           FlushReason flush_reason, LogBuffer* log_buffer,
+           FSDirectory* db_directory, FSDirectory* output_file_directory,
            CompressionType output_compression, Statistics* stats,
            EventLogger* event_logger, bool measure_io_stats,
            const bool sync_output_directory, const bool write_manifest,
@@ -150,6 +150,7 @@ class FlushJob {
   SequenceNumber earliest_write_conflict_snapshot_;
   SnapshotChecker* snapshot_checker_;
   JobContext* job_context_;
+  FlushReason flush_reason_;
   LogBuffer* log_buffer_;
   FSDirectory* db_directory_;
   FSDirectory* output_file_directory_;

db/flush_job_test.cc

@@ -164,15 +164,15 @@ TEST_F(FlushJobTest, Empty) {
   auto cfd = versions_->GetColumnFamilySet()->GetDefault();
   EventLogger event_logger(db_options_.info_log.get());
   SnapshotChecker* snapshot_checker = nullptr;  // not relavant
-  FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
-                     db_options_, *cfd->GetLatestMutableCFOptions(),
-                     std::numeric_limits<uint64_t>::max() /* memtable_id */,
-                     env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     {}, kMaxSequenceNumber, snapshot_checker, &job_context,
-                     nullptr, nullptr, nullptr, kNoCompression, nullptr,
-                     &event_logger, false, true /* sync_output_directory */,
-                     true /* write_manifest */, Env::Priority::USER,
-                     nullptr /*IOTracer*/, empty_seqno_to_time_mapping_);
+  FlushJob flush_job(
+      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
+      *cfd->GetLatestMutableCFOptions(),
+      std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
+      versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
+      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
+      nullptr, kNoCompression, nullptr, &event_logger, false,
+      true /* sync_output_directory */, true /* write_manifest */,
+      Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_);
   {
     InstrumentedMutexLock l(&mutex_);
     flush_job.PickMemTable();
@@ -255,9 +255,9 @@ TEST_F(FlushJobTest, NonEmpty) {
       *cfd->GetLatestMutableCFOptions(),
       std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
       versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
-      snapshot_checker, &job_context, nullptr, nullptr, nullptr, kNoCompression,
-      db_options_.statistics.get(), &event_logger, true,
-      true /* sync_output_directory */, true /* write_manifest */,
+      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
+      nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
+      true, true /* sync_output_directory */, true /* write_manifest */,
       Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_);
 
   HistogramData hist;
@@ -318,9 +318,9 @@ TEST_F(FlushJobTest, FlushMemTablesSingleColumnFamily) {
       dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
       *cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_,
       versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
-      snapshot_checker, &job_context, nullptr, nullptr, nullptr, kNoCompression,
-      db_options_.statistics.get(), &event_logger, true,
-      true /* sync_output_directory */, true /* write_manifest */,
+      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
+      nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
+      true, true /* sync_output_directory */, true /* write_manifest */,
       Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_);
   HistogramData hist;
   FileMetaData file_meta;
@@ -391,8 +391,8 @@ TEST_F(FlushJobTest, FlushMemtablesMultipleColumnFamilies) {
         dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
         memtable_ids[k], env_options_, versions_.get(), &mutex_,
         &shutting_down_, snapshot_seqs, kMaxSequenceNumber, snapshot_checker,
-        &job_context, nullptr, nullptr, nullptr, kNoCompression,
-        db_options_.statistics.get(), &event_logger, true,
+        &job_context, FlushReason::kTest, nullptr, nullptr, nullptr,
+        kNoCompression, db_options_.statistics.get(), &event_logger, true,
         false /* sync_output_directory */, false /* write_manifest */,
         Env::Priority::USER, nullptr /*IOTracer*/,
         empty_seqno_to_time_mapping_));
@@ -520,9 +520,9 @@ TEST_F(FlushJobTest, Snapshots) {
       *cfd->GetLatestMutableCFOptions(),
       std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
       versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
-      snapshot_checker, &job_context, nullptr, nullptr, nullptr, kNoCompression,
-      db_options_.statistics.get(), &event_logger, true,
-      true /* sync_output_directory */, true /* write_manifest */,
+      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
+      nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
+      true, true /* sync_output_directory */, true /* write_manifest */,
       Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_);
   mutex_.Lock();
   flush_job.PickMemTable();
@@ -576,9 +576,9 @@ TEST_F(FlushJobTest, GetRateLimiterPriorityForWrite) {
       dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
       *cfd->GetLatestMutableCFOptions(), flush_memtable_id, env_options_,
       versions_.get(), &mutex_, &shutting_down_, {}, kMaxSequenceNumber,
-      snapshot_checker, &job_context, nullptr, nullptr, nullptr, kNoCompression,
-      db_options_.statistics.get(), &event_logger, true,
-      true /* sync_output_directory */, true /* write_manifest */,
+      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
+      nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
+      true, true /* sync_output_directory */, true /* write_manifest */,
       Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_);
 
   // When the state from WriteController is normal.
@@ -656,9 +656,9 @@ TEST_F(FlushJobTimestampTest, AllKeysExpired) {
       dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
       std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
       versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
-      snapshot_checker, &job_context, nullptr, nullptr, nullptr, kNoCompression,
-      db_options_.statistics.get(), &event_logger, true,
-      true /* sync_output_directory */, true /* write_manifest */,
+      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
+      nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
+      true, true /* sync_output_directory */, true /* write_manifest */,
       Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_,
       /*db_id=*/"",
       /*db_session_id=*/"", full_history_ts_low);
@@ -709,9 +709,9 @@ TEST_F(FlushJobTimestampTest, NoKeyExpired) {
       dbname_, cfd, db_options_, *cfd->GetLatestMutableCFOptions(),
       std::numeric_limits<uint64_t>::max() /* memtable_id */, env_options_,
       versions_.get(), &mutex_, &shutting_down_, snapshots, kMaxSequenceNumber,
-      snapshot_checker, &job_context, nullptr, nullptr, nullptr, kNoCompression,
-      db_options_.statistics.get(), &event_logger, true,
-      true /* sync_output_directory */, true /* write_manifest */,
+      snapshot_checker, &job_context, FlushReason::kTest, nullptr, nullptr,
+      nullptr, kNoCompression, db_options_.statistics.get(), &event_logger,
+      true, true /* sync_output_directory */, true /* write_manifest */,
       Env::Priority::USER, nullptr /*IOTracer*/, empty_seqno_to_time_mapping_,
       /*db_id=*/"",
       /*db_session_id=*/"", full_history_ts_low);