Add CircleCI gadget

CircleCI is stably running. Need to add a gadget. Also since Circle builds some Windows and Linux, rename Travis and Appveyor builds to their names.
Test CircleCI with CLANG-10 (#7025 )
2026-11-06 02:52:35 -05:00 · 2020-06-24 17:06:33 -07:00 · 2020-06-24 16:22:49 -07:00 · 2020-06-24 13:40:48 -07:00 · 2020-06-24 11:05:24 -07:00 · 2020-06-23 16:27:20 -07:00
171 changed files with 5136 additions and 1456 deletions
@@ -11,7 +11,73 @@ executors:
      shell: bash.exe

 jobs:
-  build:
+  build-linux:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: pyenv global 3.5.2
+      - run: sudo apt-get update -y
+      - run: sudo apt-get install -y libgflags-dev
+      - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 make J=32 all check -j32
+
+  build-linux-release:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: make release -j32
+
+  build-linux-lite:
+    machine:
+      image: ubuntu-1604:201903-01 
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: pyenv global 3.5.2
+      - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 LITE=1 make J=32 all check -j32
+
+  build-linux-lite-release:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: large
+    steps:
+      - checkout # check out the code in the project directory
+      - run: make release -j32
+
+  build-linux-clang-no-test:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: sudo apt-get update -y
+      - run: sudo apt-get install -y clang
+      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make all -j32
+
+  build-linux-clang10-no-test:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: sudo sh -c 'echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" >> /etc/apt/sources.list'
+      - run: sudo sh -c 'echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" >> /etc/apt/sources.list'
+      - run: sudo apt-get update -y
+      - run: sudo apt-get install -y clang-10
+      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make all -j32 # aligned new doesn't work for reason we haven't figured out
+
+  build-linux-cmake:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: 2xlarge
+    steps:
+      - checkout # check out the code in the project directory
+      - run: mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make -j32
+
+  build-windows:
    executor: windows-2xlarge
    
    environment:
@@ -54,3 +120,29 @@ jobs:
          shell: powershell.exe
          command: |
            build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,env_basic_test,env_test,db_merge_operand_test -Concurrency 16
+
+workflows:
+  build-linux:
+    jobs:
+      - build-linux
+  build-linux-lite:
+    jobs:
+      - build-linux-lite
+  build-linux-release:
+    jobs:
+      - build-linux-release
+  build-linux-lite-release:
+    jobs:
+      - build-linux-lite-release
+  build-linux-clang-no-test:
+    jobs:
+      - build-linux-clang-no-test
+  build-linux-clang10-no-test:
+    jobs:
+      - build-linux-clang10-no-test
+  build-linux-cmake:
+    jobs:
+      - build-linux-cmake
+  build-windows:
+    jobs:
+      - build-windows
@@ -95,7 +95,7 @@ if(MSVC)
  option(WITH_XPRESS "build with windows built in compression" OFF)
  include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
 else()
-  if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+  if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
    # FreeBSD has jemalloc as default malloc
    # but it does not have all the jemalloc files in include/...
    set(WITH_JEMALLOC ON)
@@ -448,6 +448,8 @@ elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
  add_definitions(-DOS_LINUX)
 elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")
  add_definitions(-DOS_SOLARIS)
+elseif(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
+  add_definitions(-DOS_GNU_KFREEBSD)
 elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
  add_definitions(-DOS_FREEBSD)
 elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
@@ -539,6 +541,9 @@ set(SOURCES
        db/blob/blob_file_addition.cc
        db/blob/blob_file_garbage.cc
        db/blob/blob_file_meta.cc
+        db/blob/blob_log_format.cc
+        db/blob/blob_log_reader.cc
+        db/blob/blob_log_writer.cc
        db/builder.cc
        db/c.cc
        db/column_family.cc
@@ -730,9 +735,6 @@ set(SOURCES
        utilities/blob_db/blob_db_impl_filesnapshot.cc
        utilities/blob_db/blob_dump_tool.cc
        utilities/blob_db/blob_file.cc
-        utilities/blob_db/blob_log_reader.cc
-        utilities/blob_db/blob_log_writer.cc
-        utilities/blob_db/blob_log_format.cc
        utilities/cassandra/cassandra_compaction_filter.cc
        utilities/cassandra/format.cc
        utilities/cassandra/merge_operator.cc
@@ -974,12 +976,19 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
  )
 endif()

+option(WITH_ALL_TESTS "Build all test, rather than a small subset" ON)
+
 if(WITH_TESTS)
  add_subdirectory(third-party/gtest-1.8.1/fused-src/gtest)
  add_library(testharness STATIC
  test_util/testharness.cc)
  target_link_libraries(testharness gtest)
  set(TESTS
+        db/db_basic_test.cc
+        env/env_basic_test.cc
+  )
+  if(WITH_ALL_TESTS)
+    list(APPEND TESTS
        cache/cache_test.cc
        cache/lru_cache_test.cc
        db/blob/blob_file_addition_test.cc
@@ -994,7 +1003,6 @@ if(WITH_TESTS)
        db/comparator_db_test.cc
        db/corruption_test.cc
        db/cuckoo_table_db_test.cc
-        db/db_basic_test.cc
        db/db_with_timestamp_basic_test.cc
        db/db_block_cache_test.cc
        db/db_bloom_filter_test.cc
@@ -1057,7 +1065,6 @@ if(WITH_TESTS)
        db/write_batch_test.cc
        db/write_callback_test.cc
        db/write_controller_test.cc
-        env/env_basic_test.cc
        env/env_test.cc
        env/io_posix_test.cc
        env/mock_env_test.cc
@@ -1090,6 +1097,7 @@ if(WITH_TESTS)
        table/sst_file_reader_test.cc
        table/table_test.cc
        table/block_fetcher_test.cc
+        test_util/testutil_test.cc
        tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc
        tools/ldb_cmd_test.cc
        tools/reduce_levels_test.cc
@@ -1139,7 +1147,8 @@ if(WITH_TESTS)
        utilities/transactions/write_unprepared_transaction_test.cc
        utilities/ttl/ttl_test.cc
        utilities/write_batch_with_index/write_batch_with_index_test.cc
-  )
+    )
+  endif()
  if(WITH_LIBRADOS)
    list(APPEND TESTS utilities/env_librados_test.cc)
  endif()
@@ -1260,3 +1269,8 @@ if(WITH_TOOLS)
  add_custom_target(tools
    DEPENDS ${tool_deps})
 endif()
+
+option(WITH_EXAMPLES "build with examples" OFF)
+if(WITH_EXAMPLES)
+  add_subdirectory(examples)
+endif()
@@ -1,8 +1,25 @@
 # Rocksdb Change Log
 ## Unreleased
-### Behavior Changes
-* Disable delete triggered compaction (NewCompactOnDeletionCollectorFactory) in universal compaction mode and num_levels = 1 in order to avoid a corruption bug.
+### Public API Change
+* Encryption file classes now exposed for inheritance in env_encryption.h

+### Behavior Changes
+* Best-efforts recovery ignores CURRENT file completely. If CURRENT file is missing during recovery, best-efforts recovery still proceeds with MANIFEST file(s).
+* In best-efforts recovery, an error that is not Corruption or IOError::kNotFound or IOError::kPathNotFound will be overwritten silently. Fix this by checking all non-ok cases and return early.
+* Compressed block cache was automatically disabled with read-only DBs by mistake. Now it is fixed: compressed block cache will be in effective with read-only DB too.
+* Fix a bug of wrong iterator result if another thread finishes an update and a DB flush between two statement.
+
+### Public API Change
+* `DB::GetDbSessionId(std::string& session_id)` is added. `session_id` stores a unique identifier that gets reset every time the DB is opened. This DB session ID should be unique among all open DB instances on all hosts, and should be unique among re-openings of the same or other DBs. This identifier is recorded in the LOG file on the line starting with "DB Session ID:".
+
+### New Features
+* DB identity (`db_id`) and DB session identity (`db_session_id`) are added to table properties and stored in SST files. SST files generated from SstFileWriter and Repairer have DB identity “SST Writer” and “DB Repairer”, respectively. Their DB session IDs are generated in the same way as `DB::GetDbSessionId`. The session ID for SstFileWriter (resp., Repairer) resets every time `SstFileWriter::Open` (resp., `Repairer::Run`) is called.
+* Added experimental option BlockBasedTableOptions::optimize_filters_for_memory for reducing allocated memory size of Bloom filters (~10% savings with Jemalloc) while preserving the same general accuracy. To have an effect, the option requires format_version=5 and malloc_usable_size. Enabling this option is forward and backward compatible with existing format_version=5.
+
+### Bug Fixes
+* Fail recovery and report once hitting a physical log record checksum mismatch, while reading MANIFEST. RocksDB should not continue processing the MANIFEST any further.
+
+## 6.11 (6/12/2020)
 ### Bug Fixes
 * Fix consistency checking error swallowing in some cases when options.force_consistency_checks = true.
 * Fix possible false NotFound status from batched MultiGet using index type kHashSearch.
@@ -12,6 +29,9 @@
 * Fix corrupt key read from ingested file when iterator direction switches from reverse to forward at a key that is a prefix of another key in the same file. It is only possible in files with a non-zero global seqno.
 * Fix abnormally large estimate from GetApproximateSizes when a range starts near the end of one SST file and near the beginning of another. Now GetApproximateSizes consistently and fairly includes the size of SST metadata in addition to data blocks, attributing metadata proportionally among the data blocks based on their size.
 * Fix potential file descriptor leakage in PosixEnv's IsDirectory() and NewRandomAccessFile().
+* Fix false negative from the VerifyChecksum() API when there is a checksum mismatch in an index partition block in a BlockBasedTable format table file (index_type is kTwoLevelIndexSearch).
+* Fix sst_dump to return non-zero exit code if the specified file is not a recognized SST file or fails requested checks.
+* Fix incorrect results from batched MultiGet for duplicate keys, when the duplicate key matches the largest key of an SST file and the value type for the key in the file is a merge value.

 ### Public API Change
 * Flush(..., column_family) may return Status::ColumnFamilyDropped() instead of Status::InvalidArgument() if column_family is dropped while processing the flush request.
@@ -20,11 +40,19 @@
 * ldb now uses options.force_consistency_checks = true by default and "--disable_consistency_checks" is added to disable it.
 * DB::OpenForReadOnly no longer creates files or directories if the named DB does not exist, unless create_if_missing is set to true.
 * The consistency checks that validate LSM state changes (table file additions/deletions during flushes and compactions) are now stricter, more efficient, and no longer optional, i.e. they are performed even if `force_consistency_checks` is `false`.
+* Disable delete triggered compaction (NewCompactOnDeletionCollectorFactory) in universal compaction mode and num_levels = 1 in order to avoid a corruption bug.
+* `pin_l0_filter_and_index_blocks_in_cache` no longer applies to L0 files larger than `1.5 * write_buffer_size` to give more predictable memory usage. Such L0 files may exist due to intra-L0 compaction, external file ingestion, or user dynamically changing `write_buffer_size` (note, however, that files that are already pinned will continue being pinned, even after such a dynamic change).
+* In point-in-time wal recovery mode, fail database recovery in case of IOError while reading the WAL to avoid data loss.
+* A new method `Env::LowerThreadPoolCPUPriority(Priority, CpuPriority)` is added to `Env` to be able to lower to a specific priority such as `CpuPriority::kIdle`.

-### New Feature
+### New Features
 * sst_dump to add a new --readahead_size argument. Users can specify read size when scanning the data. Sst_dump also tries to prefetch tail part of the SST files so usually some number of I/Os are saved there too.
 * Generate file checksum in SstFileWriter if Options.file_checksum_gen_factory is set. The checksum and checksum function name are stored in ExternalSstFileInfo after the sst file write is finished.
 * Add a value_size_soft_limit in read options which limits the cumulative value size of keys read in batches in MultiGet. Once the cumulative value size of found keys exceeds read_options.value_size_soft_limit, all the remaining keys are returned with status Abort without further finding their values. By default the value_size_soft_limit is std::numeric_limits<uint64_t>::max().
+* Enable SST file ingestion with file checksum information when calling IngestExternalFiles(const std::vector<IngestExternalFileArg>& args). Added files_checksums and files_checksum_func_names to IngestExternalFileArg such that user can ingest the sst files with their file checksum information. Added verify_file_checksum to IngestExternalFileOptions (default is True). To be backward compatible, if DB does not enable file checksum or user does not provide checksum information (vectors of files_checksums and files_checksum_func_names are both empty), verification of file checksum is always sucessful. If DB enables file checksum, DB will always generate the checksum for each ingested SST file during Prepare stage of ingestion and store the checksum in Manifest, unless verify_file_checksum is False and checksum information is provided by the application. In this case, we only verify the checksum function name and directly store the ingested checksum in Manifest. If verify_file_checksum is set to True, DB will verify the ingested checksum and function name with the genrated ones. Any mismatch will fail the ingestion. Note that, if IngestExternalFileOptions::write_global_seqno is True, the seqno will be changed in the ingested file. Therefore, the checksum of the file will be changed. In this case, a new checksum will be generated after the seqno is updated and be stored in the Manifest.
+
+### Performance Improvements
+* Eliminate redundant key comparisons during random access in block-based tables.

 ## 6.10 (5/2/2020)
 ### Bug Fixes
@@ -639,6 +639,8 @@ TESTS = \
 	blob_file_garbage_test \
 	timer_test \
 	db_with_timestamp_compaction_test \
+	testutil_test \
+	io_tracer_test \

 PARALLEL_TEST = \
 	backupable_db_test \
@@ -673,9 +675,9 @@ ifdef COMPILE_WITH_UBSAN
        TESTS := $(shell echo $(TESTS) | sed 's/\boptions_settable_test\b//g')
 endif
 ifdef ASSERT_STATUS_CHECKED
-	# This is a new check for which we will add support incrementally. The
-	# whitelist can be removed once support is fully added.
-	TESTS_WHITELIST = \
+	# This is a new check for which we will add support incrementally. This
+	# list can be removed once support is fully added.
+	TESTS_PASSING_ASC = \
 		arena_test \
 		autovector_test \
 		blob_file_addition_test \
@@ -721,8 +723,8 @@ ifdef ASSERT_STATUS_CHECKED
 		work_queue_test \
 		write_controller_test \

-	TESTS := $(filter $(TESTS_WHITELIST),$(TESTS))
-	PARALLEL_TEST := $(filter $(TESTS_WHITELIST),$(PARALLEL_TEST))
+	TESTS := $(filter $(TESTS_PASSING_ASC),$(TESTS))
+	PARALLEL_TEST := $(filter $(TESTS_PASSING_ASC),$(PARALLEL_TEST))
 endif
 SUBSET := $(TESTS)
 ifdef ROCKSDBTESTS_START
@@ -843,7 +845,8 @@ endif  # PLATFORM_SHARED_EXT
 	dbg rocksdbjavastatic rocksdbjava install install-static install-shared uninstall \
 	analyze tools tools_lib \
 	blackbox_crash_test_with_atomic_flush whitebox_crash_test_with_atomic_flush  \
-	blackbox_crash_test_with_txn whitebox_crash_test_with_txn
+	blackbox_crash_test_with_txn whitebox_crash_test_with_txn \
+	blackbox_crash_test_with_best_efforts_recovery


 all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)
@@ -975,6 +978,12 @@ J ?= 100%
 # Use this regexp to select the subset of tests whose names match.
 tests-regexp = .

+ifeq ($(PRINT_PARALLEL_OUTPUTS), 1)
+	parallel_com = '{}'
+else
+	parallel_com = '{} >& t/log-{/}'
+endif
+
 .PHONY: check_0
 check_0:
 	$(AM_V_GEN)export TEST_TMPDIR=$(TMPD); \
@@ -988,9 +997,12 @@ check_0:
 	} \
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
-	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu '{} >& t/log-{/}'
+	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu  $(parallel_com) ; \
+	parallel_retcode=$$? ; \
+	awk '{ if ($$7 != 0 || $$8 != 0) { if ($$7 == "Exitval") { h = $$0; } else { if (!f) print h; print; f = 1 } } } END { if(f) exit 1; }' < LOG ; \
+	if [ $$parallel_retcode -ne 0 ] ; then exit 1 ; fi

-valgrind-blacklist-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized|MySQLStyleTransactionTest.TransactionStressTest
+valgrind-exclude-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized|MySQLStyleTransactionTest.TransactionStressTest

 .PHONY: valgrind_check_0
 valgrind_check_0:
@@ -1005,7 +1017,7 @@ valgrind_check_0:
 	}								\
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
-	  | grep -E -v '$(valgrind-blacklist-regexp)'					\
+	  | grep -E -v '$(valgrind-exclude-regexp)'					\
 	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu \
 	  '(if [[ "{}" == "./"* ]] ; then $(DRIVER) {}; else {}; fi) ' \
 	  '>& t/valgrind_log-{/}'
@@ -1048,8 +1060,10 @@ ifndef ASSERT_STATUS_CHECKED # not yet working with these tests
 endif
 endif
 endif
+ifndef SKIP_FORMAT_BUCK_CHECKS
 	$(MAKE) check-format
 	$(MAKE) check-buck-targets
+endif

 # TODO add ldb_tests
 check_some: $(SUBSET)
@@ -1065,6 +1079,8 @@ crash_test_with_atomic_flush: whitebox_crash_test_with_atomic_flush blackbox_cra

 crash_test_with_txn: whitebox_crash_test_with_txn blackbox_crash_test_with_txn

+crash_test_with_best_efforts_recovery: blackbox_crash_test_with_best_efforts_recovery
+
 blackbox_crash_test: db_stress
 	$(PYTHON) -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
 	$(PYTHON) -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
@@ -1075,6 +1091,9 @@ blackbox_crash_test_with_atomic_flush: db_stress
 blackbox_crash_test_with_txn: db_stress
 	$(PYTHON) -u tools/db_crashtest.py --txn blackbox $(CRASH_TEST_EXT_ARGS)

+blackbox_crash_test_with_best_efforts_recovery: db_stress
+	$(PYTHON) -u tools/db_crashtest.py --test_best_efforts_recovery blackbox $(CRASH_TEST_EXT_ARGS)
+
 ifeq ($(CRASH_TEST_KILL_ODD),)
  CRASH_TEST_KILL_ODD=888887
 endif
@@ -1113,6 +1132,11 @@ asan_crash_test_with_txn:
 	COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_txn
 	$(MAKE) clean

+asan_crash_test_with_best_efforts_recovery:
+	$(MAKE) clean
+	COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_best_efforts_recovery
+	$(MAKE) clean
+
 ubsan_check:
 	$(MAKE) clean
 	COMPILE_WITH_UBSAN=1 $(MAKE) check -j32
@@ -1133,6 +1157,11 @@ ubsan_crash_test_with_txn:
 	COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_txn
 	$(MAKE) clean

+ubsan_crash_test_with_best_efforts_recovery:
+	$(MAKE) clean
+	COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_best_efforts_recovery
+	$(MAKE) clean
+
 valgrind_test:
 	ROCKSDB_VALGRIND_RUN=1 DISABLE_JEMALLOC=1 $(MAKE) valgrind_check

@@ -1866,6 +1895,11 @@ blob_file_garbage_test: db/blob/blob_file_garbage_test.o $(LIBOBJECTS) $(TESTHAR
 timer_test: util/timer_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

+testutil_test: test_util/testutil_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+io_tracer_test: trace_replay/io_tracer_test.o trace_replay/io_tracer.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 #-------------------------------------------------
 # make install related stuff
 INSTALL_PATH ?= /usr/local
@@ -1,7 +1,8 @@
 ## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage

-[![Linux/Mac Build Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
-[![Windows Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
+[![CircleCI Status](https://circleci.com/gh/facebook/rocksdb.svg?style=svg)](https://circleci.com/gh/facebook/rocksdb)
+[![TravisCI Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
+[![Appveyor Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
 [![PPC64le Build Status](http://140.211.168.68:8080/buildStatus/icon?job=Rocksdb)](http://140.211.168.68:8080/job/Rocksdb)

 RocksDB is developed and maintained by Facebook Database Engineering Team.
@@ -78,7 +78,6 @@ ROCKSDB_PREPROCESSOR_FLAGS = [
    # Directories with files for #include
    "-I" + REPO_PATH + "include/",
    "-I" + REPO_PATH,
-    "-I" + REPO_PATH + "third-party/gtest-1.8.1/fused-src/",
 ]

 ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {
@@ -125,6 +124,9 @@ cpp_library(
        "db/blob/blob_file_addition.cc",
        "db/blob/blob_file_garbage.cc",
        "db/blob/blob_file_meta.cc",
+        "db/blob/blob_log_format.cc",
+        "db/blob/blob_log_reader.cc",
+        "db/blob/blob_log_writer.cc",
        "db/builder.cc",
        "db/c.cc",
        "db/column_family.cc",
@@ -292,6 +294,7 @@ cpp_library(
        "tools/ldb_tool.cc",
        "tools/sst_dump_tool.cc",
        "trace_replay/block_cache_tracer.cc",
+        "trace_replay/io_tracer.cc",
        "trace_replay/trace_replay.cc",
        "util/build_version.cc",
        "util/coding.cc",
@@ -319,9 +322,6 @@ cpp_library(
        "utilities/blob_db/blob_db_impl_filesnapshot.cc",
        "utilities/blob_db/blob_dump_tool.cc",
        "utilities/blob_db/blob_file.cc",
-        "utilities/blob_db/blob_log_format.cc",
-        "utilities/blob_db/blob_log_reader.cc",
-        "utilities/blob_db/blob_log_writer.cc",
        "utilities/cassandra/cassandra_compaction_filter.cc",
        "utilities/cassandra/format.cc",
        "utilities/cassandra/merge_operator.cc",
@@ -398,11 +398,10 @@ cpp_library(
    os_deps = ROCKSDB_OS_DEPS,
    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    deps = [
-        ":rocksdb_lib",
-        ":rocksdb_third_party_gtest",
+    deps = [":rocksdb_lib"],
+    external_deps = ROCKSDB_EXTERNAL_DEPS + [
+        ("googletest", None, "gtest"),
    ],
-    external_deps = ROCKSDB_EXTERNAL_DEPS,
 )

 cpp_library(
@@ -449,19 +448,6 @@ cpp_library(
    external_deps = ROCKSDB_EXTERNAL_DEPS,
 )

-cpp_library(
-    name = "rocksdb_third_party_gtest",
-    srcs = ["third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc"],
-    headers = ["third-party/gtest-1.8.1/fused-src/gtest/gtest.h"],
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    os_deps = ROCKSDB_OS_DEPS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    deps = [],
-    external_deps = ROCKSDB_EXTERNAL_DEPS,
-)
-
 cpp_library(
    name = "env_basic_test_lib",
    srcs = ["env/env_basic_test.cc"],
@@ -1170,6 +1156,13 @@ ROCKS_TESTS = [
        [],
        [],
    ],
+    [
+        "io_tracer_test",
+        "trace_replay/io_tracer_test.cc",
+        "serial",
+        [],
+        [],
+    ],
    [
        "iostats_context_test",
        "monitoring/iostats_context_test.cc",
@@ -1485,6 +1478,13 @@ ROCKS_TESTS = [
        [],
        [],
    ],
+    [
+        "testutil_test",
+        "test_util/testutil_test.cc",
+        "serial",
+        [],
+        [],
+    ],
    [
        "thread_list_test",
        "util/thread_list_test.cc",
@@ -1638,18 +1638,17 @@ ROCKS_TESTS = [
 # Do not build the tests in opt mode, since SyncPoint and other test code
 # will not be included.
 [
-    test_binary(
-        extra_compiler_flags = extra_compiler_flags,
-        extra_deps = extra_deps,
-        parallelism = parallelism,
-        rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-        rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
-        rocksdb_external_deps = ROCKSDB_EXTERNAL_DEPS,
-        rocksdb_os_deps = ROCKSDB_OS_DEPS,
-        rocksdb_os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-        rocksdb_preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-        test_cc = test_cc,
-        test_name = test_name,
+    cpp_unittest(
+        name = test_name,
+        srcs = [test_cc],
+        arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
+        os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
+        compiler_flags = ROCKSDB_COMPILER_FLAGS,
+        preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
+        deps = [":rocksdb_test_lib"] + extra_deps,
+        external_deps = ROCKSDB_EXTERNAL_DEPS + [
+            ("googletest", None, "gtest"),
+        ],
    )
    for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
    if not is_opt_mode
@@ -68,7 +68,7 @@ before_build:
  - md %APPVEYOR_BUILD_FOLDER%\build
  - cd %APPVEYOR_BUILD_FOLDER%\build
  - if DEFINED CMAKE_PLATEFORM_NAME (set "PLATEFORM_OPT=-A %CMAKE_PLATEFORM_NAME%")
-  - cmake .. -G "%CMAKE_GENERATOR%" %PLATEFORM_OPT% %CMAKE_OPT% -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1
+  - cmake .. -G "%CMAKE_GENERATOR%" %PLATEFORM_OPT% %CMAKE_OPT% -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1 -DWITH_ALL_TESTS=0
  - cd ..

 build:
@@ -79,7 +79,7 @@ build:
 test:

 test_script:
-  - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_with_timestamp_basic_test,db_test2,db_test,env_basic_test,env_test,db_merge_operand_test -Concurrency 8
+  - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,env_basic_test -Concurrency 8

 on_failure:
  - cmd: 7z a build-failed.zip %APPVEYOR_BUILD_FOLDER%\build\ && appveyor PushArtifact build-failed.zip
@@ -108,16 +108,6 @@ def get_tests(repo_path):

    return tests

-
-# Get gtest dir from Makefile
-def get_gtest_dir(repo_path):
-    for line in open(repo_path + "/Makefile"):
-        if line.strip().startswith("GTEST_DIR ="):
-            return line.split("=")[1].strip()
-    # if not found
-    exit_with_error("Unable to find GTEST_DIR in Makefile")
-
-
 # Parse extra dependencies passed by user from command line
 def get_dependencies():
    deps_map = {
@@ -151,13 +141,11 @@ def generate_targets(repo_path, deps_map):
    cc_files = get_cc_files(repo_path)
    # get tests from Makefile
    tests = get_tests(repo_path)
-    # get gtest dir
-    gtest_dir = get_gtest_dir(repo_path) + "/"

    if src_mk is None or cc_files is None or tests is None:
        return False

-    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path, gtest_dir)
+    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)

    # rocksdb_lib
    TARGETS.add_library(
@@ -171,7 +159,10 @@ def generate_targets(repo_path, deps_map):
        src_mk.get("TEST_LIB_SOURCES", []) +
        src_mk.get("EXP_LIB_SOURCES", []) +
        src_mk.get("ANALYZER_LIB_SOURCES", []),
-        [":rocksdb_lib", ":rocksdb_third_party_gtest"])
+        [":rocksdb_lib"],
+        extra_external_deps=""" + [
+        ("googletest", None, "gtest"),
+    ]""")
    # rocksdb_tools_lib
    TARGETS.add_library(
        "rocksdb_tools_lib",
@@ -185,12 +176,6 @@ def generate_targets(repo_path, deps_map):
        src_mk.get("ANALYZER_LIB_SOURCES", [])
        + src_mk.get('STRESS_LIB_SOURCES', [])
        + ["test_util/testutil.cc"])
-    # rocksdb_third_party_gtest
-    TARGETS.add_library(
-        "rocksdb_third_party_gtest",
-        [gtest_dir + "gtest/gtest-all.cc"],
-        [],
-        [gtest_dir + "gtest/gtest.h"])

    print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
    # test for every test we found in the Makefile
@@ -25,11 +25,10 @@ def pretty_list(lst, indent=8):


 class TARGETSBuilder(object):
-    def __init__(self, path, gtest_dir):
+    def __init__(self, path):
        self.path = path
        self.targets_file = open(path, 'w')
-        header = targets_cfg.rocksdb_target_header_template.format(
-            gtest_dir=gtest_dir)
+        header = targets_cfg.rocksdb_target_header_template
        self.targets_file.write(header)
        self.total_lib = 0
        self.total_bin = 0
@@ -39,7 +38,8 @@ class TARGETSBuilder(object):
    def __del__(self):
        self.targets_file.close()

-    def add_library(self, name, srcs, deps=None, headers=None):
+    def add_library(self, name, srcs, deps=None, headers=None,
+                    extra_external_deps=""):
        headers_attr_prefix = ""
        if headers is None:
            headers_attr_prefix = "auto_"
@@ -51,7 +51,8 @@ class TARGETSBuilder(object):
            srcs=pretty_list(srcs),
            headers_attr_prefix=headers_attr_prefix,
            headers=headers,
-            deps=pretty_list(deps)))
+            deps=pretty_list(deps),
+            extra_external_deps=extra_external_deps))
        self.total_lib = self.total_lib + 1

    def add_rocksdb_library(self, name, srcs, headers=None):
@@ -85,14 +85,13 @@ ROCKSDB_PREPROCESSOR_FLAGS = [
    # Directories with files for #include
    "-I" + REPO_PATH + "include/",
    "-I" + REPO_PATH,
-    "-I" + REPO_PATH + "{gtest_dir}",
 ]

-ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {{
+ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {
    "x86_64": [
        "-DHAVE_PCLMUL",
    ],
-}}
+}

 build_mode = read_config("fbcode", "build_mode")

@@ -134,7 +133,7 @@ cpp_library(
    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
    deps = [{deps}],
-    external_deps = ROCKSDB_EXTERNAL_DEPS,
+    external_deps = ROCKSDB_EXTERNAL_DEPS{extra_external_deps},
 )
 """

@@ -183,18 +182,17 @@ ROCKS_TESTS = [
 # Do not build the tests in opt mode, since SyncPoint and other test code
 # will not be included.
 [
-    test_binary(
-        extra_compiler_flags = extra_compiler_flags,
-        extra_deps = extra_deps,
-        parallelism = parallelism,
-        rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-        rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
-        rocksdb_external_deps = ROCKSDB_EXTERNAL_DEPS,
-        rocksdb_os_deps = ROCKSDB_OS_DEPS,
-        rocksdb_os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-        rocksdb_preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-        test_cc = test_cc,
-        test_name = test_name,
+    cpp_unittest(
+        name = test_name,
+        srcs = [test_cc],
+        arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
+        os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
+        compiler_flags = ROCKSDB_COMPILER_FLAGS,
+        preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
+        deps = [":rocksdb_test_lib"] + extra_deps,
+        external_deps = ROCKSDB_EXTERNAL_DEPS + [
+            ("googletest", None, "gtest"),
+        ],
    )
    for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
    if not is_opt_mode
@@ -192,6 +192,17 @@ EOF
        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
        # PORT_FILES=port/freebsd/freebsd_specific.cc
        ;;
+    GNU/kFreeBSD)
+        PLATFORM=OS_GNU_KFREEBSD
+        COMMON_FLAGS="$COMMON_FLAGS -DOS_GNU_KFREEBSD"
+        if [ -z "$USE_CLANG" ]; then
+            COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
+        else
+            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
+        fi
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
+        # PORT_FILES=port/gnu_kfreebsd/gnu_kfreebsd_specific.cc
+        ;;
    NetBSD)
        PLATFORM=OS_NETBSD
        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
@@ -56,8 +56,9 @@ Status ArenaWrappedDBIter::Refresh() {
  // TODO(yiwu): For last_seq_same_as_publish_seq_==false, this is not the
  // correct behavior. Will be corrected automatically when we take a snapshot
  // here for the case of WritePreparedTxnDB.
-  SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
  uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
+  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:1");
+  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:2");
  if (sv_number_ != cur_sv_number) {
    Env* env = db_iter_->env();
    db_iter_->~DBIter();
@@ -65,6 +66,7 @@ Status ArenaWrappedDBIter::Refresh() {
    new (&arena_) Arena();

    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_);
+    SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
    if (read_callback_) {
      read_callback_->Refresh(latest_seq);
    }
@@ -78,7 +80,7 @@ Status ArenaWrappedDBIter::Refresh() {
        latest_seq, /* allow_unprepared_value */ true);
    SetIterUnderDBIter(internal_iter);
  } else {
-    db_iter_->set_sequence(latest_seq);
+    db_iter_->set_sequence(db_impl_->GetLatestSequenceNumber());
    db_iter_->set_valid(false);
  }
  return Status::OK();
@@ -38,8 +38,15 @@ std::string BlobFileMetaData::DebugString() const {
 std::ostream& operator<<(std::ostream& os, const BlobFileMetaData& meta) {
  const auto& shared_meta = meta.GetSharedMeta();
  assert(shared_meta);
+  os << (*shared_meta);

-  os << (*shared_meta) << " garbage_blob_count: " << meta.GetGarbageBlobCount()
+  os << " linked_ssts: {";
+  for (uint64_t file_number : meta.GetLinkedSsts()) {
+    os << ' ' << file_number;
+  }
+  os << " }";
+
+  os << " garbage_blob_count: " << meta.GetGarbageBlobCount()
     << " garbage_blob_bytes: " << meta.GetGarbageBlobBytes();

  return os;
@@ -5,12 +5,13 @@

 #pragma once

-#include "rocksdb/rocksdb_namespace.h"
-
 #include <cassert>
 #include <iosfwd>
 #include <memory>
 #include <string>
+#include <unordered_set>
+
+#include "rocksdb/rocksdb_namespace.h"

 namespace ROCKSDB_NAMESPACE {

@@ -90,11 +91,15 @@ std::ostream& operator<<(std::ostream& os,

 class BlobFileMetaData {
 public:
+  using LinkedSsts = std::unordered_set<uint64_t>;
+
  static std::shared_ptr<BlobFileMetaData> Create(
      std::shared_ptr<SharedBlobFileMetaData> shared_meta,
-      uint64_t garbage_blob_count, uint64_t garbage_blob_bytes) {
-    return std::shared_ptr<BlobFileMetaData>(new BlobFileMetaData(
-        std::move(shared_meta), garbage_blob_count, garbage_blob_bytes));
+      LinkedSsts linked_ssts, uint64_t garbage_blob_count,
+      uint64_t garbage_blob_bytes) {
+    return std::shared_ptr<BlobFileMetaData>(
+        new BlobFileMetaData(std::move(shared_meta), std::move(linked_ssts),
+                             garbage_blob_count, garbage_blob_bytes));
  }

  BlobFileMetaData(const BlobFileMetaData&) = delete;
@@ -128,6 +133,8 @@ class BlobFileMetaData {
    return shared_meta_->GetChecksumValue();
  }

+  const LinkedSsts& GetLinkedSsts() const { return linked_ssts_; }
+
  uint64_t GetGarbageBlobCount() const { return garbage_blob_count_; }
  uint64_t GetGarbageBlobBytes() const { return garbage_blob_bytes_; }

@@ -135,8 +142,10 @@ class BlobFileMetaData {

 private:
  BlobFileMetaData(std::shared_ptr<SharedBlobFileMetaData> shared_meta,
-                   uint64_t garbage_blob_count, uint64_t garbage_blob_bytes)
+                   LinkedSsts linked_ssts, uint64_t garbage_blob_count,
+                   uint64_t garbage_blob_bytes)
      : shared_meta_(std::move(shared_meta)),
+        linked_ssts_(std::move(linked_ssts)),
        garbage_blob_count_(garbage_blob_count),
        garbage_blob_bytes_(garbage_blob_bytes) {
    assert(shared_meta_);
@@ -145,6 +154,7 @@ class BlobFileMetaData {
  }

  std::shared_ptr<SharedBlobFileMetaData> shared_meta_;
+  LinkedSsts linked_ssts_;
  uint64_t garbage_blob_count_;
  uint64_t garbage_blob_bytes_;
 };
@@ -5,7 +5,7 @@
 //
 #ifndef ROCKSDB_LITE

-#include "utilities/blob_db/blob_log_format.h"
+#include "db/blob/blob_log_format.h"

 #include "util/coding.h"
 #include "util/crc32c.h"
@@ -9,7 +9,6 @@

 #ifndef ROCKSDB_LITE

-#include <limits>
 #include <memory>
 #include <utility>

@@ -23,7 +22,6 @@ namespace blob_db {

 constexpr uint32_t kMagicNumber = 2395959;  // 0x00248f37
 constexpr uint32_t kVersion1 = 1;
-constexpr uint64_t kNoExpiration = std::numeric_limits<uint64_t>::max();

 using ExpirationRange = std::pair<uint64_t, uint64_t>;

@@ -5,7 +5,7 @@
 //
 #ifndef ROCKSDB_LITE

-#include "utilities/blob_db/blob_log_reader.h"
+#include "db/blob/blob_log_reader.h"

 #include <algorithm>

@@ -10,12 +10,12 @@
 #include <memory>
 #include <string>

+#include "db/blob/blob_log_format.h"
 #include "file/random_access_file_reader.h"
 #include "rocksdb/env.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/statistics.h"
 #include "rocksdb/status.h"
-#include "utilities/blob_db/blob_log_format.h"

 namespace ROCKSDB_NAMESPACE {

@@ -4,17 +4,17 @@
 //  (found in the LICENSE.Apache file in the root directory).
 #ifndef ROCKSDB_LITE

-#include "utilities/blob_db/blob_log_writer.h"
+#include "db/blob/blob_log_writer.h"

 #include <cstdint>
 #include <string>

+#include "db/blob/blob_log_format.h"
 #include "file/writable_file_writer.h"
 #include "monitoring/statistics.h"
 #include "rocksdb/env.h"
 #include "util/coding.h"
 #include "util/stop_watch.h"
-#include "utilities/blob_db/blob_log_format.h"

 namespace ROCKSDB_NAMESPACE {
 namespace blob_db {
@@ -10,12 +10,12 @@
 #include <memory>
 #include <string>

+#include "db/blob/blob_log_format.h"
 #include "rocksdb/env.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/statistics.h"
 #include "rocksdb/status.h"
 #include "rocksdb/types.h"
-#include "utilities/blob_db/blob_log_format.h"

 namespace ROCKSDB_NAMESPACE {

@@ -51,7 +51,8 @@ TableBuilder* NewTableBuilder(
    uint64_t sample_for_compression, const CompressionOptions& compression_opts,
    int level, const bool skip_filters, const uint64_t creation_time,
    const uint64_t oldest_key_time, const uint64_t target_file_size,
-    const uint64_t file_creation_time) {
+    const uint64_t file_creation_time, const std::string& db_id,
+    const std::string& db_session_id) {
  assert((column_family_id ==
          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
         column_family_name.empty());
@@ -61,7 +62,7 @@ TableBuilder* NewTableBuilder(
                          sample_for_compression, compression_opts,
                          skip_filters, column_family_name, level,
                          creation_time, oldest_key_time, target_file_size,
-                          file_creation_time),
+                          file_creation_time, db_id, db_session_id),
      column_family_id, file);
 }

@@ -85,7 +86,8 @@ Status BuildTable(
    EventLogger* event_logger, int job_id, const Env::IOPriority io_priority,
    TableProperties* table_properties, int level, const uint64_t creation_time,
    const uint64_t oldest_key_time, Env::WriteLifeTimeHint write_hint,
-    const uint64_t file_creation_time) {
+    const uint64_t file_creation_time, const std::string& db_id,
+    const std::string& db_session_id) {
  assert((column_family_id ==
          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
         column_family_name.empty());
@@ -142,7 +144,7 @@ Status BuildTable(
          column_family_name, file_writer.get(), compression,
          sample_for_compression, compression_opts_for_flush, level,
          false /* skip_filters */, creation_time, oldest_key_time,
-          0 /*target_file_size*/, file_creation_time);
+          0 /*target_file_size*/, file_creation_time, db_id, db_session_id);
    }

    MergeHelper merge(env, internal_comparator.user_comparator(),
@@ -239,7 +241,9 @@ Status BuildTable(
          (internal_stats == nullptr) ? nullptr
                                      : internal_stats->GetFileReadHist(0),
          TableReaderCaller::kFlush, /*arena=*/nullptr,
-          /*skip_filter=*/false, level, /*smallest_compaction_key=*/nullptr,
+          /*skip_filter=*/false, level,
+          MaxFileSizeForL0MetaPin(mutable_cf_options),
+          /*smallest_compaction_key=*/nullptr,
          /*largest_compaction_key*/ nullptr,
          /*allow_unprepared_value*/ false));
      s = it->status();
@@ -51,7 +51,8 @@ TableBuilder* NewTableBuilder(
    const CompressionOptions& compression_opts, int level,
    const bool skip_filters = false, const uint64_t creation_time = 0,
    const uint64_t oldest_key_time = 0, const uint64_t target_file_size = 0,
-    const uint64_t file_creation_time = 0);
+    const uint64_t file_creation_time = 0, const std::string& db_id = "",
+    const std::string& db_session_id = "");

 // Build a Table file from the contents of *iter.  The generated file
 // will be named according to number specified in meta. On success, the rest of
@@ -83,6 +84,7 @@ extern Status BuildTable(
    TableProperties* table_properties = nullptr, int level = -1,
    const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0,
    Env::WriteLifeTimeHint write_hint = Env::WLTH_NOT_SET,
-    const uint64_t file_creation_time = 0);
+    const uint64_t file_creation_time = 0, const std::string& db_id = "",
+    const std::string& db_session_id = "");

 }  // namespace ROCKSDB_NAMESPACE
@@ -305,12 +305,15 @@ CompactionJob::CompactionJob(
    const SnapshotChecker* snapshot_checker, std::shared_ptr<Cache> table_cache,
    EventLogger* event_logger, bool paranoid_file_checks, bool measure_io_stats,
    const std::string& dbname, CompactionJobStats* compaction_job_stats,
-    Env::Priority thread_pri, const std::atomic<bool>* manual_compaction_paused)
+    Env::Priority thread_pri, const std::atomic<bool>* manual_compaction_paused,
+    const std::string& db_id, const std::string& db_session_id)
    : job_id_(job_id),
      compact_(new CompactionState(compaction)),
      compaction_job_stats_(compaction_job_stats),
      compaction_stats_(compaction->compaction_reason(), 1),
      dbname_(dbname),
+      db_id_(db_id),
+      db_session_id_(db_session_id),
      db_options_(db_options),
      file_options_(file_options),
      env_(db_options.env),
@@ -654,6 +657,8 @@ Status CompactionJob::Run() {
                compact_->compaction->output_level()),
            TableReaderCaller::kCompactionRefill, /*arena=*/nullptr,
            /*skip_filters=*/false, compact_->compaction->output_level(),
+            MaxFileSizeForL0MetaPin(
+                *compact_->compaction->mutable_cf_options()),
            /*smallest_compaction_key=*/nullptr,
            /*largest_compaction_key=*/nullptr,
            /*allow_unprepared_value=*/false);
@@ -1552,7 +1557,8 @@ Status CompactionJob::OpenCompactionOutputFile(
      sub_compact->compaction->output_compression_opts(),
      sub_compact->compaction->output_level(), skip_filters,
      oldest_ancester_time, 0 /* oldest_key_time */,
-      sub_compact->compaction->max_output_file_size(), current_time));
+      sub_compact->compaction->max_output_file_size(), current_time, db_id_,
+      db_session_id_));
  LogFlush(db_options_.info_log);
  return s;
 }
@@ -78,7 +78,9 @@ class CompactionJob {
                const std::string& dbname,
                CompactionJobStats* compaction_job_stats,
                Env::Priority thread_pri,
-                const std::atomic<bool>* manual_compaction_paused = nullptr);
+                const std::atomic<bool>* manual_compaction_paused = nullptr,
+                const std::string& db_id = "",
+                const std::string& db_session_id = "");

  ~CompactionJob();

@@ -152,6 +154,8 @@ class CompactionJob {

  // DBImpl state
  const std::string& dbname_;
+  const std::string db_id_;
+  const std::string db_session_id_;
  const ImmutableDBOptions& db_options_;
  const FileOptions file_options_;

@@ -157,42 +157,6 @@ class CorruptionTest : public testing::Test {
    ASSERT_GE(max_expected, correct);
  }

-  void CorruptFile(const std::string& fname, int offset, int bytes_to_corrupt) {
-    struct stat sbuf;
-    if (stat(fname.c_str(), &sbuf) != 0) {
-      const char* msg = strerror(errno);
-      FAIL() << fname << ": " << msg;
-    }
-
-    if (offset < 0) {
-      // Relative to end of file; make it absolute
-      if (-offset > sbuf.st_size) {
-        offset = 0;
-      } else {
-        offset = static_cast<int>(sbuf.st_size + offset);
-      }
-    }
-    if (offset > sbuf.st_size) {
-      offset = static_cast<int>(sbuf.st_size);
-    }
-    if (offset + bytes_to_corrupt > sbuf.st_size) {
-      bytes_to_corrupt = static_cast<int>(sbuf.st_size - offset);
-    }
-
-    // Do it
-    std::string contents;
-    Status s = ReadFileToString(Env::Default(), fname, &contents);
-    ASSERT_TRUE(s.ok()) << s.ToString();
-    for (int i = 0; i < bytes_to_corrupt; i++) {
-      contents[i + offset] ^= 0x80;
-    }
-    s = WriteStringToFile(Env::Default(), contents, fname);
-    ASSERT_TRUE(s.ok()) << s.ToString();
-    Options options;
-    EnvOptions env_options;
-    ASSERT_NOK(VerifySstFileChecksum(options, env_options, fname));
-  }
-
  void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
    // Pick file to corrupt
    std::vector<std::string> filenames;
@@ -211,7 +175,7 @@ class CorruptionTest : public testing::Test {
    }
    ASSERT_TRUE(!fname.empty()) << filetype;

-    CorruptFile(fname, offset, bytes_to_corrupt);
+    test::CorruptFile(fname, offset, bytes_to_corrupt);
  }

  // corrupts exactly one file at level `level`. if no file found at level,
@@ -221,7 +185,7 @@ class CorruptionTest : public testing::Test {
    db_->GetLiveFilesMetaData(&metadata);
    for (const auto& m : metadata) {
      if (m.level == level) {
-        CorruptFile(dbname_ + "/" + m.name, offset, bytes_to_corrupt);
+        test::CorruptFile(dbname_ + "/" + m.name, offset, bytes_to_corrupt);
        return;
      }
    }
@@ -556,7 +520,7 @@ TEST_F(CorruptionTest, RangeDeletionCorrupted) {
      ImmutableCFOptions(options_), kRangeDelBlock, &range_del_handle));

  ASSERT_OK(TryReopen());
-  CorruptFile(filename, static_cast<int>(range_del_handle.offset()), 1);
+  test::CorruptFile(filename, static_cast<int>(range_del_handle.offset()), 1);
  ASSERT_TRUE(TryReopen().IsCorruption());
 }

@@ -9,6 +9,7 @@

 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
+#include "rocksdb/merge_operator.h"
 #include "rocksdb/perf_context.h"
 #include "rocksdb/utilities/debug.h"
 #include "table/block_based/block_based_table_reader.h"
@@ -17,6 +18,8 @@
 #if !defined(ROCKSDB_LITE)
 #include "test_util/sync_point.h"
 #endif
+#include "utilities/merge_operators.h"
+#include "utilities/merge_operators/string_append/stringappend.h"

 namespace ROCKSDB_NAMESPACE {

@@ -38,6 +41,56 @@ TEST_F(DBBasicTest, OpenWhenOpen) {
  delete db2;
 }

+TEST_F(DBBasicTest, UniqueSession) {
+  Options options = CurrentOptions();
+  std::string sid1, sid2, sid3, sid4;
+
+  db_->GetDbSessionId(sid1);
+  Reopen(options);
+  db_->GetDbSessionId(sid2);
+  ASSERT_OK(Put("foo", "v1"));
+  db_->GetDbSessionId(sid4);
+  Reopen(options);
+  db_->GetDbSessionId(sid3);
+
+  ASSERT_NE(sid1, sid2);
+  ASSERT_NE(sid1, sid3);
+  ASSERT_NE(sid2, sid3);
+
+  ASSERT_EQ(sid2, sid4);
+
+#ifndef ROCKSDB_LITE
+  Close();
+  ASSERT_OK(ReadOnlyReopen(options));
+  db_->GetDbSessionId(sid1);
+  // Test uniqueness between readonly open (sid1) and regular open (sid3)
+  ASSERT_NE(sid1, sid3);
+  Close();
+  ASSERT_OK(ReadOnlyReopen(options));
+  db_->GetDbSessionId(sid2);
+  ASSERT_EQ("v1", Get("foo"));
+  db_->GetDbSessionId(sid3);
+
+  ASSERT_NE(sid1, sid2);
+
+  ASSERT_EQ(sid2, sid3);
+#endif  // ROCKSDB_LITE
+
+  CreateAndReopenWithCF({"goku"}, options);
+  db_->GetDbSessionId(sid1);
+  ASSERT_OK(Put("bar", "e1"));
+  db_->GetDbSessionId(sid2);
+  ASSERT_EQ("e1", Get("bar"));
+  db_->GetDbSessionId(sid3);
+  ReopenWithColumnFamilies({"default", "goku"}, options);
+  db_->GetDbSessionId(sid4);
+
+  ASSERT_EQ(sid1, sid2);
+  ASSERT_EQ(sid2, sid3);
+
+  ASSERT_NE(sid1, sid4);
+}
+
 #ifndef ROCKSDB_LITE
 TEST_F(DBBasicTest, ReadOnlyDB) {
  ASSERT_OK(Put("foo", "v1"));
@@ -1340,6 +1393,57 @@ TEST_F(DBBasicTest, MultiGetBatchedSortedMultiFile) {
  } while (ChangeOptions());
 }

+TEST_F(DBBasicTest, MultiGetBatchedDuplicateKeys) {
+  Options opts = CurrentOptions();
+  opts.merge_operator = MergeOperators::CreateStringAppendOperator();
+  CreateAndReopenWithCF({"pikachu"}, opts);
+  SetPerfLevel(kEnableCount);
+  // To expand the power of this test, generate > 1 table file and
+  // mix with memtable
+  ASSERT_OK(Merge(1, "k1", "v1"));
+  ASSERT_OK(Merge(1, "k2", "v2"));
+  Flush(1);
+  MoveFilesToLevel(2, 1);
+  ASSERT_OK(Merge(1, "k3", "v3"));
+  ASSERT_OK(Merge(1, "k4", "v4"));
+  Flush(1);
+  MoveFilesToLevel(2, 1);
+  ASSERT_OK(Merge(1, "k4", "v4_2"));
+  ASSERT_OK(Merge(1, "k6", "v6"));
+  Flush(1);
+  MoveFilesToLevel(2, 1);
+  ASSERT_OK(Merge(1, "k7", "v7"));
+  ASSERT_OK(Merge(1, "k8", "v8"));
+  Flush(1);
+  MoveFilesToLevel(2, 1);
+
+  get_perf_context()->Reset();
+
+  std::vector<Slice> keys({"k8", "k8", "k8", "k4", "k4", "k1", "k3"});
+  std::vector<PinnableSlice> values(keys.size());
+  std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);
+  std::vector<Status> s(keys.size());
+
+  db_->MultiGet(ReadOptions(), handles_[1], keys.size(), keys.data(),
+                values.data(), s.data(), false);
+
+  ASSERT_EQ(values.size(), keys.size());
+  ASSERT_EQ(std::string(values[0].data(), values[0].size()), "v8");
+  ASSERT_EQ(std::string(values[1].data(), values[1].size()), "v8");
+  ASSERT_EQ(std::string(values[2].data(), values[2].size()), "v8");
+  ASSERT_EQ(std::string(values[3].data(), values[3].size()), "v4,v4_2");
+  ASSERT_EQ(std::string(values[4].data(), values[4].size()), "v4,v4_2");
+  ASSERT_EQ(std::string(values[5].data(), values[5].size()), "v1");
+  ASSERT_EQ(std::string(values[6].data(), values[6].size()), "v3");
+  ASSERT_EQ(24, (int)get_perf_context()->multiget_read_bytes);
+
+  for (Status& status : s) {
+    ASSERT_OK(status);
+  }
+
+  SetPerfLevel(kDisable);
+}
+
 TEST_F(DBBasicTest, MultiGetBatchedMultiLevel) {
  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
@@ -2145,6 +2249,35 @@ TEST_F(DBBasicTest, RecoverWithNoCurrentFile) {
  }
 }

+TEST_F(DBBasicTest, RecoverWithNoManifest) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("foo", "value"));
+  ASSERT_OK(Flush());
+  Close();
+  {
+    // Delete all MANIFEST.
+    std::vector<std::string> files;
+    ASSERT_OK(env_->GetChildren(dbname_, &files));
+    for (const auto& file : files) {
+      uint64_t number = 0;
+      FileType type = kLogFile;
+      if (ParseFileName(file, &number, &type) && type == kDescriptorFile) {
+        ASSERT_OK(env_->DeleteFile(dbname_ + "/" + file));
+      }
+    }
+  }
+  options.best_efforts_recovery = true;
+  options.create_if_missing = false;
+  Status s = TryReopen(options);
+  ASSERT_TRUE(s.IsInvalidArgument());
+  options.create_if_missing = true;
+  Reopen(options);
+  // Since no MANIFEST exists, best-efforts recovery creates a new, empty db.
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+}
+
 TEST_F(DBBasicTest, SkipWALIfMissingTableFiles) {
  Options options = CurrentOptions();
  DestroyAndReopen(options);
@@ -2185,6 +2318,33 @@ TEST_F(DBBasicTest, SkipWALIfMissingTableFiles) {
 }
 #endif  // !ROCKSDB_LITE

+TEST_F(DBBasicTest, ManifestChecksumMismatch) {
+  Options options = CurrentOptions();
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("bar", "value"));
+  ASSERT_OK(Flush());
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->SetCallBack(
+      "LogWriter::EmitPhysicalRecord:BeforeEncodeChecksum", [&](void* arg) {
+        auto* crc = reinterpret_cast<uint32_t*>(arg);
+        *crc = *crc + 1;
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  WriteOptions write_opts;
+  write_opts.disableWAL = true;
+  Status s = db_->Put(write_opts, "foo", "value");
+  ASSERT_OK(s);
+  ASSERT_OK(Flush());
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  ASSERT_OK(Put("foo", "value1"));
+  ASSERT_OK(Flush());
+  s = TryReopen(options);
+  ASSERT_TRUE(s.IsCorruption());
+}
+
 class DBBasicTestMultiGet : public DBTestBase {
 public:
  DBBasicTestMultiGet(std::string test_dir, int num_cfs, bool compressed_cache,
@@ -3184,6 +3184,15 @@ TEST_P(DBCompactionTestWithParam, IntraL0Compaction) {
  options.level0_file_num_compaction_trigger = 5;
  options.max_background_compactions = 2;
  options.max_subcompactions = max_subcompactions_;
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+  options.write_buffer_size = 2 << 20;  // 2MB
+
+  BlockBasedTableOptions table_options;
+  table_options.block_cache = NewLRUCache(64 << 20);  // 64MB
+  table_options.cache_index_and_filter_blocks = true;
+  table_options.pin_l0_filter_and_index_blocks_in_cache = true;
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
+
  DestroyAndReopen(options);

  const size_t kValueSize = 1 << 20;
@@ -3214,6 +3223,7 @@ TEST_P(DBCompactionTestWithParam, IntraL0Compaction) {
      ASSERT_OK(Put(Key(i + 1), value));
    }
    ASSERT_OK(Flush());
+    ASSERT_EQ(i + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
  }
  dbfull()->TEST_WaitForCompact();
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
@@ -3228,6 +3238,16 @@ TEST_P(DBCompactionTestWithParam, IntraL0Compaction) {
  for (int i = 0; i < 2; ++i) {
    ASSERT_GE(level_to_files[0][i].fd.file_size, 1 << 21);
  }
+
+  // The index/filter in the file produced by intra-L0 should not be pinned.
+  // That means clearing unref'd entries in block cache and re-accessing the
+  // file produced by intra-L0 should bump the index block miss count.
+  uint64_t prev_index_misses =
+      TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS);
+  table_options.block_cache->EraseUnRefEntries();
+  ASSERT_EQ("", Get(Key(0)));
+  ASSERT_EQ(prev_index_misses + 1,
+            TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
 }

 TEST_P(DBCompactionTestWithParam, IntraL0CompactionDoesNotObsoleteDeletions) {
@@ -5163,7 +5183,6 @@ TEST_P(DBCompactionTestWithParam,
  // Put one key, to make biggest log sequence number in this memtable is bigger
  // than sst which would be ingested in next step.
  ASSERT_OK(Put(Key(2), "b"));
-  ASSERT_EQ(10, NumTableFilesAtLevel(0));
  dbfull()->TEST_WaitForCompact();
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
  std::vector<std::vector<FileMetaData>> level_to_files;
@@ -249,7 +249,8 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
      new ColumnFamilyMemTablesImpl(versions_->GetColumnFamilySet()));

  DumpRocksDBBuildVersion(immutable_db_options_.info_log.get());
-  DumpDBFileSummary(immutable_db_options_, dbname_);
+  SetDbSessionId();
+  DumpDBFileSummary(immutable_db_options_, dbname_, db_session_id_);
  immutable_db_options_.Dump(immutable_db_options_.info_log.get());
  mutable_db_options_.Dump(immutable_db_options_.info_log.get());
  DumpSupportInfo(immutable_db_options_.info_log.get());
@@ -2670,7 +2671,8 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
        " guaranteed to be preserved, try larger iter_start_seqnum opt."));
  }
  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
-  auto cfd = cfh->cfd();
+  ColumnFamilyData* cfd = cfh->cfd();
+  assert(cfd != nullptr);
  ReadCallback* read_callback = nullptr;  // No read callback provided.
  if (read_options.tailing) {
 #ifdef ROCKSDB_LITE
@@ -2691,10 +2693,11 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
    // Note: no need to consider the special case of
    // last_seq_same_as_publish_seq_==false since NewIterator is overridden in
    // WritePreparedTxnDB
-    auto snapshot = read_options.snapshot != nullptr
-                        ? read_options.snapshot->GetSequenceNumber()
-                        : versions_->LastSequence();
-    result = NewIteratorImpl(read_options, cfd, snapshot, read_callback);
+    result = NewIteratorImpl(read_options, cfd,
+                             (read_options.snapshot != nullptr)
+                                 ? read_options.snapshot->GetSequenceNumber()
+                                 : kMaxSequenceNumber,
+                             read_callback);
  }
  return result;
 }
@@ -2707,6 +2710,24 @@ ArenaWrappedDBIter* DBImpl::NewIteratorImpl(const ReadOptions& read_options,
                                            bool allow_refresh) {
  SuperVersion* sv = cfd->GetReferencedSuperVersion(this);

+  TEST_SYNC_POINT("DBImpl::NewIterator:1");
+  TEST_SYNC_POINT("DBImpl::NewIterator:2");
+
+  if (snapshot == kMaxSequenceNumber) {
+    // Note that the snapshot is assigned AFTER referencing the super
+    // version because otherwise a flush happening in between may compact away
+    // data for the snapshot, so the reader would see neither data that was be
+    // visible to the snapshot before compaction nor the newer data inserted
+    // afterwards.
+    // Note that the super version might not contain all the data available
+    // to this snapshot, but in that case it can see all the data in the
+    // super version, which is a valid consistent state after the user
+    // calls NewIterator().
+    snapshot = versions_->LastSequence();
+    TEST_SYNC_POINT("DBImpl::NewIterator:3");
+    TEST_SYNC_POINT("DBImpl::NewIterator:4");
+  }
+
  // Try to generate a DB iterator tree in continuous memory area to be
  // cache friendly. Here is an example of result:
  // +-------------------------------+
@@ -3601,6 +3622,21 @@ Status DBImpl::GetDbIdentityFromIdentityFile(std::string* identity) const {
  return s;
 }

+Status DBImpl::GetDbSessionId(std::string& session_id) const {
+  session_id.assign(db_session_id_);
+  return Status::OK();
+}
+
+void DBImpl::SetDbSessionId() {
+  // GenerateUniqueId() generates an identifier
+  // that has a negligible probability of being duplicated
+  db_session_id_ = env_->GenerateUniqueId();
+  // Remove the extra '\n' at the end if there is one
+  if (!db_session_id_.empty() && db_session_id_.back() == '\n') {
+    db_session_id_.pop_back();
+  }
+}
+
 // Default implementation -- returns not supported status
 Status DB::CreateColumnFamily(const ColumnFamilyOptions& /*cf_options*/,
                              const std::string& /*column_family_name*/,
@@ -4180,7 +4216,8 @@ Status DBImpl::IngestExternalFiles(
        static_cast<ColumnFamilyHandleImpl*>(args[i].column_family)->cfd();
    SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
    exec_results[i].second = ingestion_jobs[i].Prepare(
-        args[i].external_files, start_file_number, super_version);
+        args[i].external_files, args[i].files_checksums,
+        args[i].files_checksum_func_names, start_file_number, super_version);
    exec_results[i].first = true;
    CleanupSuperVersion(super_version);
  }
@@ -4191,7 +4228,8 @@ Status DBImpl::IngestExternalFiles(
        static_cast<ColumnFamilyHandleImpl*>(args[0].column_family)->cfd();
    SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
    exec_results[0].second = ingestion_jobs[0].Prepare(
-        args[0].external_files, next_file_number, super_version);
+        args[0].external_files, args[0].files_checksums,
+        args[0].files_checksum_func_names, next_file_number, super_version);
    exec_results[0].first = true;
    CleanupSuperVersion(super_version);
  }
@@ -350,6 +350,8 @@ class DBImpl : public DB {

  virtual Status GetDbIdentityFromIdentityFile(std::string* identity) const;

+  virtual Status GetDbSessionId(std::string& session_id) const override;
+
  ColumnFamilyHandle* DefaultColumnFamily() const override;

  ColumnFamilyHandle* PersistentStatsColumnFamily() const;
@@ -477,6 +479,7 @@ class DBImpl : public DB {
  Status GetImpl(const ReadOptions& options, const Slice& key,
                 GetImplOptions& get_impl_options);

+  // If `snapshot` == kMaxSequenceNumber, set a recent one inside the file.
  ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& options,
                                      ColumnFamilyData* cfd,
                                      SequenceNumber snapshot,
@@ -980,6 +983,9 @@ class DBImpl : public DB {
 protected:
  const std::string dbname_;
  std::string db_id_;
+  // db_session_id_ is an identifier that gets reset
+  // every time the DB is opened
+  std::string db_session_id_;
  std::unique_ptr<VersionSet> versions_;
  // Flag to check whether we allocated and own the info log file
  bool own_info_log_;
@@ -1161,6 +1167,10 @@ class DBImpl : public DB {
  // bump up the version set's next_file_number_ to be 1 + largest_file_number.
  Status FinishBestEffortsRecovery();

+  // SetDbSessionId() should be called in the constuctor DBImpl()
+  // to ensure that db_session_id_ gets updated every time the DB is opened
+  void SetDbSessionId();
+
 private:
  friend class DB;
  friend class ErrorHandler;
@@ -154,7 +154,8 @@ Status DBImpl::FlushMemTableToOutputFile(
      GetDataDir(cfd, 0U),
      GetCompressionFlush(*cfd->ioptions(), mutable_cf_options), stats_,
      &event_logger_, mutable_cf_options.report_bg_io_stats,
-      true /* sync_output_directory */, true /* write_manifest */, thread_pri);
+      true /* sync_output_directory */, true /* write_manifest */, thread_pri,
+      db_id_, db_session_id_);
  FileMetaData file_meta;

  TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:BeforePickMemtables");
@@ -345,7 +346,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
        data_dir, GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
        stats_, &event_logger_, mutable_cf_options.report_bg_io_stats,
        false /* sync_output_directory */, false /* write_manifest */,
-        thread_pri));
+        thread_pri, db_id_, db_session_id_));
    jobs.back()->PickMemTable();
  }

@@ -1031,7 +1032,8 @@ Status DBImpl::CompactFilesImpl(
      snapshot_checker, table_cache_, &event_logger_,
      c->mutable_cf_options()->paranoid_file_checks,
      c->mutable_cf_options()->report_bg_io_stats, dbname_,
-      &compaction_job_stats, Env::Priority::USER, &manual_compaction_paused_);
+      &compaction_job_stats, Env::Priority::USER, &manual_compaction_paused_,
+      db_id_, db_session_id_);

  // Creating a compaction influences the compaction score because the score
  // takes running compactions into account (by skipping files that are already
@@ -2822,7 +2824,8 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
        &event_logger_, c->mutable_cf_options()->paranoid_file_checks,
        c->mutable_cf_options()->report_bg_io_stats, dbname_,
        &compaction_job_stats, thread_pri,
-        is_manual ? &manual_compaction_paused_ : nullptr);
+        is_manual ? &manual_compaction_paused_ : nullptr, db_id_,
+        db_session_id_);
    compaction_job.Prepare();

    NotifyOnCompactionBegin(c->column_family_data(), c.get(), status,
@@ -370,7 +370,30 @@ Status DBImpl::Recover(
    }

    std::string current_fname = CurrentFileName(dbname_);
-    s = env_->FileExists(current_fname);
+    // Path to any MANIFEST file in the db dir. It does not matter which one.
+    // Since best-efforts recovery ignores CURRENT file, existence of a
+    // MANIFEST indicates the recovery to recover existing db. If no MANIFEST
+    // can be found, a new db will be created.
+    std::string manifest_path;
+    if (!immutable_db_options_.best_efforts_recovery) {
+      s = env_->FileExists(current_fname);
+    } else {
+      s = Status::NotFound();
+      std::vector<std::string> files;
+      // No need to check return value
+      env_->GetChildren(dbname_, &files);
+      for (const std::string& file : files) {
+        uint64_t number = 0;
+        FileType type = kLogFile;  // initialize
+        if (ParseFileName(file, &number, &type) && type == kDescriptorFile) {
+          // Found MANIFEST (descriptor log), thus best-efforts recovery does
+          // not have to treat the db as empty.
+          s = Status::OK();
+          manifest_path = dbname_ + "/" + file;
+          break;
+        }
+      }
+    }
    if (s.IsNotFound()) {
      if (immutable_db_options_.create_if_missing) {
        s = NewDB();
@@ -398,14 +421,14 @@ Status DBImpl::Recover(
      FileOptions customized_fs(file_options_);
      customized_fs.use_direct_reads |=
          immutable_db_options_.use_direct_io_for_flush_and_compaction;
-      s = fs_->NewRandomAccessFile(current_fname, customized_fs, &idfile,
-                                   nullptr);
+      const std::string& fname =
+          manifest_path.empty() ? current_fname : manifest_path;
+      s = fs_->NewRandomAccessFile(fname, customized_fs, &idfile, nullptr);
      if (!s.ok()) {
        std::string error_str = s.ToString();
        // Check if unsupported Direct I/O is the root cause
        customized_fs.use_direct_reads = false;
-        s = fs_->NewRandomAccessFile(current_fname, customized_fs, &idfile,
-                                     nullptr);
+        s = fs_->NewRandomAccessFile(fname, customized_fs, &idfile, nullptr);
        if (s.ok()) {
          return Status::InvalidArgument(
              "Direct I/O is not supported by the specified DB.");
@@ -704,10 +727,10 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
    Status* status;  // nullptr if immutable_db_options_.paranoid_checks==false
    void Corruption(size_t bytes, const Status& s) override {
      ROCKS_LOG_WARN(info_log, "%s%s: dropping %d bytes; %s",
-                     (this->status == nullptr ? "(ignoring error) " : ""),
-                     fname, static_cast<int>(bytes), s.ToString().c_str());
-      if (this->status != nullptr && this->status->ok()) {
-        *this->status = s;
+                     (status == nullptr ? "(ignoring error) " : ""), fname,
+                     static_cast<int>(bytes), s.ToString().c_str());
+      if (status != nullptr && status->ok()) {
+        *status = s;
      }
    }
  };
@@ -830,6 +853,8 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
    Slice record;
    WriteBatch batch;

+    TEST_SYNC_POINT_CALLBACK("DBImpl::RecoverLogFiles:BeforeReadWal",
+                             /*arg=*/nullptr);
    while (!stop_replay_by_wal_filter &&
           reader.ReadRecord(&record, &scratch,
                             immutable_db_options_.wal_recovery_mode) &&
@@ -994,6 +1019,16 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
        status = Status::OK();
      } else if (immutable_db_options_.wal_recovery_mode ==
                 WALRecoveryMode::kPointInTimeRecovery) {
+        if (status.IsIOError()) {
+          ROCKS_LOG_ERROR(immutable_db_options_.info_log,
+                          "IOError during point-in-time reading log #%" PRIu64
+                          " seq #%" PRIu64
+                          ". %s. This likely mean loss of synced WAL, "
+                          "thus recovery fails.",
+                          log_number, *next_sequence,
+                          status.ToString().c_str());
+          return status;
+        }
        // We should ignore the error but not continue replaying
        status = Status::OK();
        stop_replay_for_corruption = true;
@@ -1254,7 +1289,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
          mutable_cf_options.compression_opts, paranoid_file_checks,
          cfd->internal_stats(), TableFileCreationReason::kRecovery, &io_s,
          &event_logger_, job_id, Env::IO_HIGH, nullptr /* table_properties */,
-          -1 /* level */, current_time, write_hint);
+          -1 /* level */, current_time, 0 /* oldest_key_time */, write_hint,
+          0 /* file_creation_time */, db_id_, db_session_id_);
      LogFlush(immutable_db_options_.info_log);
      ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                      "[%s] [WriteLevel0TableForRecovery]"
@@ -1569,6 +1605,12 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
  auto sfm = static_cast<SstFileManagerImpl*>(
      impl->immutable_db_options_.sst_file_manager.get());
  if (s.ok() && sfm) {
+    // Set Statistics ptr for SstFileManager to dump the stats of
+    // DeleteScheduler.
+    sfm->SetStatisticsPtr(impl->immutable_db_options_.statistics);
+    ROCKS_LOG_INFO(impl->immutable_db_options_.info_log,
+                   "SstFileManager instance %p", sfm);
+
    // Notify SstFileManager about all sst files that already exist in
    // db_paths[0] and cf_paths[0] when the DB is opened.

@@ -17,7 +17,8 @@
 namespace ROCKSDB_NAMESPACE {

 void DumpDBFileSummary(const ImmutableDBOptions& options,
-                       const std::string& dbname) {
+                       const std::string& dbname,
+                       const std::string& session_id) {
  if (options.info_log == nullptr) {
    return;
  }
@@ -32,6 +33,8 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
  std::string file_info, wal_info;

  Header(options.info_log, "DB SUMMARY\n");
+  Header(options.info_log, "DB Session ID:  %s\n", session_id.c_str());
+
  // Get files in dbname dir
  if (!env->GetChildren(dbname, &files).ok()) {
    Error(options.info_log,
@@ -10,5 +10,6 @@

 namespace ROCKSDB_NAMESPACE {
 void DumpDBFileSummary(const ImmutableDBOptions& options,
-                       const std::string& dbname);
+                       const std::string& dbname,
+                       const std::string& session_id = "");
 }  // namespace ROCKSDB_NAMESPACE
@@ -97,7 +97,8 @@ struct StressTestIterator : public InternalIterator {

  bool MaybeFail() {
    if (rnd->Next() >=
-        std::numeric_limits<uint64_t>::max() * error_probability) {
+        static_cast<double>(std::numeric_limits<uint64_t>::max()) *
+            error_probability) {
      return false;
    }
    if (rnd->Next() % 2) {
@@ -114,7 +115,8 @@ struct StressTestIterator : public InternalIterator {

  void MaybeMutate() {
    if (rnd->Next() >=
-        std::numeric_limits<uint64_t>::max() * mutation_probability) {
+        static_cast<double>(std::numeric_limits<uint64_t>::max()) *
+            mutation_probability) {
      return;
    }
    do {
@@ -126,8 +128,9 @@ struct StressTestIterator : public InternalIterator {
      if (data->hidden.empty()) {
        hide_probability = 1;
      }
-      bool do_hide =
-          rnd->Next() < std::numeric_limits<uint64_t>::max() * hide_probability;
+      bool do_hide = rnd->Next() <
+                     static_cast<double>(std::numeric_limits<uint64_t>::max()) *
+                         hide_probability;
      if (do_hide) {
        // Hide a random entry.
        size_t idx = rnd->Next() % data->entries.size();
@@ -303,14 +303,14 @@ TEST_F(DBMemTableTest, InsertWithHint) {
  ASSERT_EQ(hint_bar, rep->last_hint_in());
  ASSERT_EQ(hint_bar, rep->last_hint_out());
  ASSERT_EQ(5, rep->num_insert_with_hint());
-  ASSERT_OK(Put("whitelisted", "vvv"));
+  ASSERT_OK(Put("NotInPrefixDomain", "vvv"));
  ASSERT_EQ(5, rep->num_insert_with_hint());
  ASSERT_EQ("foo_v1", Get("foo_k1"));
  ASSERT_EQ("foo_v2", Get("foo_k2"));
  ASSERT_EQ("foo_v3", Get("foo_k3"));
  ASSERT_EQ("bar_v1", Get("bar_k1"));
  ASSERT_EQ("bar_v2", Get("bar_k2"));
-  ASSERT_EQ("vvv", Get("whitelisted"));
+  ASSERT_EQ("vvv", Get("NotInPrefixDomain"));
 }

 TEST_F(DBMemTableTest, ColumnFamilyId) {
@@ -39,7 +39,7 @@ class DBOptionsTest : public DBTestBase {
    std::unordered_map<std::string, std::string> options_map;
    StringToMap(options_str, &options_map);
    std::unordered_map<std::string, std::string> mutable_map;
-    for (const auto opt : db_options_type_info) {
+    for (const auto& opt : db_options_type_info) {
      if (opt.second.IsMutable() && opt.second.ShouldSerialize()) {
        mutable_map[opt.first] = options_map[opt.first];
      }
@@ -57,7 +57,7 @@ class DBOptionsTest : public DBTestBase {
    std::unordered_map<std::string, std::string> options_map;
    StringToMap(options_str, &options_map);
    std::unordered_map<std::string, std::string> mutable_map;
-    for (const auto opt : cf_options_type_info) {
+    for (const auto& opt : cf_options_type_info) {
      if (opt.second.IsMutable() && opt.second.ShouldSerialize()) {
        mutable_map[opt.first] = options_map[opt.first];
      }
@@ -376,6 +376,7 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
  env_->SetTimeElapseOnlySleep(&options);
  options.disable_auto_compactions = true;
  options.env = env_;
+  options.statistics = CreateDBStatistics();

  int64_t rate_bytes_per_sec = 1024 * 10;  // 10 Kbs / Sec
  Status s;
@@ -425,6 +426,9 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
  }
  ASSERT_GT(time_spent_deleting, expected_penlty * 0.9);
  ASSERT_LT(time_spent_deleting, expected_penlty * 1.1);
+  ASSERT_EQ(4, options.statistics->getAndResetTickerCount(FILES_MARKED_TRASH));
+  ASSERT_EQ(
+      0, options.statistics->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
@@ -816,6 +820,12 @@ TEST_F(DBSSTTest, CancellingCompactionsWorks) {
  ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0);
  // Make sure the stat is bumped
  ASSERT_GT(dbfull()->immutable_db_options().statistics.get()->getTickerCount(COMPACTION_CANCELLED), 0);
+  ASSERT_EQ(0,
+            dbfull()->immutable_db_options().statistics.get()->getTickerCount(
+                FILES_MARKED_TRASH));
+  ASSERT_EQ(4,
+            dbfull()->immutable_db_options().statistics.get()->getTickerCount(
+                FILES_DELETED_IMMEDIATELY));
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

@@ -252,6 +252,26 @@ TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) {
  }
 }

+TEST_F(DBTablePropertiesTest, GetDbIdentifiersProperty) {
+  CreateAndReopenWithCF({"goku"}, CurrentOptions());
+
+  for (uint32_t cf = 0; cf < 2; ++cf) {
+    Put(cf, "key", "val");
+    Put(cf, "foo", "bar");
+    Flush(cf);
+
+    TablePropertiesCollection fname_to_props;
+    ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props));
+    ASSERT_EQ(1U, fname_to_props.size());
+
+    std::string id, sid;
+    db_->GetDbIdentity(id);
+    db_->GetDbSessionId(sid);
+    ASSERT_EQ(id, fname_to_props.begin()->second->db_id);
+    ASSERT_EQ(sid, fname_to_props.begin()->second->db_session_id);
+  }
+}
+
 class DeletionTriggeredCompactionTestListener : public EventListener {
 public:
  void OnCompactionBegin(DB* , const CompactionJobInfo& ci) override {
@@ -3031,6 +3031,10 @@ class ModelDB : public DB {
    return Status::OK();
  }

+  Status GetDbSessionId(std::string& /*session_id*/) const override {
+    return Status::OK();
+  }
+
  SequenceNumber GetLatestSequenceNumber() const override { return 0; }

  bool SetPreserveDeletesSequenceNumber(SequenceNumber /*seqnum*/) override {
@@ -25,6 +25,7 @@ class DBTest2 : public DBTestBase {
  DBTest2() : DBTestBase("/db_test2") {}
 };

+#ifndef ROCKSDB_LITE
 TEST_F(DBTest2, OpenForReadOnly) {
  DB* db_ptr = nullptr;
  std::string dbname = test::PerThreadDBPath("db_readonly");
@@ -86,6 +87,65 @@ TEST_F(DBTest2, OpenForReadOnlyWithColumnFamilies) {
  ASSERT_NOK(env_->FileExists(dbname));
 }

+class TestReadOnlyWithCompressedCache
+    : public DBTestBase,
+      public testing::WithParamInterface<std::tuple<int, bool>> {
+ public:
+  TestReadOnlyWithCompressedCache()
+      : DBTestBase("/test_readonly_with_compressed_cache") {
+    max_open_files_ = std::get<0>(GetParam());
+    use_mmap_ = std::get<1>(GetParam());
+  }
+  int max_open_files_;
+  bool use_mmap_;
+};
+
+TEST_P(TestReadOnlyWithCompressedCache, ReadOnlyWithCompressedCache) {
+  if (use_mmap_ && !IsMemoryMappedAccessSupported()) {
+    return;
+  }
+  ASSERT_OK(Put("foo", "bar"));
+  ASSERT_OK(Put("foo2", "barbarbarbarbarbarbarbar"));
+  ASSERT_OK(Flush());
+
+  DB* db_ptr = nullptr;
+  Options options = CurrentOptions();
+  options.allow_mmap_reads = use_mmap_;
+  options.max_open_files = max_open_files_;
+  options.compression = kSnappyCompression;
+  BlockBasedTableOptions table_options;
+  table_options.block_cache_compressed = NewLRUCache(8 * 1024 * 1024);
+  table_options.no_block_cache = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  options.statistics = CreateDBStatistics();
+
+  ASSERT_OK(DB::OpenForReadOnly(options, dbname_, &db_ptr));
+
+  std::string v;
+  ASSERT_OK(db_ptr->Get(ReadOptions(), "foo", &v));
+  ASSERT_EQ("bar", v);
+  ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_COMPRESSED_HIT));
+  ASSERT_OK(db_ptr->Get(ReadOptions(), "foo", &v));
+  ASSERT_EQ("bar", v);
+  if (Snappy_Supported()) {
+    if (use_mmap_) {
+      ASSERT_EQ(0,
+                options.statistics->getTickerCount(BLOCK_CACHE_COMPRESSED_HIT));
+    } else {
+      ASSERT_EQ(1,
+                options.statistics->getTickerCount(BLOCK_CACHE_COMPRESSED_HIT));
+    }
+  }
+
+  delete db_ptr;
+}
+
+INSTANTIATE_TEST_CASE_P(TestReadOnlyWithCompressedCache,
+                        TestReadOnlyWithCompressedCache,
+                        ::testing::Combine(::testing::Values(-1, 100),
+                                           ::testing::Bool()));
+#endif  // ROCKSDB_LITE
+
 class PrefixFullBloomWithReverseComparator
    : public DBTestBase,
      public ::testing::WithParamInterface<bool> {
@@ -1329,7 +1389,42 @@ class CompactionCompressionListener : public EventListener {
  const Options* db_options_;
 };

-TEST_F(DBTest2, CompressionFailures) {
+enum CompressionFailureType {
+  kTestCompressionFail,
+  kTestDecompressionFail,
+  kTestDecompressionCorruption
+};
+
+class CompressionFailuresTest
+    : public DBTest2,
+      public testing::WithParamInterface<std::tuple<
+          CompressionFailureType, CompressionType, uint32_t, uint32_t>> {
+ public:
+  CompressionFailuresTest() {
+    std::tie(compression_failure_type_, compression_type_,
+             compression_max_dict_bytes_, compression_parallel_threads_) =
+        GetParam();
+  }
+
+  CompressionFailureType compression_failure_type_ = kTestCompressionFail;
+  CompressionType compression_type_ = kNoCompression;
+  uint32_t compression_max_dict_bytes_ = 0;
+  uint32_t compression_parallel_threads_ = 0;
+};
+
+INSTANTIATE_TEST_CASE_P(
+    DBTest2, CompressionFailuresTest,
+    ::testing::Combine(::testing::Values(kTestCompressionFail,
+                                         kTestDecompressionFail,
+                                         kTestDecompressionCorruption),
+                       ::testing::ValuesIn(GetSupportedCompressions()),
+                       ::testing::Values(0, 10), ::testing::Values(1, 4)));
+
+TEST_P(CompressionFailuresTest, CompressionFailures) {
+  if (compression_type_ == kNoCompression) {
+    return;
+  }
+
  Options options = CurrentOptions();
  options.level0_file_num_compaction_trigger = 2;
  options.max_bytes_for_level_base = 1024;
@@ -1343,50 +1438,41 @@ TEST_F(DBTest2, CompressionFailures) {
  table_options.verify_compression = true;
  options.table_factory.reset(new BlockBasedTableFactory(table_options));

-  enum CompressionFailureType {
-    kTestCompressionFail,
-    kTestDecompressionFail,
-    kTestDecompressionCorruption
-  } curr_compression_failure_type;
-  std::vector<CompressionFailureType> compression_failure_types = {
-      kTestCompressionFail, kTestDecompressionFail,
-      kTestDecompressionCorruption};
+  options.compression = compression_type_;
+  options.compression_opts.parallel_threads = compression_parallel_threads_;
+  options.compression_opts.max_dict_bytes = compression_max_dict_bytes_;
+  options.bottommost_compression_opts.parallel_threads =
+      compression_parallel_threads_;
+  options.bottommost_compression_opts.max_dict_bytes =
+      compression_max_dict_bytes_;

-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "BlockBasedTableBuilder::CompressBlockInternal:TamperWithReturnValue",
-      [&curr_compression_failure_type](void* arg) {
-        bool* ret = static_cast<bool*>(arg);
-        if (curr_compression_failure_type == kTestCompressionFail) {
+  if (compression_failure_type_ == kTestCompressionFail) {
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "BlockBasedTableBuilder::CompressBlockInternal:TamperWithReturnValue",
+        [](void* arg) {
+          bool* ret = static_cast<bool*>(arg);
          *ret = false;
-        }
-      });
-
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "UncompressBlockContentsForCompressionType:TamperWithReturnValue",
-      [&curr_compression_failure_type](void* arg) {
-        Status* ret = static_cast<Status*>(arg);
-        ASSERT_OK(*ret);
-        if (curr_compression_failure_type == kTestDecompressionFail) {
+        });
+  } else if (compression_failure_type_ == kTestDecompressionFail) {
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "UncompressBlockContentsForCompressionType:TamperWithReturnValue",
+        [](void* arg) {
+          Status* ret = static_cast<Status*>(arg);
+          ASSERT_OK(*ret);
          *ret = Status::Corruption("kTestDecompressionFail");
-        }
-      });
-
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "UncompressBlockContentsForCompressionType:"
-      "TamperWithDecompressionOutput",
-      [&curr_compression_failure_type](void* arg) {
-        if (curr_compression_failure_type == kTestDecompressionCorruption) {
+        });
+  } else if (compression_failure_type_ == kTestDecompressionCorruption) {
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "UncompressBlockContentsForCompressionType:"
+        "TamperWithDecompressionOutput",
+        [](void* arg) {
          BlockContents* contents = static_cast<BlockContents*>(arg);
          // Ensure uncompressed data != original data
          const size_t len = contents->data.size() + 1;
          std::unique_ptr<char[]> fake_data(new char[len]());
          *contents = BlockContents(std::move(fake_data), len);
-        }
-      });
-
-  std::vector<CompressionType> compression_types = GetSupportedCompressions();
-  std::vector<uint32_t> compression_max_dict_bytes = {0, 10};
-  std::vector<uint32_t> compression_parallel_threads = {1, 4};
+        });
+  }

  std::map<std::string, std::string> key_value_written;

@@ -1397,74 +1483,55 @@ TEST_F(DBTest2, CompressionFailures) {

  Status s = Status::OK();

-  for (auto compression_failure_type : compression_failure_types) {
-    curr_compression_failure_type = compression_failure_type;
-    for (auto compression_type : compression_types) {
-      if (compression_type == kNoCompression) {
-        continue;
+  DestroyAndReopen(options);
+  // Write 10 random files
+  for (int i = 0; i < 10; i++) {
+    for (int j = 0; j < 5; j++) {
+      std::string key = RandomString(&rnd, kKeySize);
+      // Ensure good compression ratio
+      std::string valueUnit = RandomString(&rnd, kValUnitSize);
+      std::string value;
+      for (int k = 0; k < kValSize; k += kValUnitSize) {
+        value += valueUnit;
      }
-      for (auto parallel_threads : compression_parallel_threads) {
-        for (auto max_dict_bytes : compression_max_dict_bytes) {
-          options.compression = compression_type;
-          options.compression_opts.parallel_threads = parallel_threads;
-          options.compression_opts.max_dict_bytes = max_dict_bytes;
-          options.bottommost_compression_opts.parallel_threads =
-              parallel_threads;
-          options.bottommost_compression_opts.max_dict_bytes = max_dict_bytes;
-
-          DestroyAndReopen(options);
-          // Write 10 random files
-          for (int i = 0; i < 10; i++) {
-            for (int j = 0; j < 5; j++) {
-              std::string key = RandomString(&rnd, kKeySize);
-              // Ensure good compression ratio
-              std::string valueUnit = RandomString(&rnd, kValUnitSize);
-              std::string value;
-              for (int k = 0; k < kValSize; k += kValUnitSize) {
-                value += valueUnit;
-              }
-              s = Put(key, value);
-              if (compression_failure_type == kTestCompressionFail) {
-                key_value_written[key] = value;
-                ASSERT_OK(s);
-              }
-            }
-            s = Flush();
-            if (compression_failure_type == kTestCompressionFail) {
-              ASSERT_OK(s);
-            }
-            s = dbfull()->TEST_WaitForCompact();
-            if (compression_failure_type == kTestCompressionFail) {
-              ASSERT_OK(s);
-            }
-            if (i == 4) {
-              // Make compression fail at the mid of table building
-              ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
-            }
-          }
-          ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-
-          if (compression_failure_type == kTestCompressionFail) {
-            // Should be kNoCompression, check content consistency
-            std::unique_ptr<Iterator> db_iter(db_->NewIterator(ReadOptions()));
-            for (db_iter->SeekToFirst(); db_iter->Valid(); db_iter->Next()) {
-              std::string key = db_iter->key().ToString();
-              std::string value = db_iter->value().ToString();
-              ASSERT_NE(key_value_written.find(key), key_value_written.end());
-              ASSERT_EQ(key_value_written[key], value);
-              key_value_written.erase(key);
-            }
-            ASSERT_EQ(0, key_value_written.size());
-          } else if (compression_failure_type == kTestDecompressionFail) {
-            ASSERT_EQ(std::string(s.getState()),
-                      "Could not decompress: kTestDecompressionFail");
-          } else if (compression_failure_type == kTestDecompressionCorruption) {
-            ASSERT_EQ(std::string(s.getState()),
-                      "Decompressed block did not match raw block");
-          }
-        }
+      s = Put(key, value);
+      if (compression_failure_type_ == kTestCompressionFail) {
+        key_value_written[key] = value;
+        ASSERT_OK(s);
      }
    }
+    s = Flush();
+    if (compression_failure_type_ == kTestCompressionFail) {
+      ASSERT_OK(s);
+    }
+    s = dbfull()->TEST_WaitForCompact();
+    if (compression_failure_type_ == kTestCompressionFail) {
+      ASSERT_OK(s);
+    }
+    if (i == 4) {
+      // Make compression fail at the mid of table building
+      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+    }
+  }
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+
+  if (compression_failure_type_ == kTestCompressionFail) {
+    // Should be kNoCompression, check content consistency
+    std::unique_ptr<Iterator> db_iter(db_->NewIterator(ReadOptions()));
+    for (db_iter->SeekToFirst(); db_iter->Valid(); db_iter->Next()) {
+      std::string key = db_iter->key().ToString();
+      std::string value = db_iter->value().ToString();
+      ASSERT_NE(key_value_written.find(key), key_value_written.end());
+      ASSERT_EQ(key_value_written[key], value);
+      key_value_written.erase(key);
+    }
+    ASSERT_EQ(0, key_value_written.size());
+  } else if (compression_failure_type_ == kTestDecompressionFail) {
+    ASSERT_EQ(std::string(s.getState()),
+              "Could not decompress: kTestDecompressionFail");
+  } else if (compression_failure_type_ == kTestDecompressionCorruption) {
+    ASSERT_EQ(std::string(s.getState()),
+              "Decompressed block did not match raw block");
  }
 }

@@ -2001,6 +2068,10 @@ class MockPersistentCache : public PersistentCache {
    return PersistentCache::StatsType();
  }

+  uint64_t NewId() override {
+    return last_id_.fetch_add(1, std::memory_order_relaxed);
+  }
+
  Status Insert(const Slice& page_key, const char* data,
                const size_t size) override {
    MutexLock _(&lock_);
@@ -2041,6 +2112,7 @@ class MockPersistentCache : public PersistentCache {
  const bool is_compressed_ = true;
  size_t size_ = 0;
  const size_t max_size_ = 10 * 1024;  // 10KiB
+  std::atomic<uint64_t> last_id_{1};
 };

 #ifdef OS_LINUX
@@ -2164,10 +2236,7 @@ TEST_F(DBTest2, TestPerfContextIterCpuTime) {
 }
 #endif  // OS_LINUX

-// GetUniqueIdFromFile is not implemented on these platforms. Persistent cache
-// breaks when that function is not implemented and no regular block cache is
-// provided.
-#if !defined(OS_SOLARIS) && !defined(OS_WIN)
+#if !defined OS_SOLARIS
 TEST_F(DBTest2, PersistentCache) {
  int num_iter = 80;

@@ -2231,7 +2300,7 @@ TEST_F(DBTest2, PersistentCache) {
    }
  }
 }
-#endif  // !defined(OS_SOLARIS) && !defined(OS_WIN)
+#endif  // !defined OS_SOLARIS

 namespace {
 void CountSyncPoint() {
@@ -2888,6 +2957,94 @@ TEST_F(DBTest2, OptimizeForSmallDB) {

 #endif  // ROCKSDB_LITE

+TEST_F(DBTest2, IterRaceFlush1) {
+  ASSERT_OK(Put("foo", "v1"));
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::NewIterator:1", "DBTest2::IterRaceFlush:1"},
+       {"DBTest2::IterRaceFlush:2", "DBImpl::NewIterator:2"}});
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  ROCKSDB_NAMESPACE::port::Thread t1([&] {
+    TEST_SYNC_POINT("DBTest2::IterRaceFlush:1");
+    ASSERT_OK(Put("foo", "v2"));
+    Flush();
+    TEST_SYNC_POINT("DBTest2::IterRaceFlush:2");
+  });
+
+  // iterator is created after the first Put(), so it should see either
+  // "v1" or "v2".
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
+    it->Seek("foo");
+    ASSERT_TRUE(it->Valid());
+    ASSERT_EQ("foo", it->key().ToString());
+  }
+
+  t1.join();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBTest2, IterRaceFlush2) {
+  ASSERT_OK(Put("foo", "v1"));
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::NewIterator:3", "DBTest2::IterRaceFlush2:1"},
+       {"DBTest2::IterRaceFlush2:2", "DBImpl::NewIterator:4"}});
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  ROCKSDB_NAMESPACE::port::Thread t1([&] {
+    TEST_SYNC_POINT("DBTest2::IterRaceFlush2:1");
+    ASSERT_OK(Put("foo", "v2"));
+    Flush();
+    TEST_SYNC_POINT("DBTest2::IterRaceFlush2:2");
+  });
+
+  // iterator is created after the first Put(), so it should see either
+  // "v1" or "v2".
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
+    it->Seek("foo");
+    ASSERT_TRUE(it->Valid());
+    ASSERT_EQ("foo", it->key().ToString());
+  }
+
+  t1.join();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBTest2, IterRefreshRaceFlush) {
+  ASSERT_OK(Put("foo", "v1"));
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+      {{"ArenaWrappedDBIter::Refresh:1", "DBTest2::IterRefreshRaceFlush:1"},
+       {"DBTest2::IterRefreshRaceFlush:2", "ArenaWrappedDBIter::Refresh:2"}});
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  ROCKSDB_NAMESPACE::port::Thread t1([&] {
+    TEST_SYNC_POINT("DBTest2::IterRefreshRaceFlush:1");
+    ASSERT_OK(Put("foo", "v2"));
+    Flush();
+    TEST_SYNC_POINT("DBTest2::IterRefreshRaceFlush:2");
+  });
+
+  // iterator is created after the first Put(), so it should see either
+  // "v1" or "v2".
+  {
+    std::unique_ptr<Iterator> it(db_->NewIterator(ReadOptions()));
+    it->Refresh();
+    it->Seek("foo");
+    ASSERT_TRUE(it->Valid());
+    ASSERT_EQ("foo", it->key().ToString());
+  }
+
+  t1.join();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+}
+
 TEST_F(DBTest2, GetRaceFlush1) {
  ASSERT_OK(Put("foo", "v1"));

@@ -4917,6 +5074,32 @@ TEST_F(DBTest2, AutoPrefixMode1) {
  }
 }
 #endif  // ROCKSDB_LITE
+
+// WAL recovery mode is WALRecoveryMode::kPointInTimeRecovery.
+TEST_F(DBTest2, PointInTimeRecoveryWithIOErrorWhileReadingWal) {
+  Options options = CurrentOptions();
+  DestroyAndReopen(options);
+  ASSERT_OK(Put("foo", "value0"));
+  Close();
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  bool should_inject_error = false;
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::RecoverLogFiles:BeforeReadWal",
+      [&](void* /*arg*/) { should_inject_error = true; });
+  SyncPoint::GetInstance()->SetCallBack(
+      "LogReader::ReadMore:AfterReadFile", [&](void* arg) {
+        if (should_inject_error) {
+          ASSERT_NE(nullptr, arg);
+          *reinterpret_cast<Status*>(arg) = Status::IOError("Injected IOError");
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  options.avoid_flush_during_recovery = true;
+  options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
+  Status s = TryReopen(options);
+  ASSERT_TRUE(s.IsIOError());
+}
 }  // namespace ROCKSDB_NAMESPACE

 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
@@ -121,7 +121,9 @@ void EventHelpers::LogAndNotifyTableFileCreationFinished(
              << table_properties.compression_options << "creation_time"
              << table_properties.creation_time << "oldest_key_time"
              << table_properties.oldest_key_time << "file_creation_time"
-              << table_properties.file_creation_time;
+              << table_properties.file_creation_time << "db_id"
+              << table_properties.db_id << "db_session_id"
+              << table_properties.db_session_id;

      // user collected properties
      for (const auto& prop : table_properties.readable_properties) {
@@ -42,6 +42,29 @@ class ExternalSSTFileBasicTest
    return db_->IngestExternalFile(files, opts);
  }

+  Status AddFileWithFileChecksum(
+      const std::vector<std::string>& files,
+      const std::vector<std::string>& files_checksums,
+      const std::vector<std::string>& files_checksum_func_names,
+      bool verify_file_checksum = true, bool move_files = false,
+      bool skip_snapshot_check = false, bool write_global_seqno = true) {
+    IngestExternalFileOptions opts;
+    opts.move_files = move_files;
+    opts.snapshot_consistency = !skip_snapshot_check;
+    opts.allow_global_seqno = false;
+    opts.allow_blocking_flush = false;
+    opts.write_global_seqno = write_global_seqno;
+    opts.verify_file_checksum = verify_file_checksum;
+
+    IngestExternalFileArg arg;
+    arg.column_family = db_->DefaultColumnFamily();
+    arg.external_files = files;
+    arg.options = opts;
+    arg.files_checksums = files_checksums;
+    arg.files_checksum_func_names = files_checksum_func_names;
+    return db_->IngestExternalFiles({arg});
+  }
+
  Status GenerateAndAddExternalFile(
      const Options options, std::vector<int> keys,
      const std::vector<ValueType>& value_types,
@@ -298,6 +321,266 @@ TEST_F(ExternalSSTFileBasicTest, BasicWithFileChecksumCrc32c) {
  DestroyAndRecreateExternalSSTFilesDir();
 }

+TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
+  Options old_options = CurrentOptions();
+  Options options = CurrentOptions();
+  options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
+  const ImmutableCFOptions ioptions(options);
+  ChecksumVerifyHelper checksum_helper(options);
+
+  SstFileWriter sst_file_writer(EnvOptions(), options);
+
+  // file01.sst (1000 => 1099)
+  std::string file1 = sst_files_dir_ + "file01.sst";
+  ASSERT_OK(sst_file_writer.Open(file1));
+  for (int k = 1000; k < 1100; k++) {
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
+  }
+  ExternalSstFileInfo file1_info;
+  Status s = sst_file_writer.Finish(&file1_info);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_EQ(file1_info.file_path, file1);
+  ASSERT_EQ(file1_info.num_entries, 100);
+  ASSERT_EQ(file1_info.smallest_key, Key(1000));
+  ASSERT_EQ(file1_info.largest_key, Key(1099));
+  std::string file_checksum1, file_checksum_func_name1;
+  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
+      file1, &file_checksum1, &file_checksum_func_name1));
+  ASSERT_EQ(file1_info.file_checksum, file_checksum1);
+  ASSERT_EQ(file1_info.file_checksum_func_name, file_checksum_func_name1);
+
+  // file02.sst (1100 => 1299)
+  std::string file2 = sst_files_dir_ + "file02.sst";
+  ASSERT_OK(sst_file_writer.Open(file2));
+  for (int k = 1100; k < 1300; k++) {
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
+  }
+  ExternalSstFileInfo file2_info;
+  s = sst_file_writer.Finish(&file2_info);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_EQ(file2_info.file_path, file2);
+  ASSERT_EQ(file2_info.num_entries, 200);
+  ASSERT_EQ(file2_info.smallest_key, Key(1100));
+  ASSERT_EQ(file2_info.largest_key, Key(1299));
+  std::string file_checksum2, file_checksum_func_name2;
+  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
+      file2, &file_checksum2, &file_checksum_func_name2));
+  ASSERT_EQ(file2_info.file_checksum, file_checksum2);
+  ASSERT_EQ(file2_info.file_checksum_func_name, file_checksum_func_name2);
+
+  // file03.sst (1300 => 1499)
+  std::string file3 = sst_files_dir_ + "file03.sst";
+  ASSERT_OK(sst_file_writer.Open(file3));
+  for (int k = 1300; k < 1500; k++) {
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
+  }
+  ExternalSstFileInfo file3_info;
+  s = sst_file_writer.Finish(&file3_info);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_EQ(file3_info.file_path, file3);
+  ASSERT_EQ(file3_info.num_entries, 200);
+  ASSERT_EQ(file3_info.smallest_key, Key(1300));
+  ASSERT_EQ(file3_info.largest_key, Key(1499));
+  std::string file_checksum3, file_checksum_func_name3;
+  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
+      file3, &file_checksum3, &file_checksum_func_name3));
+  ASSERT_EQ(file3_info.file_checksum, file_checksum3);
+  ASSERT_EQ(file3_info.file_checksum_func_name, file_checksum_func_name3);
+
+  // file04.sst (1500 => 1799)
+  std::string file4 = sst_files_dir_ + "file04.sst";
+  ASSERT_OK(sst_file_writer.Open(file4));
+  for (int k = 1500; k < 1800; k++) {
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
+  }
+  ExternalSstFileInfo file4_info;
+  s = sst_file_writer.Finish(&file4_info);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_EQ(file4_info.file_path, file4);
+  ASSERT_EQ(file4_info.num_entries, 300);
+  ASSERT_EQ(file4_info.smallest_key, Key(1500));
+  ASSERT_EQ(file4_info.largest_key, Key(1799));
+  std::string file_checksum4, file_checksum_func_name4;
+  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
+      file4, &file_checksum4, &file_checksum_func_name4));
+  ASSERT_EQ(file4_info.file_checksum, file_checksum4);
+  ASSERT_EQ(file4_info.file_checksum_func_name, file_checksum_func_name4);
+
+  // file05.sst (1800 => 1899)
+  std::string file5 = sst_files_dir_ + "file05.sst";
+  ASSERT_OK(sst_file_writer.Open(file5));
+  for (int k = 1800; k < 2000; k++) {
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
+  }
+  ExternalSstFileInfo file5_info;
+  s = sst_file_writer.Finish(&file5_info);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_EQ(file5_info.file_path, file5);
+  ASSERT_EQ(file5_info.num_entries, 200);
+  ASSERT_EQ(file5_info.smallest_key, Key(1800));
+  ASSERT_EQ(file5_info.largest_key, Key(1999));
+  std::string file_checksum5, file_checksum_func_name5;
+  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
+      file5, &file_checksum5, &file_checksum_func_name5));
+  ASSERT_EQ(file5_info.file_checksum, file_checksum5);
+  ASSERT_EQ(file5_info.file_checksum_func_name, file_checksum_func_name5);
+
+  // file06.sst (2000 => 2199)
+  std::string file6 = sst_files_dir_ + "file06.sst";
+  ASSERT_OK(sst_file_writer.Open(file6));
+  for (int k = 2000; k < 2200; k++) {
+    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
+  }
+  ExternalSstFileInfo file6_info;
+  s = sst_file_writer.Finish(&file6_info);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_EQ(file6_info.file_path, file6);
+  ASSERT_EQ(file6_info.num_entries, 200);
+  ASSERT_EQ(file6_info.smallest_key, Key(2000));
+  ASSERT_EQ(file6_info.largest_key, Key(2199));
+  std::string file_checksum6, file_checksum_func_name6;
+  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
+      file6, &file_checksum6, &file_checksum_func_name6));
+  ASSERT_EQ(file6_info.file_checksum, file_checksum6);
+  ASSERT_EQ(file6_info.file_checksum_func_name, file_checksum_func_name6);
+
+  s = AddFileWithFileChecksum({file1}, {file_checksum1, "xyz"},
+                              {file_checksum1}, true, false, false, false);
+  // does not care the checksum input since db does not enable file checksum
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_OK(env_->FileExists(file1));
+  std::vector<LiveFileMetaData> live_files;
+  dbfull()->GetLiveFilesMetaData(&live_files);
+  std::set<std::string> set1;
+  for (auto f : live_files) {
+    set1.insert(f.name);
+    ASSERT_EQ(f.file_checksum, kUnknownFileChecksum);
+    ASSERT_EQ(f.file_checksum_func_name, kUnknownFileChecksumFuncName);
+  }
+
+  // Reopen Db with checksum enabled
+  Reopen(options);
+  // Enable verify_file_checksum option
+  // The checksum vector does not match, fail the ingestion
+  s = AddFileWithFileChecksum({file2}, {file_checksum2, "xyz"},
+                              {file_checksum_func_name2}, true, false, false,
+                              false);
+  ASSERT_FALSE(s.ok()) << s.ToString();
+
+  // Enable verify_file_checksum option
+  // The checksum name does not match, fail the ingestion
+  s = AddFileWithFileChecksum({file2}, {file_checksum2}, {"xyz"}, true, false,
+                              false, false);
+  ASSERT_FALSE(s.ok()) << s.ToString();
+
+  // Enable verify_file_checksum option
+  // The checksum itself does not match, fail the ingestion
+  s = AddFileWithFileChecksum({file2}, {"xyz"}, {file_checksum_func_name2},
+                              true, false, false, false);
+  ASSERT_FALSE(s.ok()) << s.ToString();
+
+  // Enable verify_file_checksum option
+  // All matches, ingestion is successful
+  s = AddFileWithFileChecksum({file2}, {file_checksum2},
+                              {file_checksum_func_name2}, true, false, false,
+                              false);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  std::vector<LiveFileMetaData> live_files1;
+  dbfull()->GetLiveFilesMetaData(&live_files1);
+  for (auto f : live_files1) {
+    if (set1.find(f.name) == set1.end()) {
+      ASSERT_EQ(f.file_checksum, file_checksum2);
+      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name2);
+      set1.insert(f.name);
+    }
+  }
+  ASSERT_OK(env_->FileExists(file2));
+
+  // Enable verify_file_checksum option
+  // No checksum information is provided, generate it when ingesting
+  std::vector<std::string> checksum, checksum_func;
+  s = AddFileWithFileChecksum({file3}, checksum, checksum_func, true, false,
+                              false, false);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  std::vector<LiveFileMetaData> live_files2;
+  dbfull()->GetLiveFilesMetaData(&live_files2);
+  for (auto f : live_files2) {
+    if (set1.find(f.name) == set1.end()) {
+      ASSERT_EQ(f.file_checksum, file_checksum3);
+      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name3);
+      set1.insert(f.name);
+    }
+  }
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_OK(env_->FileExists(file3));
+
+  // Does not enable verify_file_checksum options
+  // The checksum name does not match, fail the ingestion
+  s = AddFileWithFileChecksum({file4}, {file_checksum4}, {"xyz"}, false, false,
+                              false, false);
+  ASSERT_FALSE(s.ok()) << s.ToString();
+
+  // Does not enable verify_file_checksum options
+  // Checksum function name matches, store the checksum being ingested.
+  s = AddFileWithFileChecksum({file4}, {"asd"}, {file_checksum_func_name4},
+                              false, false, false, false);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  std::vector<LiveFileMetaData> live_files3;
+  dbfull()->GetLiveFilesMetaData(&live_files3);
+  for (auto f : live_files3) {
+    if (set1.find(f.name) == set1.end()) {
+      ASSERT_FALSE(f.file_checksum == file_checksum4);
+      ASSERT_EQ(f.file_checksum, "asd");
+      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name4);
+      set1.insert(f.name);
+    }
+  }
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_OK(env_->FileExists(file4));
+
+  // enable verify_file_checksum options, DB enable checksum, and enable
+  // write_global_seq. So the checksum stored is different from the one
+  // ingested due to the sequence number changes.
+  s = AddFileWithFileChecksum({file5}, {file_checksum5},
+                              {file_checksum_func_name5}, true, false, false,
+                              true);
+  ASSERT_OK(s);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  std::vector<LiveFileMetaData> live_files4;
+  dbfull()->GetLiveFilesMetaData(&live_files4);
+  for (auto f : live_files4) {
+    if (set1.find(f.name) == set1.end()) {
+      std::string cur_checksum5, cur_checksum_func_name5;
+      ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
+          dbname_ + f.name, &cur_checksum5, &cur_checksum_func_name5));
+      ASSERT_EQ(f.file_checksum, cur_checksum5);
+      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name5);
+      set1.insert(f.name);
+    }
+  }
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_OK(env_->FileExists(file5));
+
+  // Does not enable verify_file_checksum options and also the ingested file
+  // checksum information is empty. DB will generate and store the checksum
+  // in Manifest.
+  std::vector<std::string> files_c6, files_name6;
+  s = AddFileWithFileChecksum({file6}, files_c6, files_name6, false, false,
+                              false, false);
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  std::vector<LiveFileMetaData> live_files6;
+  dbfull()->GetLiveFilesMetaData(&live_files6);
+  for (auto f : live_files6) {
+    if (set1.find(f.name) == set1.end()) {
+      ASSERT_EQ(f.file_checksum, file_checksum6);
+      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name6);
+      set1.insert(f.name);
+    }
+  }
+  ASSERT_TRUE(s.ok()) << s.ToString();
+  ASSERT_OK(env_->FileExists(file6));
+}
+
 TEST_F(ExternalSSTFileBasicTest, NoCopy) {
  Options options = CurrentOptions();
  const ImmutableCFOptions ioptions(options);
@@ -28,6 +28,8 @@ namespace ROCKSDB_NAMESPACE {

 Status ExternalSstFileIngestionJob::Prepare(
    const std::vector<std::string>& external_files_paths,
+    const std::vector<std::string>& files_checksums,
+    const std::vector<std::string>& files_checksum_func_names,
    uint64_t next_file_number, SuperVersion* sv) {
  Status status;

@@ -142,6 +144,9 @@ Status ExternalSstFileIngestionJob::Prepare(
      break;
    }
    f.internal_file_path = path_inside_db;
+    // Initialize the checksum information of ingested files.
+    f.file_checksum = kUnknownFileChecksum;
+    f.file_checksum_func_name = kUnknownFileChecksumFuncName;
    ingestion_path_ids.insert(f.fd.GetPathId());
  }

@@ -160,6 +165,128 @@ Status ExternalSstFileIngestionJob::Prepare(
  }
  TEST_SYNC_POINT("ExternalSstFileIngestionJob::AfterSyncDir");

+  // Generate and check the sst file checksum. Note that, if
+  // IngestExternalFileOptions::write_global_seqno is true, we will not update
+  // the checksum information in the files_to_ingests_ here, since the file is
+  // upadted with the new global_seqno. After global_seqno is updated, DB will
+  // generate the new checksum and store it in the Manifest. In all other cases
+  // if ingestion_options_.write_global_seqno == true and
+  // verify_file_checksum is false, we only check the checksum function name.
+  if (status.ok() && db_options_.file_checksum_gen_factory != nullptr) {
+    if (ingestion_options_.verify_file_checksum == false &&
+        files_checksums.size() == files_to_ingest_.size() &&
+        files_checksum_func_names.size() == files_to_ingest_.size()) {
+      // Only when verify_file_checksum == false and the checksum for ingested
+      // files are provided, DB will use the provided checksum and does not
+      // generate the checksum for ingested files.
+      need_generate_file_checksum_ = false;
+    } else {
+      need_generate_file_checksum_ = true;
+    }
+    FileChecksumGenContext gen_context;
+    std::unique_ptr<FileChecksumGenerator> file_checksum_gen =
+        db_options_.file_checksum_gen_factory->CreateFileChecksumGenerator(
+            gen_context);
+    std::vector<std::string> generated_checksums;
+    std::vector<std::string> generated_checksum_func_names;
+    // Step 1: generate the checksum for ingested sst file.
+    if (need_generate_file_checksum_) {
+      for (size_t i = 0; i < files_to_ingest_.size(); i++) {
+        std::string generated_checksum, generated_checksum_func_name;
+        IOStatus io_s = GenerateOneFileChecksum(
+            fs_, files_to_ingest_[i].internal_file_path,
+            db_options_.file_checksum_gen_factory.get(), &generated_checksum,
+            &generated_checksum_func_name,
+            ingestion_options_.verify_checksums_readahead_size,
+            db_options_.allow_mmap_reads);
+        if (!io_s.ok()) {
+          status = io_s;
+          ROCKS_LOG_WARN(db_options_.info_log,
+                         "Sst file checksum generation of file: %s failed: %s",
+                         files_to_ingest_[i].internal_file_path.c_str(),
+                         status.ToString().c_str());
+          break;
+        }
+        if (ingestion_options_.write_global_seqno == false) {
+          files_to_ingest_[i].file_checksum = generated_checksum;
+          files_to_ingest_[i].file_checksum_func_name =
+              generated_checksum_func_name;
+        }
+        generated_checksums.push_back(generated_checksum);
+        generated_checksum_func_names.push_back(generated_checksum_func_name);
+      }
+    }
+
+    // Step 2: based on the verify_file_checksum and ingested checksum
+    // information, do the verification.
+    if (status.ok()) {
+      if (files_checksums.size() == files_to_ingest_.size() &&
+          files_checksum_func_names.size() == files_to_ingest_.size()) {
+        // Verify the checksum and checksum function name.
+        if (ingestion_options_.verify_file_checksum) {
+          for (size_t i = 0; i < files_to_ingest_.size(); i++) {
+            if (files_checksum_func_names[i] !=
+                generated_checksum_func_names[i]) {
+              status = Status::InvalidArgument(
+                  "Checksum function name does not match with the checksum "
+                  "function name of this DB");
+              ROCKS_LOG_WARN(
+                  db_options_.info_log,
+                  "Sst file checksum verification of file: %s failed: %s",
+                  external_files_paths[i].c_str(), status.ToString().c_str());
+              break;
+            }
+            if (files_checksums[i] != generated_checksums[i]) {
+              status = Status::Corruption(
+                  "Ingested checksum does not match with the generated "
+                  "checksum");
+              ROCKS_LOG_WARN(
+                  db_options_.info_log,
+                  "Sst file checksum verification of file: %s failed: %s",
+                  files_to_ingest_[i].internal_file_path.c_str(),
+                  status.ToString().c_str());
+              break;
+            }
+          }
+        } else {
+          // If verify_file_checksum is not enabled, we only verify the
+          // checksum function name. If it does not match, fail the ingestion.
+          // If matches, we trust the ingested checksum information and store
+          // in the Manifest.
+          for (size_t i = 0; i < files_to_ingest_.size(); i++) {
+            if (files_checksum_func_names[i] != file_checksum_gen->Name()) {
+              status = Status::InvalidArgument(
+                  "Checksum function name does not match with the checksum "
+                  "function name of this DB");
+              ROCKS_LOG_WARN(
+                  db_options_.info_log,
+                  "Sst file checksum verification of file: %s failed: %s",
+                  external_files_paths[i].c_str(), status.ToString().c_str());
+              break;
+            }
+            files_to_ingest_[i].file_checksum = files_checksums[i];
+            files_to_ingest_[i].file_checksum_func_name =
+                files_checksum_func_names[i];
+          }
+        }
+      } else if (files_checksums.size() != files_checksum_func_names.size() ||
+                 (files_checksums.size() == files_checksum_func_names.size() &&
+                  files_checksums.size() != 0)) {
+        // The checksum or checksum function name vector are not both empty
+        // and they are incomplete.
+        status = Status::InvalidArgument(
+            "The checksum information of ingested sst files are nonempty and "
+            "the size of checksums or the size of the checksum function "
+            "names "
+            "does not match with the number of ingested sst files");
+        ROCKS_LOG_WARN(
+            db_options_.info_log,
+            "The ingested sst files checksum information is incomplete: %s",
+            status.ToString().c_str());
+      }
+    }
+  }
+
  // TODO: The following is duplicated with Cleanup().
  if (!status.ok()) {
    // We failed, remove all files that we copied into the db
@@ -245,6 +372,11 @@ Status ExternalSstFileIngestionJob::Run() {
      return status;
    }

+    status = GenerateChecksumForIngestedFile(&f);
+    if (!status.ok()) {
+      return status;
+    }
+
    // We use the import time as the ancester time. This is the time the data
    // is written to the database.
    int64_t temp_current_time = 0;
@@ -255,11 +387,11 @@ Status ExternalSstFileIngestionJob::Run() {
          static_cast<uint64_t>(temp_current_time);
    }

-    edit_.AddFile(
-        f.picked_level, f.fd.GetNumber(), f.fd.GetPathId(), f.fd.GetFileSize(),
-        f.smallest_internal_key, f.largest_internal_key, f.assigned_seqno,
-        f.assigned_seqno, false, kInvalidBlobFileNumber, oldest_ancester_time,
-        current_time, kUnknownFileChecksum, kUnknownFileChecksumFuncName);
+    edit_.AddFile(f.picked_level, f.fd.GetNumber(), f.fd.GetPathId(),
+                  f.fd.GetFileSize(), f.smallest_internal_key,
+                  f.largest_internal_key, f.assigned_seqno, f.assigned_seqno,
+                  false, kInvalidBlobFileNumber, oldest_ancester_time,
+                  current_time, f.file_checksum, f.file_checksum_func_name);
  }
  return status;
 }
@@ -687,6 +819,31 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
  return Status::OK();
 }

+IOStatus ExternalSstFileIngestionJob::GenerateChecksumForIngestedFile(
+    IngestedFileInfo* file_to_ingest) {
+  if (db_options_.file_checksum_gen_factory == nullptr ||
+      need_generate_file_checksum_ == false ||
+      ingestion_options_.write_global_seqno == false) {
+    // If file_checksum_gen_factory is not set, we are not able to generate
+    // the checksum. if write_global_seqno is false, it means we will use
+    // file checksum generated during Prepare(). This step will be skipped.
+    return IOStatus::OK();
+  }
+  std::string file_checksum, file_checksum_func_name;
+  IOStatus io_s = GenerateOneFileChecksum(
+      fs_, file_to_ingest->internal_file_path,
+      db_options_.file_checksum_gen_factory.get(), &file_checksum,
+      &file_checksum_func_name,
+      ingestion_options_.verify_checksums_readahead_size,
+      db_options_.allow_mmap_reads);
+  if (!io_s.ok()) {
+    return io_s;
+  }
+  file_to_ingest->file_checksum = file_checksum;
+  file_to_ingest->file_checksum_func_name = file_checksum_func_name;
+  return IOStatus::OK();
+}
+
 bool ExternalSstFileIngestionJob::IngestedFileFitInLevel(
    const IngestedFileInfo* file_to_ingest, int level) {
  if (level == 0) {
@@ -63,6 +63,10 @@ struct IngestedFileInfo {
  // ingestion_options.move_files is false by default, thus copy_file is true
  // by default.
  bool copy_file = true;
+  // The checksum of ingested file
+  std::string file_checksum;
+  // The name of checksum function that generate the checksum
+  std::string file_checksum_func_name;
 };

 class ExternalSstFileIngestionJob {
@@ -90,6 +94,8 @@ class ExternalSstFileIngestionJob {

  // Prepare the job by copying external files into the DB.
  Status Prepare(const std::vector<std::string>& external_files_paths,
+                 const std::vector<std::string>& files_checksums,
+                 const std::vector<std::string>& files_checksum_func_names,
                 uint64_t next_file_number, SuperVersion* sv);

  // Check if we need to flush the memtable before running the ingestion job
@@ -148,6 +154,8 @@ class ExternalSstFileIngestionJob {
  // Set the file global sequence number to `seqno`
  Status AssignGlobalSeqnoForIngestedFile(IngestedFileInfo* file_to_ingest,
                                          SequenceNumber seqno);
+  // Generate the file checksum and store in the IngestedFileInfo
+  IOStatus GenerateChecksumForIngestedFile(IngestedFileInfo* file_to_ingest);

  // Check if `file_to_ingest` can fit in level `level`
  // REQUIRES: Mutex held
@@ -175,6 +183,9 @@ class ExternalSstFileIngestionJob {
  // Set in ExternalSstFileIngestionJob::Prepare(), if true all files are
  // ingested in L0
  bool files_overlap_{false};
+  // Set in ExternalSstFileIngestionJob::Prepare(), if true and DB
+  // file_checksum_gen_factory is set, DB will generate checksum each file.
+  bool need_generate_file_checksum_{true};
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -12,6 +12,7 @@
 #include "file/filename.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
+#include "rocksdb/sst_file_reader.h"
 #include "rocksdb/sst_file_writer.h"
 #include "test_util/fault_injection_test_env.h"
 #include "test_util/testutil.h"
@@ -2393,6 +2394,41 @@ TEST_F(ExternalSSTFileTest, IngestFileWrittenWithCompressionDictionary) {
  ASSERT_EQ(1, num_compression_dicts);
 }

+// Very slow, not worth the cost to run regularly
+TEST_F(ExternalSSTFileTest, DISABLED_HugeBlockChecksum) {
+  int max_checksum = static_cast<int>(kxxHash64);
+  for (int i = 0; i <= max_checksum; ++i) {
+    BlockBasedTableOptions table_options;
+    table_options.checksum = static_cast<ChecksumType>(i);
+    Options options = CurrentOptions();
+    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+    SstFileWriter sst_file_writer(EnvOptions(), options);
+
+    // 2^32 - 1, will lead to data block with more than 2^32 bytes
+    size_t huge_size = port::kMaxUint32;
+
+    std::string f = sst_files_dir_ + "f.sst";
+    ASSERT_OK(sst_file_writer.Open(f));
+    {
+      Random64 r(123);
+      std::string huge(huge_size, 0);
+      for (size_t j = 0; j + 7 < huge_size; j += 8) {
+        EncodeFixed64(&huge[j], r.Next());
+      }
+      ASSERT_OK(sst_file_writer.Put("Huge", huge));
+    }
+
+    ExternalSstFileInfo f_info;
+    ASSERT_OK(sst_file_writer.Finish(&f_info));
+    ASSERT_GT(f_info.file_size, uint64_t{huge_size} + 10);
+
+    SstFileReader sst_file_reader(options);
+    ASSERT_OK(sst_file_reader.Open(f));
+    ASSERT_OK(sst_file_reader.VerifyChecksum());
+  }
+}
+
 TEST_P(ExternalSSTFileTest, IngestFilesIntoMultipleColumnFamilies_Success) {
  std::unique_ptr<FaultInjectionTestEnv> fault_injection_env(
      new FaultInjectionTestEnv(env_));
@@ -97,8 +97,11 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
                   CompressionType output_compression, Statistics* stats,
                   EventLogger* event_logger, bool measure_io_stats,
                   const bool sync_output_directory, const bool write_manifest,
-                   Env::Priority thread_pri)
+                   Env::Priority thread_pri, const std::string& db_id,
+                   const std::string& db_session_id)
    : dbname_(dbname),
+      db_id_(db_id),
+      db_session_id_(db_session_id),
      cfd_(cfd),
      db_options_(db_options),
      mutable_cf_options_(mutable_cf_options),
@@ -393,7 +396,8 @@ Status FlushJob::WriteLevel0Table() {
          mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
          TableFileCreationReason::kFlush, &io_s, event_logger_,
          job_context_->job_id, Env::IO_HIGH, &table_properties_, 0 /* level */,
-          creation_time, oldest_key_time, write_hint, current_time);
+          creation_time, oldest_key_time, write_hint, current_time, db_id_,
+          db_session_id_);
      if (!io_s.ok()) {
        io_status_ = io_s;
      }
@@ -72,7 +72,8 @@ class FlushJob {
           CompressionType output_compression, Statistics* stats,
           EventLogger* event_logger, bool measure_io_stats,
           const bool sync_output_directory, const bool write_manifest,
-           Env::Priority thread_pri);
+           Env::Priority thread_pri, const std::string& db_id = "",
+           const std::string& db_session_id = "");

  ~FlushJob();

@@ -103,6 +104,8 @@ class FlushJob {
 #endif  // !ROCKSDB_LITE

  const std::string& dbname_;
+  const std::string db_id_;
+  const std::string db_session_id_;
  ColumnFamilyData* cfd_;
  const ImmutableDBOptions& db_options_;
  const MutableCFOptions& mutable_cf_options_;
@@ -282,7 +282,6 @@ TEST_F(FlushJobTest, FlushMemTablesSingleColumnFamily) {
  assert(memtable_ids.size() == num_mems);
  uint64_t smallest_memtable_id = memtable_ids.front();
  uint64_t flush_memtable_id = smallest_memtable_id + num_mems_to_flush - 1;
-
  FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
                     db_options_, *cfd->GetLatestMutableCFOptions(),
                     &flush_memtable_id, env_options_, versions_.get(), &mutex_,
@@ -84,9 +84,9 @@ class ForwardLevelIterator : public InternalIterator {
        prefix_extractor_, /*table_reader_ptr=*/nullptr,
        /*file_read_hist=*/nullptr, TableReaderCaller::kUserIterator,
        /*arena=*/nullptr, /*skip_filters=*/false, /*level=*/-1,
+        /*max_file_size_for_l0_meta_pin=*/0,
        /*smallest_compaction_key=*/nullptr,
-        /*largest_compaction_key=*/nullptr,
-        allow_unprepared_value_);
+        /*largest_compaction_key=*/nullptr, allow_unprepared_value_);
    file_iter_->SetPinnedItersMgr(pinned_iters_mgr_);
    valid_ = false;
    if (!range_del_agg.IsEmpty()) {
@@ -686,9 +686,9 @@ void ForwardIterator::RebuildIterators(bool refresh_sv) {
        /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr,
        TableReaderCaller::kUserIterator, /*arena=*/nullptr,
        /*skip_filters=*/false, /*level=*/-1,
+        MaxFileSizeForL0MetaPin(sv_->mutable_cf_options),
        /*smallest_compaction_key=*/nullptr,
-        /*largest_compaction_key=*/nullptr,
-        allow_unprepared_value_));
+        /*largest_compaction_key=*/nullptr, allow_unprepared_value_));
  }
  BuildLevelIterators(vstorage);
  current_ = nullptr;
@@ -764,9 +764,9 @@ void ForwardIterator::RenewIterators() {
        /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr,
        TableReaderCaller::kUserIterator, /*arena=*/nullptr,
        /*skip_filters=*/false, /*level=*/-1,
+        MaxFileSizeForL0MetaPin(svnew->mutable_cf_options),
        /*smallest_compaction_key=*/nullptr,
-        /*largest_compaction_key=*/nullptr,
-        allow_unprepared_value_));
+        /*largest_compaction_key=*/nullptr, allow_unprepared_value_));
  }

  for (auto* f : l0_iters_) {
@@ -830,9 +830,9 @@ void ForwardIterator::ResetIncompleteIterators() {
        /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr,
        TableReaderCaller::kUserIterator, /*arena=*/nullptr,
        /*skip_filters=*/false, /*level=*/-1,
+        MaxFileSizeForL0MetaPin(sv_->mutable_cf_options),
        /*smallest_compaction_key=*/nullptr,
-        /*largest_compaction_key=*/nullptr,
-        allow_unprepared_value_);
+        /*largest_compaction_key=*/nullptr, allow_unprepared_value_);
    l0_iters_[i]->SetPinnedItersMgr(pinned_iters_mgr_);
  }

@@ -14,7 +14,7 @@ class ImportColumnFamilyTest : public DBTestBase {
  ImportColumnFamilyTest() : DBTestBase("/import_column_family_test") {
    sst_files_dir_ = dbname_ + "/sst_files/";
    DestroyAndRecreateExternalSSTFilesDir();
-    export_files_dir_ = test::TmpDir(env_) + "/export";
+    export_files_dir_ = test::PerThreadDBPath(env_, "export");
    import_cfh_ = nullptr;
    import_cfh2_ = nullptr;
    metadata_ptr_ = nullptr;
@@ -281,6 +281,7 @@ bool Reader::ReadMore(size_t* drop_size, int *error) {
    // Last read was a full read, so this is a trailer to skip
    buffer_.clear();
    Status status = file_->Read(kBlockSize, &buffer_, backing_store_);
+    TEST_SYNC_POINT_CALLBACK("LogReader::ReadMore:AfterReadFile", &status);
    end_of_buffer_offset_ += buffer_.size();
    if (!status.ok()) {
      buffer_.clear();
@@ -147,6 +147,8 @@ IOStatus Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
  // Compute the crc of the record type and the payload.
  crc = crc32c::Extend(crc, ptr, n);
  crc = crc32c::Mask(crc);  // Adjust for storage
+  TEST_SYNC_POINT_CALLBACK("LogWriter::EmitPhysicalRecord:BeforeEncodeChecksum",
+                           &crc);
  EncodeFixed32(buf, crc);

  // Write the header and the payload
@@ -183,6 +183,8 @@ class Repairer {
      }
      // Just create a DBImpl temporarily so we can reuse NewDB()
      DBImpl* db_impl = new DBImpl(db_options_, dbname_);
+      // Also use this temp DBImpl to get a session id
+      db_impl->GetDbSessionId(db_session_id_);
      status = db_impl->NewDB();
      delete db_impl;
    }
@@ -229,6 +231,7 @@ class Repairer {
  };

  std::string const dbname_;
+  std::string db_session_id_;
  Env* const env_;
  const EnvOptions env_options_;
  const DBOptions db_options_;
@@ -435,7 +438,9 @@ class Repairer {
          CompressionOptions(), false, nullptr /* internal_stats */,
          TableFileCreationReason::kRecovery, &io_s, nullptr /* event_logger */,
          0 /* job_id */, Env::IO_HIGH, nullptr /* table_properties */,
-          -1 /* level */, current_time, write_hint);
+          -1 /* level */, current_time, 0 /* oldest_key_time */, write_hint,
+          0 /* file_creation_time */, "DB Repairer" /* db_id */,
+          db_session_id_);
      ROCKS_LOG_INFO(db_options_.info_log,
                     "Log #%" PRIu64 ": %d ops saved to Table #%" PRIu64 " %s",
                     log, counter, meta.fd.GetNumber(),
@@ -528,7 +533,8 @@ class Repairer {
          cfd->GetLatestMutableCFOptions()->prefix_extractor.get(),
          /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr,
          TableReaderCaller::kRepair, /*arena=*/nullptr, /*skip_filters=*/false,
-          /*level=*/-1, /*smallest_compaction_key=*/nullptr,
+          /*level=*/-1, /*max_file_size_for_l0_meta_pin=*/0,
+          /*smallest_compaction_key=*/nullptr,
          /*largest_compaction_key=*/nullptr,
          /*allow_unprepared_value=*/false);
      ParsedInternalKey parsed;
@@ -97,7 +97,8 @@ Status TableCache::GetTableReader(
    bool sequential_mode, bool record_read_stats, HistogramImpl* file_read_hist,
    std::unique_ptr<TableReader>* table_reader,
    const SliceTransform* prefix_extractor, bool skip_filters, int level,
-    bool prefetch_index_and_filter_in_cache) {
+    bool prefetch_index_and_filter_in_cache,
+    size_t max_file_size_for_l0_meta_pin) {
  std::string fname =
      TableFileName(ioptions_.cf_paths, fd.GetNumber(), fd.GetPathId());
  std::unique_ptr<FSRandomAccessFile> file;
@@ -124,7 +125,8 @@ Status TableCache::GetTableReader(
        TableReaderOptions(ioptions_, prefix_extractor, file_options,
                           internal_comparator, skip_filters, immortal_tables_,
                           false /* force_direct_prefetch */, level,
-                           fd.largest_seqno, block_cache_tracer_),
+                           fd.largest_seqno, block_cache_tracer_,
+                           max_file_size_for_l0_meta_pin),
        std::move(file_reader), fd.GetFileSize(), table_reader,
        prefetch_index_and_filter_in_cache);
    TEST_SYNC_POINT("TableCache::GetTableReader:0");
@@ -145,8 +147,8 @@ Status TableCache::FindTable(const FileOptions& file_options,
                             const SliceTransform* prefix_extractor,
                             const bool no_io, bool record_read_stats,
                             HistogramImpl* file_read_hist, bool skip_filters,
-                             int level,
-                             bool prefetch_index_and_filter_in_cache) {
+                             int level, bool prefetch_index_and_filter_in_cache,
+                             size_t max_file_size_for_l0_meta_pin) {
  PERF_TIMER_GUARD_WITH_ENV(find_table_nanos, ioptions_.env);
  Status s;
  uint64_t number = fd.GetNumber();
@@ -170,7 +172,8 @@ Status TableCache::FindTable(const FileOptions& file_options,
    s = GetTableReader(file_options, internal_comparator, fd,
                       false /* sequential mode */, record_read_stats,
                       file_read_hist, &table_reader, prefix_extractor,
-                       skip_filters, level, prefetch_index_and_filter_in_cache);
+                       skip_filters, level, prefetch_index_and_filter_in_cache,
+                       max_file_size_for_l0_meta_pin);
    if (!s.ok()) {
      assert(table_reader == nullptr);
      RecordTick(ioptions_.statistics, NO_FILE_ERRORS);
@@ -194,6 +197,7 @@ InternalIterator* TableCache::NewIterator(
    RangeDelAggregator* range_del_agg, const SliceTransform* prefix_extractor,
    TableReader** table_reader_ptr, HistogramImpl* file_read_hist,
    TableReaderCaller caller, Arena* arena, bool skip_filters, int level,
+    size_t max_file_size_for_l0_meta_pin,
    const InternalKey* smallest_compaction_key,
    const InternalKey* largest_compaction_key, bool allow_unprepared_value) {
  PERF_TIMER_GUARD(new_table_iterator_nanos);
@@ -211,7 +215,9 @@ InternalIterator* TableCache::NewIterator(
    s = FindTable(file_options, icomparator, fd, &handle, prefix_extractor,
                  options.read_tier == kBlockCacheTier /* no_io */,
                  !for_compaction /* record_read_stats */, file_read_hist,
-                  skip_filters, level);
+                  skip_filters, level,
+                  true /* prefetch_index_and_filter_in_cache */,
+                  max_file_size_for_l0_meta_pin);
    if (s.ok()) {
      table_reader = GetTableReaderFromHandle(handle);
    }
@@ -372,7 +378,7 @@ Status TableCache::Get(const ReadOptions& options,
                       GetContext* get_context,
                       const SliceTransform* prefix_extractor,
                       HistogramImpl* file_read_hist, bool skip_filters,
-                       int level) {
+                       int level, size_t max_file_size_for_l0_meta_pin) {
  auto& fd = file_meta.fd;
  std::string* row_cache_entry = nullptr;
  bool done = false;
@@ -397,10 +403,12 @@ Status TableCache::Get(const ReadOptions& options,
  Cache::Handle* handle = nullptr;
  if (!done && s.ok()) {
    if (t == nullptr) {
-      s = FindTable(
-          file_options_, internal_comparator, fd, &handle, prefix_extractor,
-          options.read_tier == kBlockCacheTier /* no_io */,
-          true /* record_read_stats */, file_read_hist, skip_filters, level);
+      s = FindTable(file_options_, internal_comparator, fd, &handle,
+                    prefix_extractor,
+                    options.read_tier == kBlockCacheTier /* no_io */,
+                    true /* record_read_stats */, file_read_hist, skip_filters,
+                    level, true /* prefetch_index_and_filter_in_cache */,
+                    max_file_size_for_l0_meta_pin);
      if (s.ok()) {
        t = GetTableReaderFromHandle(handle);
      }
@@ -72,7 +72,8 @@ class TableCache {
      const FileMetaData& file_meta, RangeDelAggregator* range_del_agg,
      const SliceTransform* prefix_extractor, TableReader** table_reader_ptr,
      HistogramImpl* file_read_hist, TableReaderCaller caller, Arena* arena,
-      bool skip_filters, int level, const InternalKey* smallest_compaction_key,
+      bool skip_filters, int level, size_t max_file_size_for_l0_meta_pin,
+      const InternalKey* smallest_compaction_key,
      const InternalKey* largest_compaction_key, bool allow_unprepared_value);

  // If a seek to internal key "k" in specified file finds an entry,
@@ -91,7 +92,7 @@ class TableCache {
             GetContext* get_context,
             const SliceTransform* prefix_extractor = nullptr,
             HistogramImpl* file_read_hist = nullptr, bool skip_filters = false,
-             int level = -1);
+             int level = -1, size_t max_file_size_for_l0_meta_pin = 0);

  // Return the range delete tombstone iterator of the file specified by
  // `file_meta`.
@@ -135,7 +136,8 @@ class TableCache {
                   const bool no_io = false, bool record_read_stats = true,
                   HistogramImpl* file_read_hist = nullptr,
                   bool skip_filters = false, int level = -1,
-                   bool prefetch_index_and_filter_in_cache = true);
+                   bool prefetch_index_and_filter_in_cache = true,
+                   size_t max_file_size_for_l0_meta_pin = 0);

  // Get TableReader from a cache handle.
  TableReader* GetTableReaderFromHandle(Cache::Handle* handle);
@@ -200,7 +202,8 @@ class TableCache {
                        std::unique_ptr<TableReader>* table_reader,
                        const SliceTransform* prefix_extractor = nullptr,
                        bool skip_filters = false, int level = -1,
-                        bool prefetch_index_and_filter_in_cache = true);
+                        bool prefetch_index_and_filter_in_cache = true,
+                        size_t max_file_size_for_l0_meta_pin = 0);

  // Create a key prefix for looking up the row cache. The prefix is of the
  // format row_cache_id + fd_number + seq_no. Later, the user key can be
@@ -90,7 +90,8 @@ class VersionBuilder::Rep {
   public:
    bool IsEmpty() const {
      return !shared_meta_ && !additional_garbage_count_ &&
-             !additional_garbage_bytes_;
+             !additional_garbage_bytes_ && newly_linked_ssts_.empty() &&
+             newly_unlinked_ssts_.empty();
    }

    std::shared_ptr<SharedBlobFileMetaData> GetSharedMeta() const {
@@ -105,6 +106,14 @@ class VersionBuilder::Rep {
      return additional_garbage_bytes_;
    }

+    const std::unordered_set<uint64_t>& GetNewlyLinkedSsts() const {
+      return newly_linked_ssts_;
+    }
+
+    const std::unordered_set<uint64_t>& GetNewlyUnlinkedSsts() const {
+      return newly_unlinked_ssts_;
+    }
+
    void SetSharedMeta(std::shared_ptr<SharedBlobFileMetaData> shared_meta) {
      assert(!shared_meta_);
      assert(shared_meta);
@@ -117,10 +126,44 @@ class VersionBuilder::Rep {
      additional_garbage_bytes_ += bytes;
    }

+    void LinkSst(uint64_t sst_file_number) {
+      assert(newly_linked_ssts_.find(sst_file_number) ==
+             newly_linked_ssts_.end());
+
+      // Reconcile with newly unlinked SSTs on the fly. (Note: an SST can be
+      // linked to and unlinked from the same blob file in the case of a trivial
+      // move.)
+      auto it = newly_unlinked_ssts_.find(sst_file_number);
+
+      if (it != newly_unlinked_ssts_.end()) {
+        newly_unlinked_ssts_.erase(it);
+      } else {
+        newly_linked_ssts_.emplace(sst_file_number);
+      }
+    }
+
+    void UnlinkSst(uint64_t sst_file_number) {
+      assert(newly_unlinked_ssts_.find(sst_file_number) ==
+             newly_unlinked_ssts_.end());
+
+      // Reconcile with newly linked SSTs on the fly. (Note: an SST can be
+      // linked to and unlinked from the same blob file in the case of a trivial
+      // move.)
+      auto it = newly_linked_ssts_.find(sst_file_number);
+
+      if (it != newly_linked_ssts_.end()) {
+        newly_linked_ssts_.erase(it);
+      } else {
+        newly_unlinked_ssts_.emplace(sst_file_number);
+      }
+    }
+
   private:
    std::shared_ptr<SharedBlobFileMetaData> shared_meta_;
    uint64_t additional_garbage_count_ = 0;
    uint64_t additional_garbage_bytes_ = 0;
+    std::unordered_set<uint64_t> newly_linked_ssts_;
+    std::unordered_set<uint64_t> newly_unlinked_ssts_;
  };

  const FileOptions& file_options_;
@@ -212,29 +255,19 @@ class VersionBuilder::Rep {
    return false;
  }

-  Status CheckConsistencyOfOldestBlobFileReference(
-      const VersionStorageInfo* vstorage, uint64_t blob_file_number) const {
-    assert(vstorage);
+  using ExpectedLinkedSsts =
+      std::unordered_map<uint64_t, BlobFileMetaData::LinkedSsts>;

-    // TODO: remove this check once we actually start recoding metadata for
-    // blob files in the MANIFEST.
-    if (vstorage->GetBlobFiles().empty()) {
-      return Status::OK();
-    }
+  static void UpdateExpectedLinkedSsts(
+      uint64_t table_file_number, uint64_t blob_file_number,
+      ExpectedLinkedSsts* expected_linked_ssts) {
+    assert(expected_linked_ssts);

    if (blob_file_number == kInvalidBlobFileNumber) {
-      return Status::OK();
+      return;
    }

-    if (!IsBlobFileInVersion(blob_file_number)) {
-      std::ostringstream oss;
-      oss << "Blob file #" << blob_file_number
-          << " is not part of this version";
-
-      return Status::Corruption("VersionBuilder", oss.str());
-    }
-
-    return Status::OK();
+    (*expected_linked_ssts)[blob_file_number].emplace(table_file_number);
  }

  Status CheckConsistency(VersionStorageInfo* vstorage) {
@@ -245,9 +278,13 @@ class VersionBuilder::Rep {
      return Status::OK();
    }
 #endif
-    // Make sure the files are sorted correctly and that the oldest blob file
-    // reference for each table file points to a valid blob file in this
-    // version.
+    // Make sure the files are sorted correctly and that the links between
+    // table files and blob files are consistent. The latter is checked using
+    // the following mapping, which is built using the forward links
+    // (table file -> blob file), and is subsequently compared with the inverse
+    // mapping stored in the BlobFileMetaData objects.
+    ExpectedLinkedSsts expected_linked_ssts;
+
    for (int level = 0; level < num_levels_; level++) {
      auto& level_files = vstorage->LevelFiles(level);

@@ -256,19 +293,14 @@ class VersionBuilder::Rep {
      }

      assert(level_files[0]);
-      Status s = CheckConsistencyOfOldestBlobFileReference(
-          vstorage, level_files[0]->oldest_blob_file_number);
-      if (!s.ok()) {
-        return s;
-      }
-
+      UpdateExpectedLinkedSsts(level_files[0]->fd.GetNumber(),
+                               level_files[0]->oldest_blob_file_number,
+                               &expected_linked_ssts);
      for (size_t i = 1; i < level_files.size(); i++) {
        assert(level_files[i]);
-        s = CheckConsistencyOfOldestBlobFileReference(
-            vstorage, level_files[i]->oldest_blob_file_number);
-        if (!s.ok()) {
-          return s;
-        }
+        UpdateExpectedLinkedSsts(level_files[i]->fd.GetNumber(),
+                                 level_files[i]->oldest_blob_file_number,
+                                 &expected_linked_ssts);

        auto f1 = level_files[i - 1];
        auto f2 = level_files[i];
@@ -278,7 +310,6 @@ class VersionBuilder::Rep {
          TEST_SYNC_POINT_CALLBACK("VersionBuilder::CheckConsistency0", &pair);
 #endif
          if (!level_zero_cmp_(f1, f2)) {
-            fprintf(stderr, "L0 files are not sorted properly");
            return Status::Corruption("L0 files are not sorted properly");
          }

@@ -287,11 +318,6 @@ class VersionBuilder::Rep {
            SequenceNumber external_file_seqno = f2->fd.smallest_seqno;
            if (!(external_file_seqno < f1->fd.largest_seqno ||
                  external_file_seqno == 0)) {
-              fprintf(stderr,
-                      "L0 file with seqno %" PRIu64 " %" PRIu64
-                      " vs. file with global_seqno %" PRIu64 "\n",
-                      f1->fd.smallest_seqno, f1->fd.largest_seqno,
-                      external_file_seqno);
              return Status::Corruption(
                  "L0 file with seqno " +
                  NumberToString(f1->fd.smallest_seqno) + " " +
@@ -301,11 +327,6 @@ class VersionBuilder::Rep {
                  NumberToString(f1->fd.GetNumber()));
            }
          } else if (f1->fd.smallest_seqno <= f2->fd.smallest_seqno) {
-            fprintf(stderr,
-                    "L0 files seqno %" PRIu64 " %" PRIu64 " vs. %" PRIu64
-                    " %" PRIu64 "\n",
-                    f1->fd.smallest_seqno, f1->fd.largest_seqno,
-                    f2->fd.smallest_seqno, f2->fd.largest_seqno);
            return Status::Corruption(
                "L0 files seqno " + NumberToString(f1->fd.smallest_seqno) +
                " " + NumberToString(f1->fd.largest_seqno) + " " +
@@ -320,7 +341,6 @@ class VersionBuilder::Rep {
          TEST_SYNC_POINT_CALLBACK("VersionBuilder::CheckConsistency1", &pair);
 #endif
          if (!level_nonzero_cmp_(f1, f2)) {
-            fprintf(stderr, "L%d files are not sorted properly", level);
            return Status::Corruption("L" + NumberToString(level) +
                                      " files are not sorted properly");
          }
@@ -328,9 +348,6 @@ class VersionBuilder::Rep {
          // Make sure there is no overlap in levels > 0
          if (vstorage->InternalComparator()->Compare(f1->largest,
                                                      f2->smallest) >= 0) {
-            fprintf(stderr, "L%d have overlapping ranges %s vs. %s\n", level,
-                    (f1->largest).DebugString(true).c_str(),
-                    (f2->smallest).DebugString(true).c_str());
            return Status::Corruption(
                "L" + NumberToString(level) + " have overlapping ranges " +
                (f1->largest).DebugString(true) + " vs. " +
@@ -343,17 +360,27 @@ class VersionBuilder::Rep {
    // Make sure that all blob files in the version have non-garbage data.
    const auto& blob_files = vstorage->GetBlobFiles();
    for (const auto& pair : blob_files) {
+      const uint64_t blob_file_number = pair.first;
      const auto& blob_file_meta = pair.second;
      assert(blob_file_meta);

      if (blob_file_meta->GetGarbageBlobCount() >=
          blob_file_meta->GetTotalBlobCount()) {
        std::ostringstream oss;
-        oss << "Blob file #" << blob_file_meta->GetBlobFileNumber()
+        oss << "Blob file #" << blob_file_number
            << " consists entirely of garbage";

        return Status::Corruption("VersionBuilder", oss.str());
      }
+
+      if (blob_file_meta->GetLinkedSsts() !=
+          expected_linked_ssts[blob_file_number]) {
+        std::ostringstream oss;
+        oss << "Links are inconsistent between table files and blob file #"
+            << blob_file_number;
+
+        return Status::Corruption("VersionBuilder", oss.str());
+      }
    }

    Status ret_s;
@@ -445,6 +472,28 @@ class VersionBuilder::Rep {
    return base_vstorage_->GetFileLocation(file_number).GetLevel();
  }

+  uint64_t GetOldestBlobFileNumberForTableFile(int level,
+                                               uint64_t file_number) const {
+    assert(level < num_levels_);
+
+    const auto& added_files = levels_[level].added_files;
+
+    auto it = added_files.find(file_number);
+    if (it != added_files.end()) {
+      const FileMetaData* const meta = it->second;
+      assert(meta);
+
+      return meta->oldest_blob_file_number;
+    }
+
+    assert(base_vstorage_);
+    const FileMetaData* const meta =
+        base_vstorage_->GetFileMetaDataByNumber(file_number);
+    assert(meta);
+
+    return meta->oldest_blob_file_number;
+  }
+
  Status ApplyFileDeletion(int level, uint64_t file_number) {
    assert(level != VersionStorageInfo::FileLocation::Invalid().GetLevel());

@@ -478,6 +527,14 @@ class VersionBuilder::Rep {
      return Status::OK();
    }

+    const uint64_t blob_file_number =
+        GetOldestBlobFileNumberForTableFile(level, file_number);
+
+    if (blob_file_number != kInvalidBlobFileNumber &&
+        IsBlobFileInVersion(blob_file_number)) {
+      blob_file_meta_deltas_[blob_file_number].UnlinkSst(file_number);
+    }
+
    auto& level_state = levels_[level];

    auto& add_files = level_state.added_files;
@@ -485,12 +542,12 @@ class VersionBuilder::Rep {
    if (add_it != add_files.end()) {
      UnrefFile(add_it->second);
      add_files.erase(add_it);
-    } else {
-      auto& del_files = level_state.deleted_files;
-      assert(del_files.find(file_number) == del_files.end());
-      del_files.emplace(file_number);
    }

+    auto& del_files = level_state.deleted_files;
+    assert(del_files.find(file_number) == del_files.end());
+    del_files.emplace(file_number);
+
    table_file_levels_[file_number] =
        VersionStorageInfo::FileLocation::Invalid().GetLevel();

@@ -529,13 +586,20 @@ class VersionBuilder::Rep {
    auto del_it = del_files.find(file_number);
    if (del_it != del_files.end()) {
      del_files.erase(del_it);
-    } else {
-      FileMetaData* const f = new FileMetaData(meta);
-      f->refs = 1;
+    }

-      auto& add_files = level_state.added_files;
-      assert(add_files.find(file_number) == add_files.end());
-      add_files.emplace(file_number, f);
+    FileMetaData* const f = new FileMetaData(meta);
+    f->refs = 1;
+
+    auto& add_files = level_state.added_files;
+    assert(add_files.find(file_number) == add_files.end());
+    add_files.emplace(file_number, f);
+
+    const uint64_t blob_file_number = f->oldest_blob_file_number;
+
+    if (blob_file_number != kInvalidBlobFileNumber &&
+        IsBlobFileInVersion(blob_file_number)) {
+      blob_file_meta_deltas_[blob_file_number].LinkSst(file_number);
    }

    table_file_levels_[file_number] = level;
@@ -552,27 +616,9 @@ class VersionBuilder::Rep {
      }
    }

-    // Delete files
-    for (const auto& deleted_file : edit->GetDeletedFiles()) {
-      const int level = deleted_file.first;
-      const uint64_t file_number = deleted_file.second;
-
-      const Status s = ApplyFileDeletion(level, file_number);
-      if (!s.ok()) {
-        return s;
-      }
-    }
-
-    // Add new files
-    for (const auto& new_file : edit->GetNewFiles()) {
-      const int level = new_file.first;
-      const FileMetaData& meta = new_file.second;
-
-      const Status s = ApplyFileAddition(level, meta);
-      if (!s.ok()) {
-        return s;
-      }
-    }
+    // Note: we process the blob file related changes first because the
+    // table file addition/deletion logic depends on the blob files
+    // already being there.

    // Add new blob files
    for (const auto& blob_file_addition : edit->GetBlobFileAdditions()) {
@@ -590,17 +636,62 @@ class VersionBuilder::Rep {
      }
    }

+    // Delete table files
+    for (const auto& deleted_file : edit->GetDeletedFiles()) {
+      const int level = deleted_file.first;
+      const uint64_t file_number = deleted_file.second;
+
+      const Status s = ApplyFileDeletion(level, file_number);
+      if (!s.ok()) {
+        return s;
+      }
+    }
+
+    // Add new table files
+    for (const auto& new_file : edit->GetNewFiles()) {
+      const int level = new_file.first;
+      const FileMetaData& meta = new_file.second;
+
+      const Status s = ApplyFileAddition(level, meta);
+      if (!s.ok()) {
+        return s;
+      }
+    }
+
    return Status::OK();
  }

+  static BlobFileMetaData::LinkedSsts ApplyLinkedSstChanges(
+      const BlobFileMetaData::LinkedSsts& base,
+      const std::unordered_set<uint64_t>& newly_linked,
+      const std::unordered_set<uint64_t>& newly_unlinked) {
+    BlobFileMetaData::LinkedSsts result(base);
+
+    for (uint64_t sst_file_number : newly_unlinked) {
+      assert(result.find(sst_file_number) != result.end());
+
+      result.erase(sst_file_number);
+    }
+
+    for (uint64_t sst_file_number : newly_linked) {
+      assert(result.find(sst_file_number) == result.end());
+
+      result.emplace(sst_file_number);
+    }
+
+    return result;
+  }
+
  static std::shared_ptr<BlobFileMetaData> CreateMetaDataForNewBlobFile(
      const BlobFileMetaDataDelta& delta) {
    auto shared_meta = delta.GetSharedMeta();
    assert(shared_meta);

-    auto meta = BlobFileMetaData::Create(std::move(shared_meta),
-                                         delta.GetAdditionalGarbageCount(),
-                                         delta.GetAdditionalGarbageBytes());
+    assert(delta.GetNewlyUnlinkedSsts().empty());
+
+    auto meta = BlobFileMetaData::Create(
+        std::move(shared_meta), delta.GetNewlyLinkedSsts(),
+        delta.GetAdditionalGarbageCount(), delta.GetAdditionalGarbageBytes());

    return meta;
  }
@@ -619,8 +710,12 @@ class VersionBuilder::Rep {
    auto shared_meta = base_meta->GetSharedMeta();
    assert(shared_meta);

+    auto linked_ssts = ApplyLinkedSstChanges(base_meta->GetLinkedSsts(),
+                                             delta.GetNewlyLinkedSsts(),
+                                             delta.GetNewlyUnlinkedSsts());
+
    auto meta = BlobFileMetaData::Create(
-        std::move(shared_meta),
+        std::move(shared_meta), std::move(linked_ssts),
        base_meta->GetGarbageBlobCount() + delta.GetAdditionalGarbageCount(),
        base_meta->GetGarbageBlobBytes() + delta.GetAdditionalGarbageBytes());

@@ -633,7 +728,8 @@ class VersionBuilder::Rep {
    assert(vstorage);
    assert(meta);

-    if (meta->GetGarbageBlobCount() < meta->GetTotalBlobCount()) {
+    if (meta->GetGarbageBlobCount() < meta->GetTotalBlobCount() ||
+        !meta->GetLinkedSsts().empty()) {
      vstorage->AddBlobFile(meta);
    }
  }
@@ -769,7 +865,8 @@ class VersionBuilder::Rep {
  Status LoadTableHandlers(InternalStats* internal_stats, int max_threads,
                           bool prefetch_index_and_filter_in_cache,
                           bool is_initial_load,
-                           const SliceTransform* prefix_extractor) {
+                           const SliceTransform* prefix_extractor,
+                           size_t max_file_size_for_l0_meta_pin) {
    assert(table_cache_ != nullptr);

    size_t table_cache_capacity = table_cache_->get_cache()->GetCapacity();
@@ -838,7 +935,7 @@ class VersionBuilder::Rep {
            file_meta->fd, &file_meta->table_reader_handle, prefix_extractor,
            false /*no_io */, true /* record_read_stats */,
            internal_stats->GetFileReadHist(level), false, level,
-            prefetch_index_and_filter_in_cache);
+            prefetch_index_and_filter_in_cache, max_file_size_for_l0_meta_pin);
        if (file_meta->table_reader_handle != nullptr) {
          // Load table_reader
          file_meta->fd.table_reader = table_cache_->GetTableReaderFromHandle(
@@ -864,12 +961,28 @@ class VersionBuilder::Rep {
  }

  void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f) {
-    if (levels_[level].deleted_files.count(f->fd.GetNumber()) > 0) {
+    const uint64_t file_number = f->fd.GetNumber();
+
+    const auto& level_state = levels_[level];
+
+    const auto& del_files = level_state.deleted_files;
+    const auto del_it = del_files.find(file_number);
+
+    if (del_it != del_files.end()) {
      // f is to-be-deleted table file
      vstorage->RemoveCurrentStats(f);
    } else {
-      assert(ioptions_);
-      vstorage->AddFile(level, f, ioptions_->info_log);
+      const auto& add_files = level_state.added_files;
+      const auto add_it = add_files.find(file_number);
+
+      // Note: if the file appears both in the base version and in the added
+      // list, the added FileMetaData supersedes the one in the base version.
+      if (add_it != add_files.end() && add_it->second != f) {
+        vstorage->RemoveCurrentStats(f);
+      } else {
+        assert(ioptions_);
+        vstorage->AddFile(level, f, ioptions_->info_log);
+      }
    }
  }
 };
@@ -897,10 +1010,11 @@ Status VersionBuilder::SaveTo(VersionStorageInfo* vstorage) {
 Status VersionBuilder::LoadTableHandlers(
    InternalStats* internal_stats, int max_threads,
    bool prefetch_index_and_filter_in_cache, bool is_initial_load,
-    const SliceTransform* prefix_extractor) {
-  return rep_->LoadTableHandlers(internal_stats, max_threads,
-                                 prefetch_index_and_filter_in_cache,
-                                 is_initial_load, prefix_extractor);
+    const SliceTransform* prefix_extractor,
+    size_t max_file_size_for_l0_meta_pin) {
+  return rep_->LoadTableHandlers(
+      internal_stats, max_threads, prefetch_index_and_filter_in_cache,
+      is_initial_load, prefix_extractor, max_file_size_for_l0_meta_pin);
 }

 BaseReferencedVersionBuilder::BaseReferencedVersionBuilder(
@@ -42,7 +42,8 @@ class VersionBuilder {
  Status LoadTableHandlers(InternalStats* internal_stats, int max_threads,
                           bool prefetch_index_and_filter_in_cache,
                           bool is_initial_load,
-                           const SliceTransform* prefix_extractor);
+                           const SliceTransform* prefix_extractor,
+                           size_t max_file_size_for_l0_meta_pin);

 private:
  class Rep;
@@ -4,8 +4,11 @@
 //  (found in the LICENSE.Apache file in the root directory).

 #include <cstring>
+#include <iomanip>
 #include <memory>
+#include <sstream>
 #include <string>
+
 #include "db/version_edit.h"
 #include "db/version_set.h"
 #include "logging/logging.h"
@@ -80,13 +83,15 @@ class VersionBuilderTest : public testing::Test {

  void AddBlob(uint64_t blob_file_number, uint64_t total_blob_count,
               uint64_t total_blob_bytes, std::string checksum_method,
-               std::string checksum_value, uint64_t garbage_blob_count,
-               uint64_t garbage_blob_bytes) {
+               std::string checksum_value,
+               BlobFileMetaData::LinkedSsts linked_ssts,
+               uint64_t garbage_blob_count, uint64_t garbage_blob_bytes) {
    auto shared_meta = SharedBlobFileMetaData::Create(
        blob_file_number, total_blob_count, total_blob_bytes,
        std::move(checksum_method), std::move(checksum_value));
-    auto meta = BlobFileMetaData::Create(
-        std::move(shared_meta), garbage_blob_count, garbage_blob_bytes);
+    auto meta =
+        BlobFileMetaData::Create(std::move(shared_meta), std::move(linked_ssts),
+                                 garbage_blob_count, garbage_blob_bytes);

    vstorage_.AddBlobFile(std::move(meta));
  }
@@ -422,8 +427,19 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) {
  constexpr uint64_t file_number = 2345;
  constexpr char smallest[] = "bar";
  constexpr char largest[] = "foo";
+  constexpr uint64_t file_size = 10000;
+  constexpr uint32_t path_id = 0;
+  constexpr SequenceNumber smallest_seq = 100;
+  constexpr SequenceNumber largest_seq = 500;
+  constexpr uint64_t num_entries = 0;
+  constexpr uint64_t num_deletions = 0;
+  constexpr bool sampled = false;
+  constexpr SequenceNumber smallest_seqno = 1;
+  constexpr SequenceNumber largest_seqno = 1000;

-  Add(level, file_number, smallest, largest);
+  Add(level, file_number, smallest, largest, file_size, path_id, smallest_seq,
+      largest_seq, num_entries, num_deletions, sampled, smallest_seqno,
+      largest_seqno);

  EnvOptions env_options;
  constexpr TableCache* table_cache = nullptr;
@@ -440,18 +456,14 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) {

  VersionEdit addition;

-  constexpr uint32_t path_id = 0;
-  constexpr uint64_t file_size = 10000;
-  constexpr SequenceNumber smallest_seqno = 100;
-  constexpr SequenceNumber largest_seqno = 1000;
  constexpr bool marked_for_compaction = false;

  addition.AddFile(level, file_number, path_id, file_size,
-                   GetInternalKey(smallest), GetInternalKey(largest),
-                   smallest_seqno, largest_seqno, marked_for_compaction,
-                   kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
-                   kUnknownFileCreationTime, kUnknownFileChecksum,
-                   kUnknownFileChecksumFuncName);
+                   GetInternalKey(smallest, smallest_seq),
+                   GetInternalKey(largest, largest_seq), smallest_seqno,
+                   largest_seqno, marked_for_compaction, kInvalidBlobFileNumber,
+                   kUnknownOldestAncesterTime, kUnknownFileCreationTime,
+                   kUnknownFileChecksum, kUnknownFileChecksumFuncName);

  ASSERT_OK(builder.Apply(&addition));

@@ -635,6 +647,7 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAddition) {
  ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes);
  ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method);
  ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value);
+  ASSERT_TRUE(new_meta->GetLinkedSsts().empty());
  ASSERT_EQ(new_meta->GetGarbageBlobCount(), 0);
  ASSERT_EQ(new_meta->GetGarbageBlobBytes(), 0);
 }
@@ -651,7 +664,8 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyInBase) {
  constexpr uint64_t garbage_blob_bytes = 456789;

  AddBlob(blob_file_number, total_blob_count, total_blob_bytes, checksum_method,
-          checksum_value, garbage_blob_count, garbage_blob_bytes);
+          checksum_value, BlobFileMetaData::LinkedSsts(), garbage_blob_count,
+          garbage_blob_bytes);

  EnvOptions env_options;
  constexpr TableCache* table_cache = nullptr;
@@ -710,7 +724,8 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) {
  constexpr uint64_t garbage_blob_bytes = 456789;

  AddBlob(blob_file_number, total_blob_count, total_blob_bytes, checksum_method,
-          checksum_value, garbage_blob_count, garbage_blob_bytes);
+          checksum_value, BlobFileMetaData::LinkedSsts(), garbage_blob_count,
+          garbage_blob_bytes);

  const auto meta =
      GetBlobFileMetaData(vstorage_.GetBlobFiles(), blob_file_number);
@@ -752,6 +767,7 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) {
  ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes);
  ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method);
  ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value);
+  ASSERT_TRUE(new_meta->GetLinkedSsts().empty());
  ASSERT_EQ(new_meta->GetGarbageBlobCount(),
            garbage_blob_count + new_garbage_blob_count);
  ASSERT_EQ(new_meta->GetGarbageBlobBytes(),
@@ -809,6 +825,7 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileAdditionApplied) {
  ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes);
  ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method);
  ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value);
+  ASSERT_TRUE(new_meta->GetLinkedSsts().empty());
  ASSERT_EQ(new_meta->GetGarbageBlobCount(), garbage_blob_count);
  ASSERT_EQ(new_meta->GetGarbageBlobBytes(), garbage_blob_bytes);
 }
@@ -849,8 +866,8 @@ TEST_F(VersionBuilderTest, SaveBlobFilesTo) {

    AddBlob(blob_file_number, total_blob_count, total_blob_bytes,
            /* checksum_method */ std::string(),
-            /* checksum_value */ std::string(), garbage_blob_count,
-            garbage_blob_bytes);
+            /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts(),
+            garbage_blob_count, garbage_blob_bytes);
  }

  EnvOptions env_options;
@@ -939,7 +956,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
  AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000,
          /* total_blob_bytes */ 1000000,
          /* checksum_method */ std::string(),
-          /* checksum_value */ std::string(),
+          /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1},
          /* garbage_blob_count */ 500, /* garbage_blob_bytes */ 300000);

  UpdateVersionStorageInfo();
@@ -988,9 +1005,9 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
  UnrefFilesInVersion(&new_vstorage);
 }

-TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesNotInVersion) {
-  // Initialize base version. The table file points to a blob file that is
-  // not in this version.
+TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesInconsistentLinks) {
+  // Initialize base version. Links between the table file and the blob file
+  // are inconsistent.

  Add(/* level */ 1, /* file_number */ 1, /* smallest */ "150",
      /* largest */ "200", /* file_size */ 100,
@@ -1002,7 +1019,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesNotInVersion) {
  AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000,
          /* total_blob_bytes */ 1000000,
          /* checksum_method */ std::string(),
-          /* checksum_value */ std::string(),
+          /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1},
          /* garbage_blob_count */ 500, /* garbage_blob_bytes */ 300000);

  UpdateVersionStorageInfo();
@@ -1022,8 +1039,9 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesNotInVersion) {

  const Status s = builder.SaveTo(&new_vstorage);
  ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(
-      std::strstr(s.getState(), "Blob file #256 is not part of this version"));
+  ASSERT_TRUE(std::strstr(
+      s.getState(),
+      "Links are inconsistent between table files and blob file #16"));

  UnrefFilesInVersion(&new_vstorage);
 }
@@ -1042,7 +1060,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbage) {
  AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000,
          /* total_blob_bytes */ 1000000,
          /* checksum_method */ std::string(),
-          /* checksum_value */ std::string(),
+          /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1},
          /* garbage_blob_count */ 1000, /* garbage_blob_bytes */ 1000000);

  UpdateVersionStorageInfo();
@@ -1068,6 +1086,224 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbage) {
  UnrefFilesInVersion(&new_vstorage);
 }

+TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbageLinkedSsts) {
+  // Initialize base version, with a table file pointing to a blob file
+  // that has no garbage at this point.
+
+  Add(/* level */ 1, /* file_number */ 1, /* smallest */ "150",
+      /* largest */ "200", /* file_size */ 100,
+      /* path_id */ 0, /* smallest_seq */ 100, /* largest_seq */ 100,
+      /* num_entries */ 0, /* num_deletions */ 0,
+      /* sampled */ false, /* smallest_seqno */ 100, /* largest_seqno */ 100,
+      /* oldest_blob_file_number */ 16);
+
+  AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000,
+          /* total_blob_bytes */ 1000000,
+          /* checksum_method */ std::string(),
+          /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1},
+          /* garbage_blob_count */ 0, /* garbage_blob_bytes */ 0);
+
+  UpdateVersionStorageInfo();
+
+  // Mark the entire blob file garbage but do not remove the linked SST.
+  EnvOptions env_options;
+  constexpr TableCache* table_cache = nullptr;
+  constexpr VersionSet* version_set = nullptr;
+
+  VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_,
+                         version_set);
+
+  VersionEdit edit;
+
+  edit.AddBlobFileGarbage(/* blob_file_number */ 16,
+                          /* garbage_blob_count */ 1000,
+                          /* garbage_blob_bytes */ 1000000);
+
+  ASSERT_OK(builder.Apply(&edit));
+
+  // Save to a new version in order to trigger consistency checks.
+  constexpr bool force_consistency_checks = true;
+  VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
+                                  kCompactionStyleLevel, &vstorage_,
+                                  force_consistency_checks);
+
+  const Status s = builder.SaveTo(&new_vstorage);
+  ASSERT_TRUE(s.IsCorruption());
+  ASSERT_TRUE(
+      std::strstr(s.getState(), "Blob file #16 consists entirely of garbage"));
+
+  UnrefFilesInVersion(&new_vstorage);
+}
+
+TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
+  // Initialize base version. Table files 1..10 are linked to blob files 1..5,
+  // while table files 11..20 are not linked to any blob files.
+
+  for (uint64_t i = 1; i <= 10; ++i) {
+    std::ostringstream oss;
+    oss << std::setw(2) << std::setfill('0') << i;
+
+    const std::string key = oss.str();
+
+    Add(/* level */ 1, /* file_number */ i, /* smallest */ key.c_str(),
+        /* largest */ key.c_str(), /* file_size */ 100,
+        /* path_id */ 0, /* smallest_seq */ i * 100, /* largest_seq */ i * 100,
+        /* num_entries */ 0, /* num_deletions */ 0,
+        /* sampled */ false, /* smallest_seqno */ i * 100,
+        /* largest_seqno */ i * 100,
+        /* oldest_blob_file_number */ ((i - 1) % 5) + 1);
+  }
+
+  for (uint64_t i = 1; i <= 5; ++i) {
+    AddBlob(/* blob_file_number */ i, /* total_blob_count */ 2000,
+            /* total_blob_bytes */ 2000000,
+            /* checksum_method */ std::string(),
+            /* checksum_value */ std::string(),
+            BlobFileMetaData::LinkedSsts{i, i + 5},
+            /* garbage_blob_count */ 1000, /* garbage_blob_bytes */ 1000000);
+  }
+
+  for (uint64_t i = 11; i <= 20; ++i) {
+    std::ostringstream oss;
+    oss << std::setw(2) << std::setfill('0') << i;
+
+    const std::string key = oss.str();
+
+    Add(/* level */ 1, /* file_number */ i, /* smallest */ key.c_str(),
+        /* largest */ key.c_str(), /* file_size */ 100,
+        /* path_id */ 0, /* smallest_seq */ i * 100, /* largest_seq */ i * 100,
+        /* num_entries */ 0, /* num_deletions */ 0,
+        /* sampled */ false, /* smallest_seqno */ i * 100,
+        /* largest_seqno */ i * 100, kInvalidBlobFileNumber);
+  }
+
+  UpdateVersionStorageInfo();
+
+  {
+    const auto& blob_files = vstorage_.GetBlobFiles();
+    ASSERT_EQ(blob_files.size(), 5);
+
+    const std::vector<BlobFileMetaData::LinkedSsts> expected_linked_ssts{
+        {1, 6}, {2, 7}, {3, 8}, {4, 9}, {5, 10}};
+
+    for (size_t i = 0; i < 5; ++i) {
+      const auto meta =
+          GetBlobFileMetaData(blob_files, /* blob_file_number */ i + 1);
+      ASSERT_NE(meta, nullptr);
+      ASSERT_EQ(meta->GetLinkedSsts(), expected_linked_ssts[i]);
+    }
+  }
+
+  VersionEdit edit;
+
+  // Add an SST that references a blob file.
+  edit.AddFile(
+      /* level */ 1, /* file_number */ 21, /* path_id */ 0,
+      /* file_size */ 100, /* smallest */ GetInternalKey("21", 2100),
+      /* largest */ GetInternalKey("21", 2100), /* smallest_seqno */ 2100,
+      /* largest_seqno */ 2100, /* marked_for_compaction */ false,
+      /* oldest_blob_file_number */ 1, kUnknownOldestAncesterTime,
+      kUnknownFileCreationTime, kUnknownFileChecksum,
+      kUnknownFileChecksumFuncName);
+
+  // Add an SST that does not reference any blob files.
+  edit.AddFile(
+      /* level */ 1, /* file_number */ 22, /* path_id */ 0,
+      /* file_size */ 100, /* smallest */ GetInternalKey("22", 2200),
+      /* largest */ GetInternalKey("22", 2200), /* smallest_seqno */ 2200,
+      /* largest_seqno */ 2200, /* marked_for_compaction */ false,
+      kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
+      kUnknownFileCreationTime, kUnknownFileChecksum,
+      kUnknownFileChecksumFuncName);
+
+  // Delete a file that references a blob file.
+  edit.DeleteFile(/* level */ 1, /* file_number */ 6);
+
+  // Delete a file that does not reference any blob files.
+  edit.DeleteFile(/* level */ 1, /* file_number */ 16);
+
+  // Trivially move a file that references a blob file. Note that we save
+  // the original BlobFileMetaData object so we can check that no new object
+  // gets created.
+  auto meta3 =
+      GetBlobFileMetaData(vstorage_.GetBlobFiles(), /* blob_file_number */ 3);
+
+  edit.DeleteFile(/* level */ 1, /* file_number */ 3);
+  edit.AddFile(/* level */ 2, /* file_number */ 3, /* path_id */ 0,
+               /* file_size */ 100, /* smallest */ GetInternalKey("03", 300),
+               /* largest */ GetInternalKey("03", 300),
+               /* smallest_seqno */ 300,
+               /* largest_seqno */ 300, /* marked_for_compaction */ false,
+               /* oldest_blob_file_number */ 3, kUnknownOldestAncesterTime,
+               kUnknownFileCreationTime, kUnknownFileChecksum,
+               kUnknownFileChecksumFuncName);
+
+  // Trivially move a file that does not reference any blob files.
+  edit.DeleteFile(/* level */ 1, /* file_number */ 13);
+  edit.AddFile(/* level */ 2, /* file_number */ 13, /* path_id */ 0,
+               /* file_size */ 100, /* smallest */ GetInternalKey("13", 1300),
+               /* largest */ GetInternalKey("13", 1300),
+               /* smallest_seqno */ 1300,
+               /* largest_seqno */ 1300, /* marked_for_compaction */ false,
+               kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
+               kUnknownFileCreationTime, kUnknownFileChecksum,
+               kUnknownFileChecksumFuncName);
+
+  // Add one more SST file that references a blob file, then promptly
+  // delete it in a second version edit before the new version gets saved.
+  // This file should not show up as linked to the blob file in the new version.
+  edit.AddFile(/* level */ 1, /* file_number */ 23, /* path_id */ 0,
+               /* file_size */ 100, /* smallest */ GetInternalKey("23", 2300),
+               /* largest */ GetInternalKey("23", 2300),
+               /* smallest_seqno */ 2300,
+               /* largest_seqno */ 2300, /* marked_for_compaction */ false,
+               /* oldest_blob_file_number */ 5, kUnknownOldestAncesterTime,
+               kUnknownFileCreationTime, kUnknownFileChecksum,
+               kUnknownFileChecksumFuncName);
+
+  VersionEdit edit2;
+
+  edit2.DeleteFile(/* level */ 1, /* file_number */ 23);
+
+  EnvOptions env_options;
+  constexpr TableCache* table_cache = nullptr;
+  constexpr VersionSet* version_set = nullptr;
+
+  VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_,
+                         version_set);
+
+  ASSERT_OK(builder.Apply(&edit));
+  ASSERT_OK(builder.Apply(&edit2));
+
+  constexpr bool force_consistency_checks = true;
+  VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
+                                  kCompactionStyleLevel, &vstorage_,
+                                  force_consistency_checks);
+
+  ASSERT_OK(builder.SaveTo(&new_vstorage));
+
+  {
+    const auto& blob_files = new_vstorage.GetBlobFiles();
+    ASSERT_EQ(blob_files.size(), 5);
+
+    const std::vector<BlobFileMetaData::LinkedSsts> expected_linked_ssts{
+        {1, 21}, {2, 7}, {3, 8}, {4, 9}, {5, 10}};
+
+    for (size_t i = 0; i < 5; ++i) {
+      const auto meta =
+          GetBlobFileMetaData(blob_files, /* blob_file_number */ i + 1);
+      ASSERT_NE(meta, nullptr);
+      ASSERT_EQ(meta->GetLinkedSsts(), expected_linked_ssts[i]);
+    }
+
+    // Make sure that no new BlobFileMetaData got created for the blob file
+    // affected by the trivial move.
+    ASSERT_EQ(GetBlobFileMetaData(blob_files, /* blob_file_number */ 3), meta3);
+  }
+
+  UnrefFilesInVersion(&new_vstorage);
+}
+
 TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) {
  Add(0, 1U, "150", "200", 100U);
  UpdateVersionStorageInfo();
@@ -18,10 +18,6 @@
 #include "util/string_util.h"

 namespace ROCKSDB_NAMESPACE {
-// The unknown file checksum.
-const std::string kUnknownFileChecksum("");
-// The unknown sst file checksum function name.
-const std::string kUnknownFileChecksumFuncName("Unknown");

 namespace {

@@ -29,9 +29,6 @@ constexpr uint64_t kFileNumberMask = 0x3FFFFFFFFFFFFFFF;
 constexpr uint64_t kUnknownOldestAncesterTime = 0;
 constexpr uint64_t kUnknownFileCreationTime = 0;

-extern const std::string kUnknownFileChecksum;
-extern const std::string kUnknownFileChecksumFuncName;
-
 extern uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id);

 // A copyable structure contains information needed to read data from an SST
@@ -27,12 +27,17 @@ VersionEditHandler::VersionEditHandler(
  assert(version_set_ != nullptr);
 }

-Status VersionEditHandler::Iterate(log::Reader& reader, std::string* db_id) {
+void VersionEditHandler::Iterate(log::Reader& reader, Status* log_read_status,
+                                 std::string* db_id) {
  Slice record;
  std::string scratch;
+  assert(log_read_status);
+  assert(log_read_status->ok());
+
  size_t recovered_edits = 0;
  Status s = Initialize();
-  while (reader.ReadRecord(&record, &scratch) && s.ok()) {
+  while (s.ok() && reader.ReadRecord(&record, &scratch) &&
+         log_read_status->ok()) {
    VersionEdit edit;
    s = edit.DecodeFrom(record);
    if (!s.ok()) {
@@ -70,13 +75,15 @@ Status VersionEditHandler::Iterate(log::Reader& reader, std::string* db_id) {
      }
    }
  }
+  if (!log_read_status->ok()) {
+    s = *log_read_status;
+  }

  CheckIterationResult(reader, &s);

  if (!s.ok()) {
    status_ = s;
  }
-  return s;
 }

 Status VersionEditHandler::Initialize() {
@@ -404,17 +411,17 @@ Status VersionEditHandler::LoadTables(ColumnFamilyData* cfd,
                                      bool is_initial_load) {
  assert(cfd != nullptr);
  assert(!cfd->IsDropped());
-  Status s;
  auto builder_iter = builders_.find(cfd->GetID());
  assert(builder_iter != builders_.end());
  assert(builder_iter->second != nullptr);
  VersionBuilder* builder = builder_iter->second->version_builder();
  assert(builder);
-  s = builder->LoadTableHandlers(
+  Status s = builder->LoadTableHandlers(
      cfd->internal_stats(),
      version_set_->db_options_->max_file_opening_threads,
      prefetch_index_and_filter_in_cache, is_initial_load,
-      cfd->GetLatestMutableCFOptions()->prefix_extractor.get());
+      cfd->GetLatestMutableCFOptions()->prefix_extractor.get(),
+      MaxFileSizeForL0MetaPin(*cfd->GetLatestMutableCFOptions()));
  if ((s.IsPathNotFound() || s.IsCorruption()) &&
      no_error_if_table_files_missing_) {
    s = Status::OK();
@@ -550,6 +557,8 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersion(
    if (s.IsPathNotFound() || s.IsNotFound() || s.IsCorruption()) {
      missing_files.insert(file_num);
      s = Status::OK();
+    } else if (!s.ok()) {
+      break;
    }
  }
  bool missing_info = !version_edit_params_.has_log_number_ ||
@@ -557,8 +566,9 @@ Status VersionEditHandlerPointInTime::MaybeCreateVersion(
                      !version_edit_params_.has_last_sequence_;

  // Create version before apply edit
-  if (!missing_info && ((!missing_files.empty() && !prev_has_missing_files) ||
-                        (missing_files.empty() && force_create_version))) {
+  if (s.ok() && !missing_info &&
+      ((!missing_files.empty() && !prev_has_missing_files) ||
+       (missing_files.empty() && force_create_version))) {
    auto builder_iter = builders_.find(cfd->GetID());
    assert(builder_iter != builders_.end());
    auto* builder = builder_iter->second->version_builder();
@@ -40,7 +40,8 @@ class VersionEditHandler {

  virtual ~VersionEditHandler() {}

-  Status Iterate(log::Reader& reader, std::string* db_id);
+  void Iterate(log::Reader& reader, Status* log_read_status,
+               std::string* db_id);

  const Status& status() const { return status_; }

@@ -10,6 +10,7 @@
 #include "db/version_set.h"

 #include <stdio.h>
+
 #include <algorithm>
 #include <array>
 #include <cinttypes>
@@ -19,6 +20,7 @@
 #include <string>
 #include <unordered_map>
 #include <vector>
+
 #include "compaction/compaction.h"
 #include "db/internal_stats.h"
 #include "db/log_reader.h"
@@ -50,6 +52,7 @@
 #include "table/table_reader.h"
 #include "table/two_level_iterator.h"
 #include "test_util/sync_point.h"
+#include "util/cast_util.h"
 #include "util/coding.h"
 #include "util/stop_watch.h"
 #include "util/string_util.h"
@@ -365,6 +368,7 @@ class FilePickerMultiGet {
        range_(range),
        batch_iter_(range->begin()),
        batch_iter_prev_(range->begin()),
+        upper_key_(range->begin()),
        maybe_repeat_key_(false),
        current_level_range_(*range, range->begin(), range->end()),
        current_file_range_(*range, range->begin(), range->end()),
@@ -481,9 +485,19 @@ class FilePickerMultiGet {
      }
      if (cmp_largest == 0) {
        // cmp_largest is 0, which means the next key will not be in this
-        // file, so stop looking further. Also don't increment megt_iter_
-        // as we may have to look for this key in the next file if we don't
-        // find it in this one
+        // file, so stop looking further. However, its possible there are
+        // duplicates in the batch, so find the upper bound for the batch
+        // in this file (upper_key_) by skipping past the duplicates. We
+        // leave batch_iter_ as is since we may have to pick up from there
+        // for the next file, if this file has a merge value rather than
+        // final value
+        upper_key_ = batch_iter_;
+        ++upper_key_;
+        while (upper_key_ != current_level_range_.end() &&
+               user_comparator_->Compare(batch_iter_->ukey, upper_key_->ukey) ==
+                   0) {
+          ++upper_key_;
+        }
        break;
      } else {
        if (curr_level_ == 0) {
@@ -503,6 +517,12 @@ class FilePickerMultiGet {
    *fd = f;
    *file_index = curr_file_index;
    *is_last_key_in_file = cmp_largest == 0;
+    if (!*is_last_key_in_file) {
+      // If the largest key in the batch overlapping the file is not the
+      // largest key in the file, upper_ley_ would not have been updated so
+      // update it here
+      upper_key_ = batch_iter_;
+    }
    return file_hit;
  }

@@ -524,7 +544,7 @@ class FilePickerMultiGet {
          // file regardless for all keys not found yet
          if (current_level_range_.CheckKeyDone(batch_iter_) ||
              curr_level_ == 0) {
-            ++batch_iter_;
+            batch_iter_ = upper_key_;
          }
        }
        // batch_iter_prev_ will become the start key for the next file
@@ -544,18 +564,20 @@ class FilePickerMultiGet {
                                      &is_last_key_in_file)) {
        search_ended_ = !PrepareNextLevel();
      } else {
-        MultiGetRange::Iterator upper_key = batch_iter_;
        if (is_last_key_in_file) {
          // Since cmp_largest is 0, batch_iter_ still points to the last key
          // that falls in this file, instead of the next one. Increment
-          // upper_key so we can set the range properly for SST MultiGet
-          ++upper_key;
-          ++(fp_ctx_array_[batch_iter_.index()].curr_index_in_curr_level);
+          // the file index for all keys between batch_iter_ and upper_key_
+          auto tmp_iter = batch_iter_;
+          while (tmp_iter != upper_key_) {
+            ++(fp_ctx_array_[tmp_iter.index()].curr_index_in_curr_level);
+            ++tmp_iter;
+          }
          maybe_repeat_key_ = true;
        }
        // Set the range for this file
        current_file_range_ =
-            MultiGetRange(next_file_range, batch_iter_prev_, upper_key);
+            MultiGetRange(next_file_range, batch_iter_prev_, upper_key_);
        returned_file_level_ = curr_level_;
        hit_file_level_ = curr_level_;
        is_hit_file_last_in_level_ =
@@ -607,6 +629,7 @@ class FilePickerMultiGet {
  // key found in the previous SST file, in order to serve as the start of
  // the batch key range for the next SST file
  MultiGetRange::Iterator batch_iter_prev_;
+  MultiGetRange::Iterator upper_key_;
  bool maybe_repeat_key_;
  MultiGetRange current_level_range_;
  MultiGetRange current_file_range_;
@@ -626,7 +649,7 @@ class FilePickerMultiGet {
      if (fp_ctx_array_[mget_iter.index()].curr_index_in_curr_level <
          curr_file_level_->num_files) {
        batch_iter_prev_ = current_level_range_.begin();
-        batch_iter_ = current_level_range_.begin();
+        upper_key_ = batch_iter_ = current_level_range_.begin();
        return true;
      }
    }
@@ -721,7 +744,7 @@ class FilePickerMultiGet {
      }
      if (level_contains_keys) {
        batch_iter_prev_ = current_level_range_.begin();
-        batch_iter_ = current_level_range_.begin();
+        upper_key_ = batch_iter_ = current_level_range_.begin();
        return true;
      }
      curr_level_++;
@@ -984,7 +1007,8 @@ class LevelIterator final : public InternalIterator {
        read_options_, file_options_, icomparator_, *file_meta.file_metadata,
        range_del_agg_, prefix_extractor_,
        nullptr /* don't need reference to table */, file_read_hist_, caller_,
-        /*arena=*/nullptr, skip_filters_, level_, smallest_compaction_key,
+        /*arena=*/nullptr, skip_filters_, level_,
+        /*max_file_size_for_l0_meta_pin=*/0, smallest_compaction_key,
        largest_compaction_key, allow_unprepared_value_);
  }

@@ -1579,10 +1603,9 @@ void Version::AddIteratorsForLevel(const ReadOptions& read_options,
          mutable_cf_options_.prefix_extractor.get(), nullptr,
          cfd_->internal_stats()->GetFileReadHist(0),
          TableReaderCaller::kUserIterator, arena,
-          /*skip_filters=*/false, /*level=*/0,
+          /*skip_filters=*/false, /*level=*/0, max_file_size_for_l0_meta_pin_,
          /*smallest_compaction_key=*/nullptr,
-          /*largest_compaction_key=*/nullptr,
-          allow_unprepared_value));
+          /*largest_compaction_key=*/nullptr, allow_unprepared_value));
    }
    if (should_sample) {
      // Count ones for every L0 files. This is done per iterator creation
@@ -1604,8 +1627,8 @@ void Version::AddIteratorsForLevel(const ReadOptions& read_options,
        mutable_cf_options_.prefix_extractor.get(), should_sample_file_read(),
        cfd_->internal_stats()->GetFileReadHist(level),
        TableReaderCaller::kUserIterator, IsFilterSkipped(level), level,
-        range_del_agg, /*largest_compaction_key=*/nullptr,
-        allow_unprepared_value));
+        range_del_agg,
+        /*compaction_boundaries=*/nullptr, allow_unprepared_value));
  }
 }

@@ -1639,7 +1662,7 @@ Status Version::OverlapWithLevelIterator(const ReadOptions& read_options,
          mutable_cf_options_.prefix_extractor.get(), nullptr,
          cfd_->internal_stats()->GetFileReadHist(0),
          TableReaderCaller::kUserIterator, &arena,
-          /*skip_filters=*/false, /*level=*/0,
+          /*skip_filters=*/false, /*level=*/0, max_file_size_for_l0_meta_pin_,
          /*smallest_compaction_key=*/nullptr,
          /*largest_compaction_key=*/nullptr,
          /*allow_unprepared_value=*/false));
@@ -1743,6 +1766,8 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset,
      refs_(0),
      file_options_(file_opt),
      mutable_cf_options_(mutable_cf_options),
+      max_file_size_for_l0_meta_pin_(
+          MaxFileSizeForL0MetaPin(mutable_cf_options_)),
      version_number_(version_number) {}

 void Version::Get(const ReadOptions& read_options, const LookupKey& k,
@@ -1806,7 +1831,7 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
        cfd_->internal_stats()->GetFileReadHist(fp.GetHitFileLevel()),
        IsFilterSkipped(static_cast<int>(fp.GetHitFileLevel()),
                        fp.IsHitFileLastInLevel()),
-        fp.GetHitFileLevel());
+        fp.GetHitFileLevel(), max_file_size_for_l0_meta_pin_);
    // TODO: examine the behavior for corrupted key
    if (timer_enabled) {
      PERF_COUNTER_BY_LEVEL_ADD(get_from_table_nanos, timer.ElapsedNanos(),
@@ -3902,7 +3927,8 @@ Status VersionSet::ProcessManifestWrites(
            cfd->internal_stats(), 1 /* max_threads */,
            true /* prefetch_index_and_filter_in_cache */,
            false /* is_initial_load */,
-            mutable_cf_options_ptrs[i]->prefix_extractor.get());
+            mutable_cf_options_ptrs[i]->prefix_extractor.get(),
+            MaxFileSizeForL0MetaPin(*mutable_cf_options_ptrs[i]));
        if (!s.ok()) {
          if (db_options_->paranoid_checks) {
            break;
@@ -4421,24 +4447,26 @@ Status VersionSet::GetCurrentManifestPath(const std::string& dbname,
  if (dbname.back() != '/') {
    manifest_path->push_back('/');
  }
-  *manifest_path += fname;
+  manifest_path->append(fname);
  return Status::OK();
 }

 Status VersionSet::ReadAndRecover(
-    log::Reader* reader, AtomicGroupReadBuffer* read_buffer,
+    log::Reader& reader, AtomicGroupReadBuffer* read_buffer,
    const std::unordered_map<std::string, ColumnFamilyOptions>& name_to_options,
    std::unordered_map<int, std::string>& column_families_not_found,
    std::unordered_map<uint32_t, std::unique_ptr<BaseReferencedVersionBuilder>>&
        builders,
-    VersionEditParams* version_edit_params, std::string* db_id) {
-  assert(reader != nullptr);
+    Status* log_read_status, VersionEditParams* version_edit_params,
+    std::string* db_id) {
  assert(read_buffer != nullptr);
+  assert(log_read_status != nullptr);
  Status s;
  Slice record;
  std::string scratch;
  size_t recovered_edits = 0;
-  while (reader->ReadRecord(&record, &scratch) && s.ok()) {
+  while (s.ok() && reader.ReadRecord(&record, &scratch) &&
+         log_read_status->ok()) {
    VersionEdit edit;
    s = edit.DecodeFrom(record);
    if (!s.ok()) {
@@ -4482,6 +4510,9 @@ Status VersionSet::ReadAndRecover(
      }
    }
  }
+  if (!log_read_status->ok()) {
+    s = *log_read_status;
+  }
  if (!s.ok()) {
    // Clear the buffer if we fail to decode/apply an edit.
    read_buffer->Clear();
@@ -4528,8 +4559,7 @@ Status VersionSet::Recover(
                                 db_options_->log_readahead_size));
  }

-  std::unordered_map<uint32_t, std::unique_ptr<BaseReferencedVersionBuilder>>
-      builders;
+  VersionBuilderMap builders;

  // add default column family
  auto default_cf_iter = cf_name_to_options.find(kDefaultColumnFamilyName);
@@ -4551,12 +4581,13 @@ Status VersionSet::Recover(
  VersionEditParams version_edit_params;
  {
    VersionSet::LogReporter reporter;
-    reporter.status = &s;
+    Status log_read_status;
+    reporter.status = &log_read_status;
    log::Reader reader(nullptr, std::move(manifest_file_reader), &reporter,
                       true /* checksum */, 0 /* log_number */);
    AtomicGroupReadBuffer read_buffer;
-    s = ReadAndRecover(&reader, &read_buffer, cf_name_to_options,
-                       column_families_not_found, builders,
+    s = ReadAndRecover(reader, &read_buffer, cf_name_to_options,
+                       column_families_not_found, builders, &log_read_status,
                       &version_edit_params, db_id);
    current_manifest_file_size = reader.GetReadOffset();
    assert(current_manifest_file_size != 0);
@@ -4629,7 +4660,8 @@ Status VersionSet::Recover(
          cfd->internal_stats(), db_options_->max_file_opening_threads,
          false /* prefetch_index_and_filter_in_cache */,
          true /* is_initial_load */,
-          cfd->GetLatestMutableCFOptions()->prefix_extractor.get());
+          cfd->GetLatestMutableCFOptions()->prefix_extractor.get(),
+          MaxFileSizeForL0MetaPin(*cfd->GetLatestMutableCFOptions()));
      if (!s.ok()) {
        if (db_options_->paranoid_checks) {
          return s;
@@ -4821,21 +4853,20 @@ Status VersionSet::TryRecoverFromOneManifest(
                                 db_options_->log_readahead_size));
  }

+  assert(s.ok());
  VersionSet::LogReporter reporter;
  reporter.status = &s;
  log::Reader reader(nullptr, std::move(manifest_file_reader), &reporter,
                     /*checksum=*/true, /*log_num=*/0);
-  {
-    VersionEditHandlerPointInTime handler_pit(read_only, column_families,
-                                              const_cast<VersionSet*>(this));
+  VersionEditHandlerPointInTime handler_pit(read_only, column_families,
+                                            const_cast<VersionSet*>(this));

-    s = handler_pit.Iterate(reader, db_id);
+  handler_pit.Iterate(reader, &s, db_id);

-    assert(nullptr != has_missing_table_file);
-    *has_missing_table_file = handler_pit.HasMissingFiles();
-  }
+  assert(nullptr != has_missing_table_file);
+  *has_missing_table_file = handler_pit.HasMissingFiles();

-  return s;
+  return handler_pit.status();
 }

 Status VersionSet::ListColumnFamilies(std::vector<std::string>* column_families,
@@ -5666,13 +5697,14 @@ InternalIterator* VersionSet::MakeInputIterator(
        for (size_t i = 0; i < flevel->num_files; i++) {
          list[num++] = cfd->table_cache()->NewIterator(
              read_options, file_options_compactions,
-              cfd->internal_comparator(),
-              *flevel->files[i].file_metadata, range_del_agg,
-              c->mutable_cf_options()->prefix_extractor.get(),
+              cfd->internal_comparator(), *flevel->files[i].file_metadata,
+              range_del_agg, c->mutable_cf_options()->prefix_extractor.get(),
              /*table_reader_ptr=*/nullptr,
              /*file_read_hist=*/nullptr, TableReaderCaller::kCompaction,
              /*arena=*/nullptr,
-              /*skip_filters=*/false, /*level=*/static_cast<int>(c->level(which)),
+              /*skip_filters=*/false,
+              /*level=*/static_cast<int>(c->level(which)),
+              MaxFileSizeForL0MetaPin(*c->mutable_cf_options()),
              /*smallest_compaction_key=*/nullptr,
              /*largest_compaction_key=*/nullptr,
              /*allow_unprepared_value=*/false);
@@ -5955,8 +5987,7 @@ Status ReactiveVersionSet::Recover(
  // In recovery, nobody else can access it, so it's fine to set it to be
  // initialized earlier.
  default_cfd->set_initialized();
-  std::unordered_map<uint32_t, std::unique_ptr<BaseReferencedVersionBuilder>>
-      builders;
+  VersionBuilderMap builders;
  std::unordered_map<int, std::string> column_families_not_found;
  builders.insert(
      std::make_pair(0, std::unique_ptr<BaseReferencedVersionBuilder>(
@@ -5964,7 +5995,7 @@ Status ReactiveVersionSet::Recover(

  manifest_reader_status->reset(new Status());
  manifest_reporter->reset(new LogReporter());
-  static_cast<LogReporter*>(manifest_reporter->get())->status =
+  static_cast_with_check<LogReporter>(manifest_reporter->get())->status =
      manifest_reader_status->get();
  Status s = MaybeSwitchManifest(manifest_reporter->get(), manifest_reader);
  log::Reader* reader = manifest_reader->get();
@@ -5973,10 +6004,9 @@ Status ReactiveVersionSet::Recover(
  VersionEdit version_edit;
  while (s.ok() && retry < 1) {
    assert(reader != nullptr);
-    Slice record;
-    std::string scratch;
-    s = ReadAndRecover(reader, &read_buffer_, cf_name_to_options,
-                       column_families_not_found, builders, &version_edit);
+    s = ReadAndRecover(*reader, &read_buffer_, cf_name_to_options,
+                       column_families_not_found, builders,
+                       manifest_reader_status->get(), &version_edit);
    if (s.ok()) {
      bool enough = version_edit.has_next_file_number_ &&
                    version_edit.has_log_number_ &&
@@ -6003,7 +6033,8 @@ Status ReactiveVersionSet::Recover(
                cfd->internal_stats(), db_options_->max_file_opening_threads,
                false /* prefetch_index_and_filter_in_cache */,
                true /* is_initial_load */,
-                cfd->GetLatestMutableCFOptions()->prefix_extractor.get());
+                cfd->GetLatestMutableCFOptions()->prefix_extractor.get(),
+                MaxFileSizeForL0MetaPin(*cfd->GetLatestMutableCFOptions()));
            if (!s.ok()) {
              enough = false;
              if (s.IsPathNotFound()) {
@@ -6279,7 +6310,8 @@ Status ReactiveVersionSet::ApplyOneVersionEditToBuilder(
        cfd->internal_stats(), db_options_->max_file_opening_threads,
        false /* prefetch_index_and_filter_in_cache */,
        false /* is_initial_load */,
-        cfd->GetLatestMutableCFOptions()->prefix_extractor.get());
+        cfd->GetLatestMutableCFOptions()->prefix_extractor.get(),
+        MaxFileSizeForL0MetaPin(*cfd->GetLatestMutableCFOptions()));
    TEST_SYNC_POINT_CALLBACK(
        "ReactiveVersionSet::ApplyOneVersionEditToBuilder:"
        "AfterLoadTableHandlers",
@@ -324,6 +324,17 @@ class VersionStorageInfo {
    return it->second;
  }

+  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
+  FileMetaData* GetFileMetaDataByNumber(uint64_t file_number) const {
+    auto location = GetFileLocation(file_number);
+
+    if (!location.IsValid()) {
+      return nullptr;
+    }
+
+    return files_[location.GetLevel()][location.GetPosition()];
+  }
+
  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
  using BlobFiles = std::map<uint64_t, std::shared_ptr<BlobFileMetaData>>;
  const BlobFiles& GetBlobFiles() const { return blob_files_; }
@@ -792,6 +803,8 @@ class Version {
  int refs_;                    // Number of live refs to this version
  const FileOptions file_options_;
  const MutableCFOptions mutable_cf_options_;
+  // Cached value to avoid recomputing it on every read.
+  const size_t max_file_size_for_l0_meta_pin_;

  // A version number that uniquely represents this version. This is
  // used for debugging and logging purposes only.
@@ -1152,6 +1165,10 @@ class VersionSet {
  void SetIOStatusOK() { io_status_ = IOStatus::OK(); }

 protected:
+  using VersionBuilderMap =
+      std::unordered_map<uint32_t,
+                         std::unique_ptr<BaseReferencedVersionBuilder>>;
+
  struct ManifestWriter;

  friend class Version;
@@ -1163,7 +1180,9 @@ class VersionSet {
  struct LogReporter : public log::Reader::Reporter {
    Status* status;
    virtual void Corruption(size_t /*bytes*/, const Status& s) override {
-      if (this->status->ok()) *this->status = s;
+      if (status->ok()) {
+        *status = s;
+      }
    }
  };

@@ -1194,13 +1213,14 @@ class VersionSet {
                                       const VersionEdit* edit);

  Status ReadAndRecover(
-      log::Reader* reader, AtomicGroupReadBuffer* read_buffer,
+      log::Reader& reader, AtomicGroupReadBuffer* read_buffer,
      const std::unordered_map<std::string, ColumnFamilyOptions>&
          name_to_options,
      std::unordered_map<int, std::string>& column_families_not_found,
      std::unordered_map<
          uint32_t, std::unique_ptr<BaseReferencedVersionBuilder>>& builders,
-      VersionEditParams* version_edit, std::string* db_id = nullptr);
+      Status* log_read_status, VersionEditParams* version_edit,
+      std::string* db_id = nullptr);

  // REQUIRES db mutex
  Status ApplyOneVersionEditToBuilder(
@@ -1329,8 +1349,7 @@ class ReactiveVersionSet : public VersionSet {
      std::unique_ptr<log::FragmentBufferedReader>* manifest_reader);

 private:
-  std::unordered_map<uint32_t, std::unique_ptr<BaseReferencedVersionBuilder>>
-      active_version_builders_;
+  VersionBuilderMap active_version_builders_;
  AtomicGroupReadBuffer read_buffer_;
  // Number of version edits to skip by ReadAndApply at the beginning of a new
  // MANIFEST created by primary.
@@ -421,7 +421,7 @@ TEST_F(VersionStorageInfoTest, GetOverlappingInputs) {
      1, {"i", 0, kTypeValue}, {"j", 0, kTypeValue}));
 }

-TEST_F(VersionStorageInfoTest, FileLocation) {
+TEST_F(VersionStorageInfoTest, FileLocationAndMetaDataByNumber) {
  Add(0, 11U, "1", "2", 5000U);
  Add(0, 12U, "1", "2", 5000U);

@@ -429,13 +429,18 @@ TEST_F(VersionStorageInfoTest, FileLocation) {

  ASSERT_EQ(vstorage_.GetFileLocation(11U),
            VersionStorageInfo::FileLocation(0, 0));
+  ASSERT_NE(vstorage_.GetFileMetaDataByNumber(11U), nullptr);
+
  ASSERT_EQ(vstorage_.GetFileLocation(12U),
            VersionStorageInfo::FileLocation(0, 1));
+  ASSERT_NE(vstorage_.GetFileMetaDataByNumber(12U), nullptr);

  ASSERT_EQ(vstorage_.GetFileLocation(7U),
            VersionStorageInfo::FileLocation(2, 0));
+  ASSERT_NE(vstorage_.GetFileMetaDataByNumber(7U), nullptr);

  ASSERT_FALSE(vstorage_.GetFileLocation(999U).IsValid());
+  ASSERT_EQ(vstorage_.GetFileMetaDataByNumber(999U), nullptr);
 }

 class VersionStorageInfoTimestampTest : public VersionStorageInfoTestBase {
@@ -146,6 +146,7 @@ DECLARE_int32(reopen);
 DECLARE_double(bloom_bits);
 DECLARE_bool(use_block_based_filter);
 DECLARE_bool(partition_filters);
+DECLARE_bool(optimize_filters_for_memory);
 DECLARE_int32(index_type);
 DECLARE_string(db);
 DECLARE_string(secondaries_base);
@@ -161,7 +162,7 @@ DECLARE_bool(statistics);
 DECLARE_bool(sync);
 DECLARE_bool(use_fsync);
 DECLARE_int32(kill_random_test);
-DECLARE_string(kill_prefix_blacklist);
+DECLARE_string(kill_exclude_prefixes);
 DECLARE_bool(disable_wal);
 DECLARE_uint64(recycle_log_file_num);
 DECLARE_int64(target_file_size_base);
@@ -236,6 +237,10 @@ DECLARE_double(blob_db_gc_cutoff);
 DECLARE_int32(approximate_size_one_in);
 DECLARE_bool(sync_fault_injection);

+DECLARE_bool(best_efforts_recovery);
+DECLARE_bool(skip_verifydb);
+DECLARE_bool(enable_compaction_filter);
+
 const long KB = 1024;
 const int kRandomValueMaxFactor = 3;
 const int kValueMaxLen = 100;
@@ -436,19 +441,10 @@ extern inline bool GetIntVal(std::string big_endian_key, uint64_t* key_p) {

  assert(size_key <= key_gen_ctx.weights.size() * sizeof(uint64_t));

-  // Pad with zeros to make it a multiple of 8. This function may be called
-  // with a prefix, in which case we return the first index that falls
-  // inside or outside that prefix, dependeing on whether the prefix is
-  // the start of upper bound of a scan
-  unsigned int pad = sizeof(uint64_t) - (size_key % sizeof(uint64_t));
-  if (pad < sizeof(uint64_t)) {
-    big_endian_key.append(pad, '\0');
-    size_key += pad;
-  }
-
  std::string little_endian_key;
  little_endian_key.resize(size_key);
-  for (size_t start = 0; start < size_key; start += sizeof(uint64_t)) {
+  for (size_t start = 0; start + sizeof(uint64_t) <= size_key;
+       start += sizeof(uint64_t)) {
    size_t end = start + sizeof(uint64_t);
    for (size_t i = 0; i < sizeof(uint64_t); ++i) {
      little_endian_key[start + i] = big_endian_key[end - 1 - i];
@@ -467,17 +463,40 @@ extern inline bool GetIntVal(std::string big_endian_key, uint64_t* key_p) {
    uint64_t pfx = prefixes[i];
    key += (pfx / key_gen_ctx.weights[i]) * key_gen_ctx.window +
           pfx % key_gen_ctx.weights[i];
+    if (i < prefixes.size() - 1) {
+      // The encoding writes a `key_gen_ctx.weights[i] - 1` that counts for
+      // `key_gen_ctx.weights[i]` when there are more prefixes to come. So we
+      // need to add back the one here as we're at a non-last prefix.
+      ++key;
+    }
  }
  *key_p = key;
  return true;
 }

+// Given a string prefix, map it to the first corresponding index in the
+// expected values buffer.
+inline bool GetFirstIntValInPrefix(std::string big_endian_prefix,
+                                   uint64_t* key_p) {
+  size_t size_key = big_endian_prefix.size();
+  // Pad with zeros to make it a multiple of 8. This function may be called
+  // with a prefix, in which case we return the first index that falls
+  // inside or outside that prefix, dependeing on whether the prefix is
+  // the start of upper bound of a scan
+  unsigned int pad = sizeof(uint64_t) - (size_key % sizeof(uint64_t));
+  if (pad < sizeof(uint64_t)) {
+    big_endian_prefix.append(pad, '\0');
+  }
+  return GetIntVal(std::move(big_endian_prefix), key_p);
+}
+
 extern inline uint64_t GetPrefixKeyCount(const std::string& prefix,
                                         const std::string& ub) {
  uint64_t start = 0;
  uint64_t end = 0;

-  if (!GetIntVal(prefix, &start) || !GetIntVal(ub, &end)) {
+  if (!GetFirstIntValInPrefix(prefix, &start) ||
+      !GetFirstIntValInPrefix(ub, &end)) {
    return 0;
  }

@@ -0,0 +1,79 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#include "rocksdb/compaction_filter.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// DbStressCompactionFilter is safe to use with db_stress as it does not perform
+// any mutation. It only makes `kRemove` decisions for keys that are already
+// non-existent according to the `SharedState`.
+class DbStressCompactionFilter : public CompactionFilter {
+ public:
+  DbStressCompactionFilter(SharedState* state, int cf_id)
+      : state_(state), cf_id_(cf_id) {}
+
+  Decision FilterV2(int /*level*/, const Slice& key, ValueType /*value_type*/,
+                    const Slice& /*existing_value*/, std::string* /*new_value*/,
+                    std::string* /*skip_until*/) const override {
+    if (state_ == nullptr) {
+      return Decision::kKeep;
+    }
+    if (key.empty() || ('0' <= key[0] && key[0] <= '9')) {
+      // It is likely leftover from a test_batches_snapshots run. Below this
+      // conditional, the test_batches_snapshots key format is not handled
+      // properly. Just keep it to be safe.
+      return Decision::kKeep;
+    }
+    uint64_t key_num = 0;
+    bool ok = GetIntVal(key.ToString(), &key_num);
+    assert(ok);
+    (void)ok;
+    MutexLock key_lock(state_->GetMutexForKey(cf_id_, key_num));
+    if (!state_->Exists(cf_id_, key_num)) {
+      return Decision::kRemove;
+    }
+    return Decision::kKeep;
+  }
+
+  const char* Name() const override { return "DbStressCompactionFilter"; }
+
+ private:
+  SharedState* const state_;
+  const int cf_id_;
+};
+
+class DbStressCompactionFilterFactory : public CompactionFilterFactory {
+ public:
+  DbStressCompactionFilterFactory() : state_(nullptr) {}
+
+  void SetSharedState(SharedState* state) {
+    MutexLock state_mutex_guard(&state_mutex_);
+    state_ = state;
+  }
+
+  std::unique_ptr<CompactionFilter> CreateCompactionFilter(
+      const CompactionFilter::Context& context) override {
+    MutexLock state_mutex_guard(&state_mutex_);
+    return std::unique_ptr<CompactionFilter>(
+        new DbStressCompactionFilter(state_, context.column_family_id));
+  }
+
+  const char* Name() const override {
+    return "DbStressCompactionFilterFactory";
+  }
+
+ private:
+  port::Mutex state_mutex_;
+  SharedState* state_;
+};
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -16,7 +16,7 @@ void ThreadBody(void* v) {
  ThreadState* thread = reinterpret_cast<ThreadState*>(v);
  SharedState* shared = thread->shared;

-  if (shared->ShouldVerifyAtBeginning()) {
+  if (!FLAGS_skip_verifydb && shared->ShouldVerifyAtBeginning()) {
    thread->shared->GetStressTest()->VerifyDb(thread);
  }
  {
@@ -42,7 +42,9 @@ void ThreadBody(void* v) {
    }
  }

-  thread->shared->GetStressTest()->VerifyDb(thread);
+  if (!FLAGS_skip_verifydb) {
+    thread->shared->GetStressTest()->VerifyDb(thread);
+  }

  {
    MutexLock l(shared->GetMutex());
@@ -55,11 +57,8 @@ void ThreadBody(void* v) {

 bool RunStressTest(StressTest* stress) {
  stress->InitDb();
-
  SharedState shared(db_stress_env, stress);
-  if (FLAGS_read_only) {
-    stress->InitReadonlyDb(&shared);
-  }
+  stress->FinishInitDb(&shared);

 #ifndef NDEBUG
  if (FLAGS_sync_fault_injection) {
@@ -118,6 +117,9 @@ bool RunStressTest(StressTest* stress) {
    if (FLAGS_test_batches_snapshots) {
      fprintf(stdout, "%s Limited verification already done during gets\n",
              db_stress_env->TimeToString((uint64_t)now / 1000000).c_str());
+    } else if (FLAGS_skip_verifydb) {
+      fprintf(stdout, "%s Verification skipped\n",
+              db_stress_env->TimeToString((uint64_t)now / 1000000).c_str());
    } else {
      fprintf(stdout, "%s Starting verification\n",
              db_stress_env->TimeToString((uint64_t)now / 1000000).c_str());
@@ -140,7 +142,8 @@ bool RunStressTest(StressTest* stress) {
    threads[i] = nullptr;
  }
  now = db_stress_env->NowMicros();
-  if (!FLAGS_test_batches_snapshots && !shared.HasVerificationFailedYet()) {
+  if (!FLAGS_skip_verifydb && !FLAGS_test_batches_snapshots &&
+      !shared.HasVerificationFailedYet()) {
    fprintf(stdout, "%s Verification successful\n",
            db_stress_env->TimeToString(now / 1000000).c_str());
  }
@@ -361,6 +361,11 @@ DEFINE_bool(partition_filters, false,
            "use partitioned filters "
            "for block-based table");

+DEFINE_bool(
+    optimize_filters_for_memory,
+    ROCKSDB_NAMESPACE::BlockBasedTableOptions().optimize_filters_for_memory,
+    "Minimize memory footprint of filters");
+
 DEFINE_int32(
    index_type,
    static_cast<int32_t>(
@@ -414,10 +419,10 @@ static const bool FLAGS_kill_random_test_dummy __attribute__((__unused__)) =
    RegisterFlagValidator(&FLAGS_kill_random_test, &ValidateInt32Positive);
 extern int rocksdb_kill_odds;

-DEFINE_string(kill_prefix_blacklist, "",
+DEFINE_string(kill_exclude_prefixes, "",
              "If non-empty, kill points with prefix in the list given will be"
              " skipped. Items are comma-separated.");
-extern std::vector<std::string> rocksdb_kill_prefix_blacklist;
+extern std::vector<std::string> rocksdb_kill_exclude_prefixes;

 DEFINE_bool(disable_wal, false, "If true, do not write WAL for write.");

@@ -691,4 +696,13 @@ DEFINE_int32(read_fault_one_in, 1000,
 DEFINE_bool(sync_fault_injection, false,
            "If true, FaultInjectionTestFS will be used for write operations, "
            " and unsynced data in DB will lost after crash.");
+
+DEFINE_bool(best_efforts_recovery, false,
+            "If true, use best efforts recovery.");
+DEFINE_bool(skip_verifydb, false, "If true, skip VerifyDb() calls.");
+
+DEFINE_bool(enable_compaction_filter, false,
+            "If true, configures a compaction filter that returns a kRemove "
+            "decision for deleted keys.");
+
 #endif  // GFLAGS
@@ -10,6 +10,7 @@

 #ifdef GFLAGS
 #include "db_stress_tool/db_stress_common.h"
+#include "db_stress_tool/db_stress_compaction_filter.h"
 #include "db_stress_tool/db_stress_driver.h"
 #include "rocksdb/convenience.h"
 #include "rocksdb/sst_file_manager.h"
@@ -195,11 +196,18 @@ void StressTest::InitDb() {
  BuildOptionsTable();
 }

-void StressTest::InitReadonlyDb(SharedState* shared) {
-  uint64_t now = db_stress_env->NowMicros();
-  fprintf(stdout, "%s Preloading db with %" PRIu64 " KVs\n",
-          db_stress_env->TimeToString(now / 1000000).c_str(), FLAGS_max_key);
-  PreloadDbAndReopenAsReadOnly(FLAGS_max_key, shared);
+void StressTest::FinishInitDb(SharedState* shared) {
+  if (FLAGS_read_only) {
+    uint64_t now = db_stress_env->NowMicros();
+    fprintf(stdout, "%s Preloading db with %" PRIu64 " KVs\n",
+            db_stress_env->TimeToString(now / 1000000).c_str(), FLAGS_max_key);
+    PreloadDbAndReopenAsReadOnly(FLAGS_max_key, shared);
+  }
+  if (FLAGS_enable_compaction_filter) {
+    reinterpret_cast<DbStressCompactionFilterFactory*>(
+        options_.compaction_filter_factory.get())
+        ->SetSharedState(shared);
+  }
 }

 bool StressTest::VerifySecondaries() {
@@ -1709,9 +1717,9 @@ void StressTest::PrintEnv() const {
  fprintf(stdout, "Memtablerep               : %s\n", memtablerep);

  fprintf(stdout, "Test kill odd             : %d\n", rocksdb_kill_odds);
-  if (!rocksdb_kill_prefix_blacklist.empty()) {
+  if (!rocksdb_kill_exclude_prefixes.empty()) {
    fprintf(stdout, "Skipping kill points prefixes:\n");
-    for (auto& p : rocksdb_kill_prefix_blacklist) {
+    for (auto& p : rocksdb_kill_exclude_prefixes) {
      fprintf(stdout, "  %s\n", p.c_str());
    }
  }
@@ -1729,6 +1737,8 @@ void StressTest::PrintEnv() const {
          static_cast<int>(FLAGS_level_compaction_dynamic_level_bytes));
  fprintf(stdout, "Read fault one in         : %d\n", FLAGS_read_fault_one_in);
  fprintf(stdout, "Sync fault injection      : %d\n", FLAGS_sync_fault_injection);
+  fprintf(stdout, "Best efforts recovery     : %d\n",
+          static_cast<int>(FLAGS_best_efforts_recovery));

  fprintf(stdout, "------------------------------------------------\n");
 }
@@ -1752,6 +1762,8 @@ void StressTest::Open() {
        static_cast<int32_t>(FLAGS_index_block_restart_interval);
    block_based_options.filter_policy = filter_policy_;
    block_based_options.partition_filters = FLAGS_partition_filters;
+    block_based_options.optimize_filters_for_memory =
+        FLAGS_optimize_filters_for_memory;
    block_based_options.index_type =
        static_cast<BlockBasedTableOptions::IndexType>(FLAGS_index_type);
    options_.table_factory.reset(
@@ -1912,6 +1924,12 @@ void StressTest::Open() {
  } else {
    options_.merge_operator = MergeOperators::CreatePutOperator();
  }
+  if (FLAGS_enable_compaction_filter) {
+    options_.compaction_filter_factory =
+        std::make_shared<DbStressCompactionFilterFactory>();
+  }
+
+  options_.best_efforts_recovery = FLAGS_best_efforts_recovery;

  fprintf(stdout, "DB path: [%s]\n", FLAGS_db.c_str());

@@ -27,7 +27,9 @@ class StressTest {
  bool BuildOptionsTable();

  void InitDb();
-  void InitReadonlyDb(SharedState*);
+  // The initialization work is split into two parts to avoid a circular
+  // dependency with `SharedState`.
+  void FinishInitDb(SharedState*);

  // Return false if verification fails.
  bool VerifySecondaries();
@@ -216,9 +216,39 @@ int db_stress_tool(int argc, char** argv) {
        "Must set -test_secondary=true if secondary_catch_up_one_in > 0.\n");
    exit(1);
  }
+  if (FLAGS_best_efforts_recovery && !FLAGS_skip_verifydb &&
+      !FLAGS_disable_wal) {
+    fprintf(stderr,
+            "With best-efforts recovery, either skip_verifydb or disable_wal "
+            "should be set to true.\n");
+    exit(1);
+  }
+  if (FLAGS_skip_verifydb) {
+    if (FLAGS_verify_db_one_in > 0) {
+      fprintf(stderr,
+              "Must set -verify_db_one_in=0 if skip_verifydb is true.\n");
+      exit(1);
+    }
+    if (FLAGS_continuous_verification_interval > 0) {
+      fprintf(stderr,
+              "Must set -continuous_verification_interval=0 if skip_verifydb "
+              "is true.\n");
+      exit(1);
+    }
+  }
+  if (FLAGS_enable_compaction_filter &&
+      (FLAGS_acquire_snapshot_one_in > 0 || FLAGS_compact_range_one_in > 0 ||
+       FLAGS_iterpercent > 0 || FLAGS_test_batches_snapshots ||
+       FLAGS_test_cf_consistency)) {
+    fprintf(
+        stderr,
+        "Error: acquire_snapshot_one_in, compact_range_one_in, iterpercent, "
+        "test_batches_snapshots  must all be 0 when using compaction filter\n");
+    exit(1);
+  }

  rocksdb_kill_odds = FLAGS_kill_random_test;
-  rocksdb_kill_prefix_blacklist = SplitString(FLAGS_kill_prefix_blacklist);
+  rocksdb_kill_exclude_prefixes = SplitString(FLAGS_kill_exclude_prefixes);

  unsigned int levels = FLAGS_max_key_len;
  std::vector<std::string> weights;
@@ -555,14 +555,18 @@ class CompositeEnvWrapper : public Env {
    return env_target_->IncBackgroundThreadsIfNeeded(num, pri);
  }

-  void LowerThreadPoolIOPriority(Priority pool = LOW) override {
+  void LowerThreadPoolIOPriority(Priority pool) override {
    env_target_->LowerThreadPoolIOPriority(pool);
  }

-  void LowerThreadPoolCPUPriority(Priority pool = LOW) override {
+  void LowerThreadPoolCPUPriority(Priority pool) override {
    env_target_->LowerThreadPoolCPUPriority(pool);
  }

+  Status LowerThreadPoolCPUPriority(Priority pool, CpuPriority pri) override {
+    return env_target_->LowerThreadPoolCPUPriority(pool, pri);
+  }
+
  std::string TimeToString(uint64_t time) override {
    return env_target_->TimeToString(time);
  }
@@ -4,13 +4,14 @@
 //
 // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.

+#include <algorithm>
 #include <memory>
 #include <string>
 #include <vector>
-#include <algorithm>

 #include "env/mock_env.h"
 #include "rocksdb/env.h"
+#include "rocksdb/env_encryption.h"
 #include "test_util/testharness.h"

 namespace ROCKSDB_NAMESPACE {
@@ -89,6 +90,24 @@ INSTANTIATE_TEST_CASE_P(EnvDefault, EnvMoreTestWithParam,
 static std::unique_ptr<Env> mock_env(new MockEnv(Env::Default()));
 INSTANTIATE_TEST_CASE_P(MockEnv, EnvBasicTestWithParam,
                        ::testing::Values(mock_env.get()));
+
+#ifndef ROCKSDB_LITE
+// next statements run env test against default encryption code.
+static ROT13BlockCipher encrypt_block_rot13(32);
+
+static CTREncryptionProvider encrypt_provider_ctr(encrypt_block_rot13);
+
+static std::unique_ptr<Env> ecpt_env(NewEncryptedEnv(Env::Default(),
+                                                     &encrypt_provider_ctr));
+
+static std::unique_ptr<Env> encrypt_env(
+    new NormalizingEnvWrapper(ecpt_env.get()));
+INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvBasicTestWithParam,
+                        ::testing::Values(encrypt_env.get()));
+INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvMoreTestWithParam,
+                        ::testing::Values(encrypt_env.get()));
+#endif  // ROCKSDB_LITE
+
 #ifndef ROCKSDB_LITE
 static std::unique_ptr<Env> mem_env(NewMemEnv(Env::Default()));
 INSTANTIATE_TEST_CASE_P(MemEnv, EnvBasicTestWithParam,
@@ -23,19 +23,6 @@ namespace ROCKSDB_NAMESPACE {

 #ifndef ROCKSDB_LITE

-class EncryptedSequentialFile : public SequentialFile {
-  private:
-    std::unique_ptr<SequentialFile> file_;
-    std::unique_ptr<BlockAccessCipherStream> stream_;
-    uint64_t offset_;
-    size_t prefixLength_;
-
-     public:
-  // Default ctor. Given underlying sequential file is supposed to be at
-  // offset == prefixLength.
-  EncryptedSequentialFile(SequentialFile* f, BlockAccessCipherStream* s, size_t prefixLength)
-      : file_(f), stream_(s), offset_(prefixLength), prefixLength_(prefixLength) {
-  }

  // Read up to "n" bytes from the file.  "scratch[0..n-1]" may be
  // written by this routine.  Sets "*result" to the data that was
@@ -45,83 +32,73 @@ class EncryptedSequentialFile : public SequentialFile {
  // If an error was encountered, returns a non-OK status.
  //
  // REQUIRES: External synchronization
-  Status Read(size_t n, Slice* result, char* scratch) override {
-    assert(scratch);
-    Status status = file_->Read(n, result, scratch);
-    if (!status.ok()) {
-      return status;
-    }
-    {
-      PERF_TIMER_GUARD(decrypt_data_nanos);
-      status = stream_->Decrypt(offset_, (char*)result->data(), result->size());
-    }
-    offset_ += result->size();  // We've already ready data from disk, so update
-                                // offset_ even if decryption fails.
+Status EncryptedSequentialFile::Read(size_t n, Slice* result, char* scratch) {
+  assert(scratch);
+  Status status = file_->Read(n, result, scratch);
+  if (!status.ok()) {
    return status;
  }
+  {
+    PERF_TIMER_GUARD(decrypt_data_nanos);
+    status = stream_->Decrypt(offset_, (char*)result->data(), result->size());
+  }
+  offset_ += result->size();  // We've already ready data from disk, so update
+                              // offset_ even if decryption fails.
+  return status;
+}

-  // Skip "n" bytes from the file. This is guaranteed to be no
-  // slower that reading the same data, but may be faster.
-  //
-  // If end of file is reached, skipping will stop at the end of the
-  // file, and Skip will return OK.
-  //
-  // REQUIRES: External synchronization
-  Status Skip(uint64_t n) override {
-    auto status = file_->Skip(n);
-    if (!status.ok()) {
-      return status;
-    }
-    offset_ += n;
+// Skip "n" bytes from the file. This is guaranteed to be no
+// slower that reading the same data, but may be faster.
+//
+// If end of file is reached, skipping will stop at the end of the
+// file, and Skip will return OK.
+//
+// REQUIRES: External synchronization
+Status EncryptedSequentialFile::Skip(uint64_t n) {
+  auto status = file_->Skip(n);
+  if (!status.ok()) {
    return status;
  }
+  offset_ += n;
+  return status;
+}

-  // Indicates the upper layers if the current SequentialFile implementation
-  // uses direct IO.
-  bool use_direct_io() const override { return file_->use_direct_io(); }
+// Indicates the upper layers if the current SequentialFile implementation
+// uses direct IO.
+bool EncryptedSequentialFile::use_direct_io() const {
+  return file_->use_direct_io();
+}

-  // Use the returned alignment value to allocate
-  // aligned buffer for Direct I/O
-  size_t GetRequiredBufferAlignment() const override {
-    return file_->GetRequiredBufferAlignment();
-  }
+// Use the returned alignment value to allocate
+// aligned buffer for Direct I/O
+size_t EncryptedSequentialFile::GetRequiredBufferAlignment() const {
+  return file_->GetRequiredBufferAlignment();
+}

  // Remove any kind of caching of data from the offset to offset+length
  // of this file. If the length is 0, then it refers to the end of file.
  // If the system is not caching the file contents, then this is a noop.
-  Status InvalidateCache(size_t offset, size_t length) override {
-    return file_->InvalidateCache(offset + prefixLength_, length);
-  }
+Status EncryptedSequentialFile::InvalidateCache(size_t offset, size_t length) {
+  return file_->InvalidateCache(offset + prefixLength_, length);
+}

  // Positioned Read for direct I/O
  // If Direct I/O enabled, offset, n, and scratch should be properly aligned
-  Status PositionedRead(uint64_t offset, size_t n, Slice* result,
-                        char* scratch) override {
-    assert(scratch);
-    offset += prefixLength_; // Skip prefix
-    auto status = file_->PositionedRead(offset, n, result, scratch);
-    if (!status.ok()) {
-      return status;
-    }
-    offset_ = offset + result->size();
-    {
-      PERF_TIMER_GUARD(decrypt_data_nanos);
-      status = stream_->Decrypt(offset, (char*)result->data(), result->size());
-    }
+Status EncryptedSequentialFile::PositionedRead(uint64_t offset, size_t n,
+                                               Slice* result, char* scratch) {
+  assert(scratch);
+  offset += prefixLength_;  // Skip prefix
+  auto status = file_->PositionedRead(offset, n, result, scratch);
+  if (!status.ok()) {
    return status;
  }
-};
-
-// A file abstraction for randomly reading the contents of a file.
-class EncryptedRandomAccessFile : public RandomAccessFile {
-  private:
-    std::unique_ptr<RandomAccessFile> file_;
-    std::unique_ptr<BlockAccessCipherStream> stream_;
-    size_t prefixLength_;
-
- public:
-  EncryptedRandomAccessFile(RandomAccessFile* f, BlockAccessCipherStream* s, size_t prefixLength)
-    : file_(f), stream_(s), prefixLength_(prefixLength) { }
+  offset_ = offset + result->size();
+  {
+    PERF_TIMER_GUARD(decrypt_data_nanos);
+    status = stream_->Decrypt(offset, (char*)result->data(), result->size());
+  }
+  return status;
+}

  // Read up to "n" bytes from the file starting at "offset".
  // "scratch[0..n-1]" may be written by this routine.  Sets "*result"
@@ -133,26 +110,26 @@ class EncryptedRandomAccessFile : public RandomAccessFile {
  //
  // Safe for concurrent use by multiple threads.
  // If Direct I/O enabled, offset, n, and scratch should be aligned properly.
-  Status Read(uint64_t offset, size_t n, Slice* result,
-              char* scratch) const override {
-    assert(scratch);
-    offset += prefixLength_;
-    auto status = file_->Read(offset, n, result, scratch);
-    if (!status.ok()) {
-      return status;
-    }
-    {
-      PERF_TIMER_GUARD(decrypt_data_nanos);
-      status = stream_->Decrypt(offset, (char*)result->data(), result->size());
-    }
+Status EncryptedRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result,
+                                       char* scratch) const {
+  assert(scratch);
+  offset += prefixLength_;
+  auto status = file_->Read(offset, n, result, scratch);
+  if (!status.ok()) {
    return status;
  }
+  {
+    PERF_TIMER_GUARD(decrypt_data_nanos);
+    status = stream_->Decrypt(offset, (char*)result->data(), result->size());
+  }
+  return status;
+}

  // Readahead the file starting from offset by n bytes for caching.
-  Status Prefetch(uint64_t offset, size_t n) override {
-    //return Status::OK();
-    return file_->Prefetch(offset + prefixLength_, n);
-  }
+Status EncryptedRandomAccessFile::Prefetch(uint64_t offset, size_t n) {
+  // return Status::OK();
+  return file_->Prefetch(offset + prefixLength_, n);
+}

  // Tries to get an unique ID for this file that will be the same each time
  // the file is opened (and will stay the same while the file is open).
@@ -169,132 +146,128 @@ class EncryptedRandomAccessFile : public RandomAccessFile {
  // a single varint.
  //
  // Note: these IDs are only valid for the duration of the process.
-  size_t GetUniqueId(char* id, size_t max_size) const override {
-    return file_->GetUniqueId(id, max_size);
-  };
+size_t EncryptedRandomAccessFile::GetUniqueId(char* id, size_t max_size) const {
+  return file_->GetUniqueId(id, max_size);
+};

-  void Hint(AccessPattern pattern) override { file_->Hint(pattern); }
+void EncryptedRandomAccessFile::Hint(AccessPattern pattern) {
+  file_->Hint(pattern);
+}

  // Indicates the upper layers if the current RandomAccessFile implementation
  // uses direct IO.
-  bool use_direct_io() const override { return file_->use_direct_io(); }
+bool EncryptedRandomAccessFile::use_direct_io() const {
+  return file_->use_direct_io();
+}

  // Use the returned alignment value to allocate
  // aligned buffer for Direct I/O
-  size_t GetRequiredBufferAlignment() const override {
-    return file_->GetRequiredBufferAlignment();
-  }
+size_t EncryptedRandomAccessFile::GetRequiredBufferAlignment() const {
+  return file_->GetRequiredBufferAlignment();
+}

  // Remove any kind of caching of data from the offset to offset+length
  // of this file. If the length is 0, then it refers to the end of file.
  // If the system is not caching the file contents, then this is a noop.
-  Status InvalidateCache(size_t offset, size_t length) override {
-    return file_->InvalidateCache(offset + prefixLength_, length);
-  }
-};
+Status EncryptedRandomAccessFile::InvalidateCache(size_t offset,
+                                                  size_t length) {
+  return file_->InvalidateCache(offset + prefixLength_, length);
+}

 // A file abstraction for sequential writing.  The implementation
 // must provide buffering since callers may append small fragments
 // at a time to the file.
-class EncryptedWritableFile : public WritableFileWrapper {
-  private:
-    std::unique_ptr<WritableFile> file_;
-    std::unique_ptr<BlockAccessCipherStream> stream_;
-    size_t prefixLength_;
-
- public:
-  // Default ctor. Prefix is assumed to be written already.
-  EncryptedWritableFile(WritableFile* f, BlockAccessCipherStream* s, size_t prefixLength)
-    : WritableFileWrapper(f), file_(f), stream_(s), prefixLength_(prefixLength) { }
-
-  Status Append(const Slice& data) override {
-    AlignedBuffer buf;
-    Status status;
-    Slice dataToAppend(data);
-    if (data.size() > 0) {
-      auto offset = file_->GetFileSize(); // size including prefix
-      // Encrypt in cloned buffer
-      buf.Alignment(GetRequiredBufferAlignment());
-      buf.AllocateNewBuffer(data.size());
-      // TODO (sagar0): Modify AlignedBuffer.Append to allow doing a memmove
-      // so that the next two lines can be replaced with buf.Append().
-      memmove(buf.BufferStart(), data.data(), data.size());
-      buf.Size(data.size());
-      {
-        PERF_TIMER_GUARD(encrypt_data_nanos);
-        status = stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize());
-      }
-      if (!status.ok()) {
-        return status;
-      }
-      dataToAppend = Slice(buf.BufferStart(), buf.CurrentSize());
+Status EncryptedWritableFile::Append(const Slice& data) {
+  AlignedBuffer buf;
+  Status status;
+  Slice dataToAppend(data);
+  if (data.size() > 0) {
+    auto offset = file_->GetFileSize();  // size including prefix
+    // Encrypt in cloned buffer
+    buf.Alignment(GetRequiredBufferAlignment());
+    buf.AllocateNewBuffer(data.size());
+    // TODO (sagar0): Modify AlignedBuffer.Append to allow doing a memmove
+    // so that the next two lines can be replaced with buf.Append().
+    memmove(buf.BufferStart(), data.data(), data.size());
+    buf.Size(data.size());
+    {
+      PERF_TIMER_GUARD(encrypt_data_nanos);
+      status = stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize());
    }
-    status = file_->Append(dataToAppend);
    if (!status.ok()) {
      return status;
    }
+    dataToAppend = Slice(buf.BufferStart(), buf.CurrentSize());
+  }
+  status = file_->Append(dataToAppend);
+  if (!status.ok()) {
    return status;
  }
+  return status;
+}

-  Status PositionedAppend(const Slice& data, uint64_t offset) override {
-    AlignedBuffer buf;
-    Status status;
-    Slice dataToAppend(data);
-    offset += prefixLength_;
-    if (data.size() > 0) {
-      // Encrypt in cloned buffer
-      buf.Alignment(GetRequiredBufferAlignment());
-      buf.AllocateNewBuffer(data.size());
-      memmove(buf.BufferStart(), data.data(), data.size());
-      buf.Size(data.size());
-      {
-        PERF_TIMER_GUARD(encrypt_data_nanos);
-        status = stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize());
-      }
-      if (!status.ok()) {
-        return status;
-      }
-      dataToAppend = Slice(buf.BufferStart(), buf.CurrentSize());
+Status EncryptedWritableFile::PositionedAppend(const Slice& data,
+                                               uint64_t offset) {
+  AlignedBuffer buf;
+  Status status;
+  Slice dataToAppend(data);
+  offset += prefixLength_;
+  if (data.size() > 0) {
+    // Encrypt in cloned buffer
+    buf.Alignment(GetRequiredBufferAlignment());
+    buf.AllocateNewBuffer(data.size());
+    memmove(buf.BufferStart(), data.data(), data.size());
+    buf.Size(data.size());
+    {
+      PERF_TIMER_GUARD(encrypt_data_nanos);
+      status = stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize());
    }
-    status = file_->PositionedAppend(dataToAppend, offset);
    if (!status.ok()) {
      return status;
    }
+    dataToAppend = Slice(buf.BufferStart(), buf.CurrentSize());
+  }
+  status = file_->PositionedAppend(dataToAppend, offset);
+  if (!status.ok()) {
    return status;
  }
+  return status;
+}

  // Indicates the upper layers if the current WritableFile implementation
  // uses direct IO.
-  bool use_direct_io() const override { return file_->use_direct_io(); }
+bool EncryptedWritableFile::use_direct_io() const {
+  return file_->use_direct_io();
+}

  // Use the returned alignment value to allocate
  // aligned buffer for Direct I/O
-  size_t GetRequiredBufferAlignment() const override {
-    return file_->GetRequiredBufferAlignment();
-  }
+size_t EncryptedWritableFile::GetRequiredBufferAlignment() const {
+  return file_->GetRequiredBufferAlignment();
+}

-    /*
-   * Get the size of valid data in the file.
-   */
-  uint64_t GetFileSize() override {
-    return file_->GetFileSize() - prefixLength_;
-  }
+/*
+ * Get the size of valid data in the file.
+ */
+uint64_t EncryptedWritableFile::GetFileSize() {
+  return file_->GetFileSize() - prefixLength_;
+}

  // Truncate is necessary to trim the file to the correct size
  // before closing. It is not always possible to keep track of the file
  // size due to whole pages writes. The behavior is undefined if called
  // with other writes to follow.
-  Status Truncate(uint64_t size) override {
-    return file_->Truncate(size + prefixLength_);
-  }
+Status EncryptedWritableFile::Truncate(uint64_t size) {
+  return file_->Truncate(size + prefixLength_);
+}

    // Remove any kind of caching of data from the offset to offset+length
  // of this file. If the length is 0, then it refers to the end of file.
  // If the system is not caching the file contents, then this is a noop.
  // This call has no effect on dirty pages in the cache.
-  Status InvalidateCache(size_t offset, size_t length) override {
-    return file_->InvalidateCache(offset + prefixLength_, length);
-  }
+Status EncryptedWritableFile::InvalidateCache(size_t offset, size_t length) {
+  return file_->InvalidateCache(offset + prefixLength_, length);
+}

  // Sync a file range with disk.
  // offset is the starting byte of the file range to be synchronized.
@@ -302,111 +275,103 @@ class EncryptedWritableFile : public WritableFileWrapper {
  // This asks the OS to initiate flushing the cached data to disk,
  // without waiting for completion.
  // Default implementation does nothing.
-  Status RangeSync(uint64_t offset, uint64_t nbytes) override {
-    return file_->RangeSync(offset + prefixLength_, nbytes);
-  }
+Status EncryptedWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) {
+  return file_->RangeSync(offset + prefixLength_, nbytes);
+}

  // PrepareWrite performs any necessary preparation for a write
  // before the write actually occurs.  This allows for pre-allocation
  // of space on devices where it can result in less file
  // fragmentation and/or less waste from over-zealous filesystem
  // pre-allocation.
-  void PrepareWrite(size_t offset, size_t len) override {
-    file_->PrepareWrite(offset + prefixLength_, len);
-  }
+void EncryptedWritableFile::PrepareWrite(size_t offset, size_t len) {
+  file_->PrepareWrite(offset + prefixLength_, len);
+}

  // Pre-allocates space for a file.
-  Status Allocate(uint64_t offset, uint64_t len) override {
-    return file_->Allocate(offset + prefixLength_, len);
-  }
-};
+Status EncryptedWritableFile::Allocate(uint64_t offset, uint64_t len) {
+  return file_->Allocate(offset + prefixLength_, len);
+}

 // A file abstraction for random reading and writing.
-class EncryptedRandomRWFile : public RandomRWFile {
-  private:
-    std::unique_ptr<RandomRWFile> file_;
-    std::unique_ptr<BlockAccessCipherStream> stream_;
-    size_t prefixLength_;

- public:
-  EncryptedRandomRWFile(RandomRWFile* f, BlockAccessCipherStream* s, size_t prefixLength)
-    : file_(f), stream_(s), prefixLength_(prefixLength) {}
-
-  // Indicates if the class makes use of direct I/O
-  // If false you must pass aligned buffer to Write()
-  bool use_direct_io() const override { return file_->use_direct_io(); }
+// Indicates if the class makes use of direct I/O
+// If false you must pass aligned buffer to Write()
+bool EncryptedRandomRWFile::use_direct_io() const {
+  return file_->use_direct_io();
+}

  // Use the returned alignment value to allocate
  // aligned buffer for Direct I/O
-  size_t GetRequiredBufferAlignment() const override {
-    return file_->GetRequiredBufferAlignment();
-  }
+size_t EncryptedRandomRWFile::GetRequiredBufferAlignment() const {
+  return file_->GetRequiredBufferAlignment();
+}

  // Write bytes in `data` at  offset `offset`, Returns Status::OK() on success.
  // Pass aligned buffer when use_direct_io() returns true.
-  Status Write(uint64_t offset, const Slice& data) override {
-    AlignedBuffer buf;
-    Status status;
-    Slice dataToWrite(data);
-    offset += prefixLength_;
-    if (data.size() > 0) {
-      // Encrypt in cloned buffer
-      buf.Alignment(GetRequiredBufferAlignment());
-      buf.AllocateNewBuffer(data.size());
-      memmove(buf.BufferStart(), data.data(), data.size());
-      buf.Size(data.size());
-      {
-        PERF_TIMER_GUARD(encrypt_data_nanos);
-        status = stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize());
-      }
-      if (!status.ok()) {
-        return status;
-      }
-      dataToWrite = Slice(buf.BufferStart(), buf.CurrentSize());
+Status EncryptedRandomRWFile::Write(uint64_t offset, const Slice& data) {
+  AlignedBuffer buf;
+  Status status;
+  Slice dataToWrite(data);
+  offset += prefixLength_;
+  if (data.size() > 0) {
+    // Encrypt in cloned buffer
+    buf.Alignment(GetRequiredBufferAlignment());
+    buf.AllocateNewBuffer(data.size());
+    memmove(buf.BufferStart(), data.data(), data.size());
+    buf.Size(data.size());
+    {
+      PERF_TIMER_GUARD(encrypt_data_nanos);
+      status = stream_->Encrypt(offset, buf.BufferStart(), buf.CurrentSize());
    }
-    status = file_->Write(offset, dataToWrite);
-    return status;
+    if (!status.ok()) {
+      return status;
+    }
+    dataToWrite = Slice(buf.BufferStart(), buf.CurrentSize());
  }
+  status = file_->Write(offset, dataToWrite);
+  return status;
+}

  // Read up to `n` bytes starting from offset `offset` and store them in
  // result, provided `scratch` size should be at least `n`.
  // Returns Status::OK() on success.
-  Status Read(uint64_t offset, size_t n, Slice* result,
-              char* scratch) const override {
-    assert(scratch);
-    offset += prefixLength_;
-    auto status = file_->Read(offset, n, result, scratch);
-    if (!status.ok()) {
-      return status;
-    }
-    {
-      PERF_TIMER_GUARD(decrypt_data_nanos);
-      status = stream_->Decrypt(offset, (char*)result->data(), result->size());
-    }
+Status EncryptedRandomRWFile::Read(uint64_t offset, size_t n, Slice* result,
+                                   char* scratch) const {
+  assert(scratch);
+  offset += prefixLength_;
+  auto status = file_->Read(offset, n, result, scratch);
+  if (!status.ok()) {
    return status;
  }
+  {
+    PERF_TIMER_GUARD(decrypt_data_nanos);
+    status = stream_->Decrypt(offset, (char*)result->data(), result->size());
+  }
+  return status;
+}

-  Status Flush() override { return file_->Flush(); }
+Status EncryptedRandomRWFile::Flush() { return file_->Flush(); }

-  Status Sync() override { return file_->Sync(); }
+Status EncryptedRandomRWFile::Sync() { return file_->Sync(); }

-  Status Fsync() override { return file_->Fsync(); }
+Status EncryptedRandomRWFile::Fsync() { return file_->Fsync(); }

-  Status Close() override { return file_->Close(); }
-};
+Status EncryptedRandomRWFile::Close() { return file_->Close(); }

-// EncryptedEnv implements an Env wrapper that adds encryption to files stored on disk.
+// EncryptedEnv implements an Env wrapper that adds encryption to files stored
+// on disk.
 class EncryptedEnv : public EnvWrapper {
 public:
-  EncryptedEnv(Env* base_env, EncryptionProvider *provider)
+  EncryptedEnv(Env* base_env, EncryptionProvider* provider)
      : EnvWrapper(base_env) {
    provider_ = provider;
  }

  // NewSequentialFile opens a file for sequential reading.
-  Status NewSequentialFile(const std::string& fname,
-                           std::unique_ptr<SequentialFile>* result,
-                           const EnvOptions& options) override {
+  virtual Status NewSequentialFile(const std::string& fname,
+                                   std::unique_ptr<SequentialFile>* result,
+                                   const EnvOptions& options) override {
    result->reset();
    if (options.use_mmap_reads) {
      return Status::InvalidArgument();
@@ -425,7 +390,8 @@ class EncryptedEnv : public EnvWrapper {
      // Read prefix
      prefixBuf.Alignment(underlying->GetRequiredBufferAlignment());
      prefixBuf.AllocateNewBuffer(prefixLength);
-      status = underlying->Read(prefixLength, &prefixSlice, prefixBuf.BufferStart());
+      status =
+          underlying->Read(prefixLength, &prefixSlice, prefixBuf.BufferStart());
      if (!status.ok()) {
        return status;
      }
@@ -433,18 +399,20 @@ class EncryptedEnv : public EnvWrapper {
    }
    // Create cipher stream
    std::unique_ptr<BlockAccessCipherStream> stream;
-    status = provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
+    status =
+        provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
    if (!status.ok()) {
      return status;
    }
-    (*result) = std::unique_ptr<SequentialFile>(new EncryptedSequentialFile(underlying.release(), stream.release(), prefixLength));
+    (*result) = std::unique_ptr<SequentialFile>(new EncryptedSequentialFile(
+        std::move(underlying), std::move(stream), prefixLength));
    return Status::OK();
  }

  // NewRandomAccessFile opens a file for random read access.
-  Status NewRandomAccessFile(const std::string& fname,
-                             std::unique_ptr<RandomAccessFile>* result,
-                             const EnvOptions& options) override {
+  virtual Status NewRandomAccessFile(const std::string& fname,
+                                     std::unique_ptr<RandomAccessFile>* result,
+                                     const EnvOptions& options) override {
    result->reset();
    if (options.use_mmap_reads) {
      return Status::InvalidArgument();
@@ -463,7 +431,8 @@ class EncryptedEnv : public EnvWrapper {
      // Read prefix
      prefixBuf.Alignment(underlying->GetRequiredBufferAlignment());
      prefixBuf.AllocateNewBuffer(prefixLength);
-      status = underlying->Read(0, prefixLength, &prefixSlice, prefixBuf.BufferStart());
+      status = underlying->Read(0, prefixLength, &prefixSlice,
+                                prefixBuf.BufferStart());
      if (!status.ok()) {
        return status;
      }
@@ -471,18 +440,20 @@ class EncryptedEnv : public EnvWrapper {
    }
    // Create cipher stream
    std::unique_ptr<BlockAccessCipherStream> stream;
-    status = provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
+    status =
+        provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
    if (!status.ok()) {
      return status;
    }
-    (*result) = std::unique_ptr<RandomAccessFile>(new EncryptedRandomAccessFile(underlying.release(), stream.release(), prefixLength));
+    (*result) = std::unique_ptr<RandomAccessFile>(new EncryptedRandomAccessFile(
+        std::move(underlying), std::move(stream), prefixLength));
    return Status::OK();
  }

  // NewWritableFile opens a file for sequential writing.
-  Status NewWritableFile(const std::string& fname,
-                         std::unique_ptr<WritableFile>* result,
-                         const EnvOptions& options) override {
+  virtual Status NewWritableFile(const std::string& fname,
+                                 std::unique_ptr<WritableFile>* result,
+                                 const EnvOptions& options) override {
    result->reset();
    if (options.use_mmap_writes) {
      return Status::InvalidArgument();
@@ -512,11 +483,13 @@ class EncryptedEnv : public EnvWrapper {
    }
    // Create cipher stream
    std::unique_ptr<BlockAccessCipherStream> stream;
-    status = provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
+    status =
+        provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
    if (!status.ok()) {
      return status;
    }
-    (*result) = std::unique_ptr<WritableFile>(new EncryptedWritableFile(underlying.release(), stream.release(), prefixLength));
+    (*result) = std::unique_ptr<WritableFile>(new EncryptedWritableFile(
+        std::move(underlying), std::move(stream), prefixLength));
    return Status::OK();
  }

@@ -527,9 +500,9 @@ class EncryptedEnv : public EnvWrapper {
  // returns non-OK.
  //
  // The returned file will only be accessed by one thread at a time.
-  Status ReopenWritableFile(const std::string& fname,
-                            std::unique_ptr<WritableFile>* result,
-                            const EnvOptions& options) override {
+  virtual Status ReopenWritableFile(const std::string& fname,
+                                    std::unique_ptr<WritableFile>* result,
+                                    const EnvOptions& options) override {
    result->reset();
    if (options.use_mmap_writes) {
      return Status::InvalidArgument();
@@ -559,26 +532,29 @@ class EncryptedEnv : public EnvWrapper {
    }
    // Create cipher stream
    std::unique_ptr<BlockAccessCipherStream> stream;
-    status = provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
+    status =
+        provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
    if (!status.ok()) {
      return status;
    }
-    (*result) = std::unique_ptr<WritableFile>(new EncryptedWritableFile(underlying.release(), stream.release(), prefixLength));
+    (*result) = std::unique_ptr<WritableFile>(new EncryptedWritableFile(
+        std::move(underlying), std::move(stream), prefixLength));
    return Status::OK();
  }

  // Reuse an existing file by renaming it and opening it as writable.
-  Status ReuseWritableFile(const std::string& fname,
-                           const std::string& old_fname,
-                           std::unique_ptr<WritableFile>* result,
-                           const EnvOptions& options) override {
+  virtual Status ReuseWritableFile(const std::string& fname,
+                                   const std::string& old_fname,
+                                   std::unique_ptr<WritableFile>* result,
+                                   const EnvOptions& options) override {
    result->reset();
    if (options.use_mmap_writes) {
      return Status::InvalidArgument();
    }
    // Open file using underlying Env implementation
    std::unique_ptr<WritableFile> underlying;
-    Status status = EnvWrapper::ReuseWritableFile(fname, old_fname, &underlying, options);
+    Status status =
+        EnvWrapper::ReuseWritableFile(fname, old_fname, &underlying, options);
    if (!status.ok()) {
      return status;
    }
@@ -601,11 +577,13 @@ class EncryptedEnv : public EnvWrapper {
    }
    // Create cipher stream
    std::unique_ptr<BlockAccessCipherStream> stream;
-    status = provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
+    status =
+        provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
    if (!status.ok()) {
      return status;
    }
-    (*result) = std::unique_ptr<WritableFile>(new EncryptedWritableFile(underlying.release(), stream.release(), prefixLength));
+    (*result) = std::unique_ptr<WritableFile>(new EncryptedWritableFile(
+        std::move(underlying), std::move(stream), prefixLength));
    return Status::OK();
  }

@@ -614,9 +592,9 @@ class EncryptedEnv : public EnvWrapper {
  // *result and returns OK.  On failure returns non-OK.
  //
  // The returned file will only be accessed by one thread at a time.
-  Status NewRandomRWFile(const std::string& fname,
-                         std::unique_ptr<RandomRWFile>* result,
-                         const EnvOptions& options) override {
+  virtual Status NewRandomRWFile(const std::string& fname,
+                                 std::unique_ptr<RandomRWFile>* result,
+                                 const EnvOptions& options) override {
    result->reset();
    if (options.use_mmap_reads || options.use_mmap_writes) {
      return Status::InvalidArgument();
@@ -639,14 +617,16 @@ class EncryptedEnv : public EnvWrapper {
      prefixBuf.AllocateNewBuffer(prefixLength);
      if (!isNewFile) {
        // File already exists, read prefix
-        status = underlying->Read(0, prefixLength, &prefixSlice, prefixBuf.BufferStart());
+        status = underlying->Read(0, prefixLength, &prefixSlice,
+                                  prefixBuf.BufferStart());
        if (!status.ok()) {
          return status;
        }
        prefixBuf.Size(prefixLength);
      } else {
        // File is new, initialize & write prefix
-        provider_->CreateNewPrefix(fname, prefixBuf.BufferStart(), prefixLength);
+        provider_->CreateNewPrefix(fname, prefixBuf.BufferStart(),
+                                   prefixLength);
        prefixBuf.Size(prefixLength);
        prefixSlice = Slice(prefixBuf.BufferStart(), prefixBuf.CurrentSize());
        // Write prefix
@@ -658,40 +638,51 @@ class EncryptedEnv : public EnvWrapper {
    }
    // Create cipher stream
    std::unique_ptr<BlockAccessCipherStream> stream;
-    status = provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
+    status =
+        provider_->CreateCipherStream(fname, options, prefixSlice, &stream);
    if (!status.ok()) {
      return status;
    }
-    (*result) = std::unique_ptr<RandomRWFile>(new EncryptedRandomRWFile(underlying.release(), stream.release(), prefixLength));
+    (*result) = std::unique_ptr<RandomRWFile>(new EncryptedRandomRWFile(
+        std::move(underlying), std::move(stream), prefixLength));
    return Status::OK();
  }

-  // Store in *result the attributes of the children of the specified directory.
-  // In case the implementation lists the directory prior to iterating the files
-  // and files are concurrently deleted, the deleted files will be omitted from
+  // Store in *result the attributes of the children of the specified
+  // directory.
+  // In case the implementation lists the directory prior to iterating the
+  // files
+  // and files are concurrently deleted, the deleted files will be omitted
+  // from
  // result.
  // The name attributes are relative to "dir".
  // Original contents of *results are dropped.
  // Returns OK if "dir" exists and "*result" contains its children.
-  //         NotFound if "dir" does not exist, the calling process does not have
+  //         NotFound if "dir" does not exist, the calling process does not
+  //         have
  //                  permission to access "dir", or if "dir" is invalid.
  //         IOError if an IO Error was encountered
-  Status GetChildrenFileAttributes(
+  virtual Status GetChildrenFileAttributes(
      const std::string& dir, std::vector<FileAttributes>* result) override {
    auto status = EnvWrapper::GetChildrenFileAttributes(dir, result);
    if (!status.ok()) {
      return status;
    }
    size_t prefixLength = provider_->GetPrefixLength();
-    for (auto it = std::begin(*result); it!=std::end(*result); ++it) {
-      assert(it->size_bytes >= prefixLength);
+    for (auto it = std::begin(*result); it != std::end(*result); ++it) {
+      // assert(it->size_bytes >= prefixLength);
+      //  breaks env_basic_test when called on directory containing
+      //  directories
+      // which makes subtraction of prefixLength worrisome since
+      // FileAttributes does not identify directories
      it->size_bytes -= prefixLength;
    }
    return Status::OK();
  }

  // Store the size of fname in *file_size.
-  Status GetFileSize(const std::string& fname, uint64_t* file_size) override {
+  virtual Status GetFileSize(const std::string& fname,
+                             uint64_t* file_size) override {
    auto status = EnvWrapper::GetFileSize(fname, file_size);
    if (!status.ok()) {
      return status;
@@ -703,7 +694,7 @@ class EncryptedEnv : public EnvWrapper {
  }

 private:
-  EncryptionProvider *provider_;
+  EncryptionProvider* provider_;
 };

 // Returns an Env that encrypts data when stored on disk and decrypts data when
@@ -12,7 +12,6 @@
 #define ROCKSDB_HDFS_FILE_C

 #include <stdio.h>
-#include <sys/time.h>
 #include <time.h>
 #include <algorithm>
 #include <iostream>
@@ -124,8 +123,9 @@ class HdfsReadableFile : virtual public SequentialFile,
    Status s;
    ROCKS_LOG_DEBUG(mylog, "[hdfs] HdfsReadableFile preading %s\n",
                    filename_.c_str());
-    ssize_t bytes_read = hdfsPread(fileSys_, hfile_, offset,
-                                   (void*)scratch, (tSize)n);
+    tSize bytes_read =
+        hdfsPread(fileSys_, hfile_, offset, static_cast<void*>(scratch),
+                  static_cast<tSize>(n));
    ROCKS_LOG_DEBUG(mylog, "[hdfs] HdfsReadableFile pread %s\n",
                    filename_.c_str());
    *result = Slice(scratch, (bytes_read < 0) ? 0 : bytes_read);
@@ -41,7 +41,7 @@
 #include <time.h>
 #include <algorithm>
 // Get nano time includes
-#if defined(OS_LINUX) || defined(OS_FREEBSD)
+#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_GNU_KFREEBSD)
 #elif defined(__MACH__)
 #include <Availability.h>
 #include <mach/clock.h>
@@ -249,7 +249,8 @@ class PosixEnv : public CompositeEnvWrapper {
  }

  uint64_t NowNanos() override {
-#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_AIX)
+#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_GNU_KFREEBSD) || \
+    defined(OS_AIX)
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC, &ts);
    return static_cast<uint64_t>(ts.tv_sec) * 1000000000 + ts.tv_nsec;
@@ -269,8 +270,8 @@ class PosixEnv : public CompositeEnvWrapper {
  }

  uint64_t NowCPUNanos() override {
-#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_AIX) || \
-    (defined(__MACH__) && defined(__MAC_10_12))
+#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_GNU_KFREEBSD) || \
+    defined(OS_AIX) || (defined(__MACH__) && defined(__MAC_10_12))
    struct timespec ts;
    clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts);
    return static_cast<uint64_t>(ts.tv_sec) * 1000000000 + ts.tv_nsec;
@@ -329,7 +330,7 @@ class PosixEnv : public CompositeEnvWrapper {
    thread_pools_[pri].IncBackgroundThreadsIfNeeded(num);
  }

-  void LowerThreadPoolIOPriority(Priority pool = LOW) override {
+  void LowerThreadPoolIOPriority(Priority pool) override {
    assert(pool >= Priority::BOTTOM && pool <= Priority::HIGH);
 #ifdef OS_LINUX
    thread_pools_[pool].LowerIOPriority();
@@ -338,13 +339,15 @@ class PosixEnv : public CompositeEnvWrapper {
 #endif
  }

-  void LowerThreadPoolCPUPriority(Priority pool = LOW) override {
+  void LowerThreadPoolCPUPriority(Priority pool) override {
    assert(pool >= Priority::BOTTOM && pool <= Priority::HIGH);
-#ifdef OS_LINUX
-    thread_pools_[pool].LowerCPUPriority();
-#else
-    (void)pool;
-#endif
+    thread_pools_[pool].LowerCPUPriority(CpuPriority::kLow);
+  }
+
+  Status LowerThreadPoolCPUPriority(Priority pool, CpuPriority pri) override {
+    assert(pool >= Priority::BOTTOM && pool <= Priority::HIGH);
+    thread_pools_[pool].LowerCPUPriority(pri);
+    return Status::OK();
  }

  std::string TimeToString(uint64_t secondsSince1970) override {
@@ -202,6 +202,79 @@ TEST_F(EnvPosixTest, DISABLED_FilePermission) {
    }
  }
 }
+
+TEST_F(EnvPosixTest, LowerThreadPoolCpuPriority) {
+  std::atomic<CpuPriority> from_priority(CpuPriority::kNormal);
+  std::atomic<CpuPriority> to_priority(CpuPriority::kNormal);
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "ThreadPoolImpl::BGThread::BeforeSetCpuPriority", [&](void* pri) {
+        from_priority.store(*reinterpret_cast<CpuPriority*>(pri));
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+      "ThreadPoolImpl::BGThread::AfterSetCpuPriority", [&](void* pri) {
+        to_priority.store(*reinterpret_cast<CpuPriority*>(pri));
+      });
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  env_->SetBackgroundThreads(1, Env::BOTTOM);
+  env_->SetBackgroundThreads(1, Env::HIGH);
+
+  auto RunTask = [&](Env::Priority pool) {
+    std::atomic<bool> called(false);
+    env_->Schedule(&SetBool, &called, pool);
+    for (int i = 0; i < kDelayMicros; i++) {
+      if (called.load()) {
+        break;
+      }
+      Env::Default()->SleepForMicroseconds(1);
+    }
+    ASSERT_TRUE(called.load());
+  };
+
+  {
+    // Same priority, no-op.
+    env_->LowerThreadPoolCPUPriority(Env::Priority::BOTTOM,
+                                     CpuPriority::kNormal);
+    RunTask(Env::Priority::BOTTOM);
+    ASSERT_EQ(from_priority, CpuPriority::kNormal);
+    ASSERT_EQ(to_priority, CpuPriority::kNormal);
+  }
+
+  {
+    // Higher priority, no-op.
+    env_->LowerThreadPoolCPUPriority(Env::Priority::BOTTOM, CpuPriority::kHigh);
+    RunTask(Env::Priority::BOTTOM);
+    ASSERT_EQ(from_priority, CpuPriority::kNormal);
+    ASSERT_EQ(to_priority, CpuPriority::kNormal);
+  }
+
+  {
+    // Lower priority from kNormal -> kLow.
+    env_->LowerThreadPoolCPUPriority(Env::Priority::BOTTOM, CpuPriority::kLow);
+    RunTask(Env::Priority::BOTTOM);
+    ASSERT_EQ(from_priority, CpuPriority::kNormal);
+    ASSERT_EQ(to_priority, CpuPriority::kLow);
+  }
+
+  {
+    // Lower priority from kLow -> kIdle.
+    env_->LowerThreadPoolCPUPriority(Env::Priority::BOTTOM, CpuPriority::kIdle);
+    RunTask(Env::Priority::BOTTOM);
+    ASSERT_EQ(from_priority, CpuPriority::kLow);
+    ASSERT_EQ(to_priority, CpuPriority::kIdle);
+  }
+
+  {
+    // Lower priority from kNormal -> kIdle for another pool.
+    env_->LowerThreadPoolCPUPriority(Env::Priority::HIGH, CpuPriority::kIdle);
+    RunTask(Env::Priority::HIGH);
+    ASSERT_EQ(from_priority, CpuPriority::kNormal);
+    ASSERT_EQ(to_priority, CpuPriority::kIdle);
+  }
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
+}
 #endif

 TEST_F(EnvPosixTest, MemoryMappedFileBuffer) {
@@ -1047,7 +1120,7 @@ TEST_P(EnvPosixTestWithParam, RandomAccessUniqueIDConcurrent) {

    // Collect and check whether the IDs are unique.
    std::unordered_set<std::string> ids;
-    for (const std::string fname : fnames) {
+    for (const std::string& fname : fnames) {
      std::unique_ptr<RandomAccessFile> file;
      std::string unique_id;
      ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions));
@@ -1061,7 +1134,7 @@ TEST_P(EnvPosixTestWithParam, RandomAccessUniqueIDConcurrent) {
    }

    // Delete the files
-    for (const std::string fname : fnames) {
+    for (const std::string& fname : fnames) {
      ASSERT_OK(env_->DeleteFile(fname));
    }

@@ -150,11 +150,11 @@ bool PosixPositionedWrite(int fd, const char* buf, size_t nbyte, off_t offset) {
 #endif

 bool IsSyncFileRangeSupported(int fd) {
-  // The approach taken in this function is to build a blacklist of cases where
-  // we know `sync_file_range` definitely will not work properly despite passing
-  // the compile-time check (`ROCKSDB_RANGESYNC_PRESENT`). If we are unsure, or
-  // if any of the checks fail in unexpected ways, we allow `sync_file_range` to
-  // be used. This way should minimize risk of impacting existing use cases.
+  // This function tracks and checks for cases where we know `sync_file_range`
+  // definitely will not work properly despite passing the compile-time check
+  // (`ROCKSDB_RANGESYNC_PRESENT`). If we are unsure, or if any of the checks
+  // fail in unexpected ways, we allow `sync_file_range` to be used. This way
+  // should minimize risk of impacting existing use cases.
  struct statfs buf;
  int ret = fstatfs(fd, &buf);
  assert(ret == 0);
@@ -176,7 +176,7 @@ bool IsSyncFileRangeSupported(int fd) {
    // ("Function not implemented").
    return false;
  }
-  // None of the cases on the blacklist matched, so allow `sync_file_range` use.
+  // None of the known cases matched, so allow `sync_file_range` use.
  return true;
 }

@@ -0,0 +1,45 @@
+add_executable(simple_example
+  simple_example.cc)
+target_link_libraries(simple_example
+  ${ROCKSDB_LIB})
+
+add_executable(column_families_example
+  column_families_example.cc)
+target_link_libraries(column_families_example
+  ${ROCKSDB_LIB})
+
+add_executable(compact_files_example
+  compact_files_example.cc)
+target_link_libraries(compact_files_example
+  ${ROCKSDB_LIB})
+
+add_executable(c_simple_example
+  c_simple_example.c)
+target_link_libraries(c_simple_example
+  ${ROCKSDB_LIB})
+
+add_executable(optimistic_transaction_example
+  optimistic_transaction_example.cc)
+target_link_libraries(optimistic_transaction_example
+  ${ROCKSDB_LIB})
+
+add_executable(transaction_example
+  transaction_example.cc)
+target_link_libraries(transaction_example
+  ${ROCKSDB_LIB})
+
+add_executable(compaction_filter_example
+  compaction_filter_example.cc)
+target_link_libraries(compaction_filter_example
+  ${ROCKSDB_LIB})
+
+add_executable(options_file_example
+  options_file_example.cc)
+target_link_libraries(options_file_example
+  ${ROCKSDB_LIB})
+
+add_executable(multi_processes_example
+  EXCLUDE_FROM_ALL
+  multi_processes_example.cc)
+target_link_libraries(multi_processes_example
+  ${ROCKSDB_LIB})
@@ -68,7 +68,7 @@ int main() {
  }

  PinnableSlice pinnable_val;
-  db->Get(ReadOptions(), db->DefaultColumnFamily(), "key1", &pinnable_val);
+  s = db->Get(ReadOptions(), db->DefaultColumnFamily(), "key1", &pinnable_val);
  assert(s.IsNotFound());
  // Reset PinnableSlice after each use and before each reuse
  pinnable_val.Reset();
@@ -7,6 +7,7 @@

 #include "file/delete_scheduler.h"

+#include <cinttypes>
 #include <thread>
 #include <vector>

@@ -65,6 +66,12 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path,
    s = fs_->DeleteFile(file_path, IOOptions(), nullptr);
    if (s.ok()) {
      sst_file_manager_->OnDeleteFile(file_path);
+      ROCKS_LOG_INFO(info_log_,
+                     "Deleted file %s immediately, rate_bytes_per_sec %" PRIi64
+                     ", total_trash_size %" PRIu64 " max_trash_db_ratio %lf",
+                     file_path.c_str(), rate_bytes_per_sec_.load(),
+                     total_trash_size_.load(), max_trash_db_ratio_.load());
+      RecordTick(stats_.get(), FILES_DELETED_IMMEDIATELY);
    }
    return s;
  }
@@ -72,6 +79,8 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path,
  // Move file to trash
  std::string trash_file;
  s = MarkAsTrash(file_path, &trash_file);
+  ROCKS_LOG_INFO(info_log_, "Mark file: %s as trash -- %s", trash_file.c_str(),
+                 s.ToString().c_str());

  if (!s.ok()) {
    ROCKS_LOG_ERROR(info_log_, "Failed to mark %s as trash -- %s",
@@ -79,10 +88,14 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path,
    s = fs_->DeleteFile(file_path, IOOptions(), nullptr);
    if (s.ok()) {
      sst_file_manager_->OnDeleteFile(file_path);
+      ROCKS_LOG_INFO(info_log_, "Deleted file %s immediately",
+                     trash_file.c_str());
+      RecordTick(stats_.get(), FILES_DELETED_IMMEDIATELY);
    }
    return s;
  }

+  RecordTick(stats_.get(), FILES_MARKED_TRASH);
  // Update the total trash size
  uint64_t trash_file_size = 0;
  fs_->GetFileSize(trash_file, IOOptions(), &trash_file_size, nullptr);
@@ -210,6 +223,8 @@ void DeleteScheduler::BackgroundEmptyTrash() {
        current_delete_rate = rate_bytes_per_sec_.load();
        start_time = env_->NowMicros();
        total_deleted_bytes = 0;
+        ROCKS_LOG_INFO(info_log_, "rate_bytes_per_sec is changed to %" PRIi64,
+                       current_delete_rate);
      }

      // Get new file to delete
@@ -233,19 +248,27 @@ void DeleteScheduler::BackgroundEmptyTrash() {
        bg_errors_[path_in_trash] = s;
      }

-      // Apply penlty if necessary
-      uint64_t total_penlty;
+      // Apply penalty if necessary
+      uint64_t total_penalty;
      if (current_delete_rate > 0) {
        // rate limiting is enabled
-        total_penlty =
+        total_penalty =
            ((total_deleted_bytes * kMicrosInSecond) / current_delete_rate);
-        while (!closing_ && !cv_.TimedWait(start_time + total_penlty)) {}
+        ROCKS_LOG_INFO(info_log_,
+                       "Rate limiting is enabled with penalty %" PRIu64
+                       "after deleting file %s",
+                       total_penalty, path_in_trash.c_str());
+        while (!closing_ && !cv_.TimedWait(start_time + total_penalty)) {
+        }
      } else {
        // rate limiting is disabled
-        total_penlty = 0;
+        total_penalty = 0;
+        ROCKS_LOG_INFO(info_log_,
+                       "Rate limiting is disabled after deleting file %s",
+                       path_in_trash.c_str());
      }
      TEST_SYNC_POINT_CALLBACK("DeleteScheduler::BackgroundEmptyTrash:Wait",
-                               &total_penlty);
+                               &total_penalty);

      if (is_complete) {
        pending_files_--;
@@ -353,9 +376,13 @@ void DeleteScheduler::WaitForEmptyTrash() {
 }

 void DeleteScheduler::MaybeCreateBackgroundThread() {
-  if(bg_thread_ == nullptr && rate_bytes_per_sec_.load() > 0) {
+  if (bg_thread_ == nullptr && rate_bytes_per_sec_.load() > 0) {
    bg_thread_.reset(
        new port::Thread(&DeleteScheduler::BackgroundEmptyTrash, this));
+    ROCKS_LOG_INFO(info_log_,
+                   "Created background thread for deletion scheduler with "
+                   "rate_bytes_per_sec: %" PRIi64,
+                   rate_bytes_per_sec_.load());
  }
 }

@@ -83,6 +83,10 @@ class DeleteScheduler {
  static Status CleanupDirectory(Env* env, SstFileManagerImpl* sfm,
                                 const std::string& path);

+  void SetStatisticsPtr(const std::shared_ptr<Statistics>& stats) {
+    stats_ = stats;
+  }
+
 private:
  Status MarkAsTrash(const std::string& file_path, std::string* path_in_trash);

@@ -137,6 +141,7 @@ class DeleteScheduler {
  // immediately
  std::atomic<double> max_trash_db_ratio_;
  static const uint64_t kMicrosInSecond = 1000 * 1000LL;
+  std::shared_ptr<Statistics> stats_;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -32,6 +32,7 @@ class DeleteSchedulerTest : public testing::Test {
          ToString(i));
      DestroyAndCreateDir(dummy_files_dirs_.back());
    }
+    stats_ = ROCKSDB_NAMESPACE::CreateDBStatistics();
  }

  ~DeleteSchedulerTest() override {
@@ -99,6 +100,7 @@ class DeleteSchedulerTest : public testing::Test {
        new SstFileManagerImpl(env_, fs, nullptr, rate_bytes_per_sec_,
                               /* max_trash_db_ratio= */ 1.1, 128 * 1024));
    delete_scheduler_ = sst_file_mgr_->delete_scheduler();
+    sst_file_mgr_->SetStatisticsPtr(stats_);
  }

  Env* env_;
@@ -106,6 +108,7 @@ class DeleteSchedulerTest : public testing::Test {
  int64_t rate_bytes_per_sec_;
  DeleteScheduler* delete_scheduler_;
  std::unique_ptr<SstFileManagerImpl> sst_file_mgr_;
+  std::shared_ptr<Statistics> stats_;
 };

 // Test the basic functionality of DeleteScheduler (Rate Limiting).
@@ -182,6 +185,8 @@ TEST_F(DeleteSchedulerTest, BasicRateLimiting) {
    ASSERT_EQ(num_files, dir_synced);

    ASSERT_EQ(CountTrashFiles(), 0);
+    ASSERT_EQ(num_files, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
+    ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
  }
 }
@@ -219,6 +224,9 @@ TEST_F(DeleteSchedulerTest, MultiDirectoryDeletionsScheduled) {
    ASSERT_EQ(0, CountTrashFiles(i));
  }

+  ASSERT_EQ(kNumFiles, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
+  ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
+
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

@@ -301,6 +309,10 @@ TEST_F(DeleteSchedulerTest, RateLimitingMultiThreaded) {

    ASSERT_EQ(CountNormalFiles(), 0);
    ASSERT_EQ(CountTrashFiles(), 0);
+    ASSERT_EQ(num_files * thread_cnt,
+              stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
+    ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
+
    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
  }
 }
@@ -318,8 +330,9 @@ TEST_F(DeleteSchedulerTest, DisableRateLimiting) {

  rate_bytes_per_sec_ = 0;
  NewDeleteScheduler();
+  constexpr int num_files = 10;

-  for (int i = 0; i < 10; i++) {
+  for (int i = 0; i < num_files; i++) {
    // Every file we delete will be deleted immediately
    std::string dummy_file = NewDummyFile("dummy.data");
    ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file, ""));
@@ -329,6 +342,9 @@ TEST_F(DeleteSchedulerTest, DisableRateLimiting) {
  }

  ASSERT_EQ(bg_delete_file, 0);
+  ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
+  ASSERT_EQ(num_files,
+            stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
@@ -365,6 +381,8 @@ TEST_F(DeleteSchedulerTest, ConflictNames) {

  auto bg_errors = delete_scheduler_->GetBackgroundErrors();
  ASSERT_EQ(bg_errors.size(), 0);
+  ASSERT_EQ(10, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
+  ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
@@ -439,9 +457,12 @@ TEST_F(DeleteSchedulerTest, StartBGEmptyTrashMultipleTimes) {

    auto bg_errors = delete_scheduler_->GetBackgroundErrors();
    ASSERT_EQ(bg_errors.size(), 0);
+    ASSERT_EQ(10, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
+    ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
  }

  ASSERT_EQ(bg_delete_file, 50);
+
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
 }

@@ -653,6 +674,8 @@ TEST_F(DeleteSchedulerTest, ImmediateDeleteOn25PercDBSize) {
  // When we end up with 26 files in trash we will start
  // deleting new files immediately
  ASSERT_EQ(fg_delete_file, 74);
+  ASSERT_EQ(26, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
+  ASSERT_EQ(74, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));

  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
@@ -122,4 +122,68 @@ bool IsWalDirSameAsDBPath(const ImmutableDBOptions* db_options) {
  return same;
 }

+IOStatus GenerateOneFileChecksum(FileSystem* fs, const std::string& file_path,
+                                 FileChecksumGenFactory* checksum_factory,
+                                 std::string* file_checksum,
+                                 std::string* file_checksum_func_name,
+                                 size_t verify_checksums_readahead_size,
+                                 bool allow_mmap_reads) {
+  if (checksum_factory == nullptr) {
+    return IOStatus::InvalidArgument("Checksum factory is invalid");
+  }
+  assert(file_checksum != nullptr);
+  assert(file_checksum_func_name != nullptr);
+
+  FileChecksumGenContext gen_context;
+  std::unique_ptr<FileChecksumGenerator> checksum_generator =
+      checksum_factory->CreateFileChecksumGenerator(gen_context);
+  uint64_t size;
+  IOStatus io_s;
+  std::unique_ptr<RandomAccessFileReader> reader;
+  {
+    std::unique_ptr<FSRandomAccessFile> r_file;
+    io_s = fs->NewRandomAccessFile(file_path, FileOptions(), &r_file, nullptr);
+    if (!io_s.ok()) {
+      return io_s;
+    }
+    io_s = fs->GetFileSize(file_path, IOOptions(), &size, nullptr);
+    if (!io_s.ok()) {
+      return io_s;
+    }
+    reader.reset(new RandomAccessFileReader(std::move(r_file), file_path));
+  }
+
+  // Found that 256 KB readahead size provides the best performance, based on
+  // experiments, for auto readahead. Experiment data is in PR #3282.
+  size_t default_max_read_ahead_size = 256 * 1024;
+  size_t readahead_size = (verify_checksums_readahead_size != 0)
+                              ? verify_checksums_readahead_size
+                              : default_max_read_ahead_size;
+
+  FilePrefetchBuffer prefetch_buffer(
+      reader.get(), readahead_size /* readadhead_size */,
+      readahead_size /* max_readahead_size */, !allow_mmap_reads /* enable */);
+
+  Slice slice;
+  uint64_t offset = 0;
+  while (size > 0) {
+    size_t bytes_to_read =
+        static_cast<size_t>(std::min(uint64_t{readahead_size}, size));
+    if (!prefetch_buffer.TryReadFromCache(offset, bytes_to_read, &slice,
+                                          false)) {
+      return IOStatus::Corruption("file read failed");
+    }
+    if (slice.size() == 0) {
+      return IOStatus::Corruption("file too small");
+    }
+    checksum_generator->Update(slice.data(), slice.size());
+    size -= slice.size();
+    offset += slice.size();
+  }
+  checksum_generator->Finalize();
+  *file_checksum = checksum_generator->GetChecksum();
+  *file_checksum_func_name = checksum_generator->Name();
+  return IOStatus::OK();
+}
+
 }  // namespace ROCKSDB_NAMESPACE
@@ -10,6 +10,7 @@
 #include "options/db_options.h"
 #include "rocksdb/env.h"
 #include "rocksdb/file_system.h"
+#include "rocksdb/sst_file_writer.h"
 #include "rocksdb/status.h"
 #include "rocksdb/types.h"

@@ -30,6 +31,12 @@ extern Status DeleteDBFile(const ImmutableDBOptions* db_options,

 extern bool IsWalDirSameAsDBPath(const ImmutableDBOptions* db_options);

+extern IOStatus GenerateOneFileChecksum(
+    FileSystem* fs, const std::string& file_path,
+    FileChecksumGenFactory* checksum_factory, std::string* file_checksum,
+    std::string* file_checksum_func_name,
+    size_t verify_checksums_readahead_size, bool allow_mmap_reads);
+
 inline IOStatus PrepareIOFromReadOptions(const ReadOptions& ro, Env* env,
                                         IOOptions& opts) {
  if (!env) {
@@ -132,7 +132,7 @@ class RandomAccessFileReader {

  FSRandomAccessFile* file() { return file_.get(); }

-  std::string file_name() const { return file_name_; }
+  const std::string& file_name() const { return file_name_; }

  bool use_direct_io() const { return file_->use_direct_io(); }

@@ -135,6 +135,11 @@ class SstFileManagerImpl : public SstFileManager {
  // once in the object's lifetime, and before the destructor
  void Close();

+  void SetStatisticsPtr(const std::shared_ptr<Statistics>& stats) override {
+    stats_ = stats;
+    delete_scheduler_.SetStatisticsPtr(stats);
+  }
+
 private:
  // REQUIRES: mutex locked
  void OnAddFileImpl(const std::string& file_path, uint64_t file_size,
@@ -190,6 +195,7 @@ class SstFileManagerImpl : public SstFileManager {
  std::list<ErrorHandler*> error_handler_list_;
  // Pointer to ErrorHandler instance that is currently processing recovery
  ErrorHandler* cur_instance_;
+  std::shared_ptr<Statistics> stats_;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -236,7 +236,7 @@ const char* WritableFileWriter::GetFileChecksumFuncName() const {
  if (checksum_generator_ != nullptr) {
    return checksum_generator_->Name();
  } else {
-    return kUnknownFileChecksumFuncName.c_str();
+    return kUnknownFileChecksumFuncName;
  }
 }

@@ -162,10 +162,7 @@ class HdfsEnv : public Env {
    return posixEnv->TimeToString(number);
  }

-  static uint64_t gettid() {
-    assert(sizeof(pthread_t) <= sizeof(uint64_t));
-    return (uint64_t)pthread_self();
-  }
+  static uint64_t gettid() { return Env::Default()->GetThreadID(); }

  uint64_t GetThreadID() const override { return HdfsEnv::gettid(); }

@@ -209,8 +206,7 @@ class HdfsEnv : public Env {
    std::string portStr = (rem == 0 ? remaining :
                           remaining.substr(0, rem));

-    tPort port;
-    port = atoi(portStr.c_str());
+    tPort port = static_cast<tPort>(atoi(portStr.c_str()));
    if (port == 0) {
      throw HdfsFatalException("Bad host-port for hdfs " + uri);
    }
@@ -111,10 +111,16 @@ struct RangePtr {
  RangePtr(const Slice* s, const Slice* l) : start(s), limit(l) {}
 };

+// It is valid that files_checksums and files_checksum_func_names are both
+// empty (no checksum informaiton is provided for ingestion). Otherwise,
+// their sizes should be the same as external_files. The file order should
+// be the same in three vectors and guaranteed by the caller.
 struct IngestExternalFileArg {
  ColumnFamilyHandle* column_family = nullptr;
  std::vector<std::string> external_files;
  IngestExternalFileOptions options;
+  std::vector<std::string> files_checksums;
+  std::vector<std::string> files_checksum_func_names;
 };

 struct GetMergeOperandsOptions {
@@ -1538,6 +1544,13 @@ class DB {
  // Returns Status::OK if identity could be set properly
  virtual Status GetDbIdentity(std::string& identity) const = 0;

+  // Return a unique identifier for each DB object that is opened
+  // This DB session ID should be unique among all open DB instances on all
+  // hosts, and should be unique among re-openings of the same or other DBs.
+  // (Two open DBs have the same identity from other function GetDbIdentity when
+  // one is physically copied from the other.)
+  virtual Status GetDbSessionId(std::string& session_id) const = 0;
+
  // Returns default column family handle
  virtual ColumnFamilyHandle* DefaultColumnFamily() const = 0;

@@ -61,6 +61,13 @@ class FileSystem;

 const size_t kDefaultPageSize = 4 * 1024;

+enum class CpuPriority {
+  kIdle = 0,
+  kLow = 1,
+  kNormal = 2,
+  kHigh = 3,
+};
+
 // Options while opening a file to read/write
 struct EnvOptions {
  // Construct with default Options
@@ -474,6 +481,13 @@ class Env {
  // Lower IO priority for threads from the specified pool.
  virtual void LowerThreadPoolIOPriority(Priority /*pool*/ = LOW) {}

+  // Lower CPU priority for threads from the specified pool.
+  virtual Status LowerThreadPoolCPUPriority(Priority /*pool*/,
+                                            CpuPriority /*pri*/) {
+    return Status::NotSupported(
+        "Env::LowerThreadPoolCPUPriority(Priority, CpuPriority) not supported");
+  }
+
  // Lower CPU priority for threads from the specified pool.
  virtual void LowerThreadPoolCPUPriority(Priority /*pool*/ = LOW) {}

@@ -1355,14 +1369,18 @@ class EnvWrapper : public Env {
    return target_->IncBackgroundThreadsIfNeeded(num, pri);
  }

-  void LowerThreadPoolIOPriority(Priority pool = LOW) override {
+  void LowerThreadPoolIOPriority(Priority pool) override {
    target_->LowerThreadPoolIOPriority(pool);
  }

-  void LowerThreadPoolCPUPriority(Priority pool = LOW) override {
+  void LowerThreadPoolCPUPriority(Priority pool) override {
    target_->LowerThreadPoolCPUPriority(pool);
  }

+  Status LowerThreadPoolCPUPriority(Priority pool, CpuPriority pri) override {
+    return target_->LowerThreadPoolCPUPriority(pool, pri);
+  }
+
  std::string TimeToString(uint64_t time) override {
    return target_->TimeToString(time);
  }
@@ -169,8 +169,10 @@ class CTREncryptionProvider : public EncryptionProvider {
  virtual ~CTREncryptionProvider() {}

  // GetPrefixLength returns the length of the prefix that is added to every
-  // file and used for storing encryption options. For optimal performance, the
-  // prefix length should be a multiple of the page size.
+  // file
+  // and used for storing encryption options.
+  // For optimal performance, the prefix length should be a multiple of
+  // the page size.
  virtual size_t GetPrefixLength() override;

  // CreateNewPrefix initialized an allocated block of prefix memory
@@ -194,13 +196,243 @@ class CTREncryptionProvider : public EncryptionProvider {
                                          size_t blockSize);

  // CreateCipherStreamFromPrefix creates a block access cipher stream for a
-  // file given given name and options. The given prefix is already decrypted.
+  // file given
+  // given name and options. The given prefix is already decrypted.
  virtual Status CreateCipherStreamFromPrefix(
      const std::string& fname, const EnvOptions& options,
      uint64_t initialCounter, const Slice& iv, const Slice& prefix,
      std::unique_ptr<BlockAccessCipherStream>* result);
 };

+class EncryptedSequentialFile : public SequentialFile {
+ protected:
+  std::unique_ptr<SequentialFile> file_;
+  std::unique_ptr<BlockAccessCipherStream> stream_;
+  uint64_t offset_;
+  size_t prefixLength_;
+
+ public:
+  // Default ctor. Given underlying sequential file is supposed to be at
+  // offset == prefixLength.
+  EncryptedSequentialFile(std::unique_ptr<SequentialFile>&& f,
+                          std::unique_ptr<BlockAccessCipherStream>&& s,
+                          size_t prefixLength)
+      : file_(std::move(f)),
+        stream_(std::move(s)),
+        offset_(prefixLength),
+        prefixLength_(prefixLength) {}
+
+  // Read up to "n" bytes from the file.  "scratch[0..n-1]" may be
+  // written by this routine.  Sets "*result" to the data that was
+  // read (including if fewer than "n" bytes were successfully read).
+  // May set "*result" to point at data in "scratch[0..n-1]", so
+  // "scratch[0..n-1]" must be live when "*result" is used.
+  // If an error was encountered, returns a non-OK status.
+  //
+  // REQUIRES: External synchronization
+  virtual Status Read(size_t n, Slice* result, char* scratch) override;
+
+  // Skip "n" bytes from the file. This is guaranteed to be no
+  // slower that reading the same data, but may be faster.
+  //
+  // If end of file is reached, skipping will stop at the end of the
+  // file, and Skip will return OK.
+  //
+  // REQUIRES: External synchronization
+  virtual Status Skip(uint64_t n) override;
+
+  // Indicates the upper layers if the current SequentialFile implementation
+  // uses direct IO.
+  virtual bool use_direct_io() const override;
+
+  // Use the returned alignment value to allocate
+  // aligned buffer for Direct I/O
+  virtual size_t GetRequiredBufferAlignment() const override;
+
+  // Remove any kind of caching of data from the offset to offset+length
+  // of this file. If the length is 0, then it refers to the end of file.
+  // If the system is not caching the file contents, then this is a noop.
+  virtual Status InvalidateCache(size_t offset, size_t length) override;
+
+  // Positioned Read for direct I/O
+  // If Direct I/O enabled, offset, n, and scratch should be properly aligned
+  virtual Status PositionedRead(uint64_t offset, size_t n, Slice* result,
+                                char* scratch) override;
+};
+
+// A file abstraction for randomly reading the contents of a file.
+class EncryptedRandomAccessFile : public RandomAccessFile {
+ protected:
+  std::unique_ptr<RandomAccessFile> file_;
+  std::unique_ptr<BlockAccessCipherStream> stream_;
+  size_t prefixLength_;
+
+ public:
+  EncryptedRandomAccessFile(std::unique_ptr<RandomAccessFile>&& f,
+                            std::unique_ptr<BlockAccessCipherStream>&& s,
+                            size_t prefixLength)
+      : file_(std::move(f)),
+        stream_(std::move(s)),
+        prefixLength_(prefixLength) {}
+
+  // Read up to "n" bytes from the file starting at "offset".
+  // "scratch[0..n-1]" may be written by this routine.  Sets "*result"
+  // to the data that was read (including if fewer than "n" bytes were
+  // successfully read).  May set "*result" to point at data in
+  // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
+  // "*result" is used.  If an error was encountered, returns a non-OK
+  // status.
+  //
+  // Safe for concurrent use by multiple threads.
+  // If Direct I/O enabled, offset, n, and scratch should be aligned properly.
+  virtual Status Read(uint64_t offset, size_t n, Slice* result,
+                      char* scratch) const override;
+
+  // Readahead the file starting from offset by n bytes for caching.
+  virtual Status Prefetch(uint64_t offset, size_t n) override;
+
+  // Tries to get an unique ID for this file that will be the same each time
+  // the file is opened (and will stay the same while the file is open).
+  // Furthermore, it tries to make this ID at most "max_size" bytes. If such an
+  // ID can be created this function returns the length of the ID and places it
+  // in "id"; otherwise, this function returns 0, in which case "id"
+  // may not have been modified.
+  //
+  // This function guarantees, for IDs from a given environment, two unique ids
+  // cannot be made equal to each other by adding arbitrary bytes to one of
+  // them. That is, no unique ID is the prefix of another.
+  //
+  // This function guarantees that the returned ID will not be interpretable as
+  // a single varint.
+  //
+  // Note: these IDs are only valid for the duration of the process.
+  virtual size_t GetUniqueId(char* id, size_t max_size) const override;
+
+  virtual void Hint(AccessPattern pattern) override;
+
+  // Indicates the upper layers if the current RandomAccessFile implementation
+  // uses direct IO.
+  virtual bool use_direct_io() const override;
+
+  // Use the returned alignment value to allocate
+  // aligned buffer for Direct I/O
+  virtual size_t GetRequiredBufferAlignment() const override;
+
+  // Remove any kind of caching of data from the offset to offset+length
+  // of this file. If the length is 0, then it refers to the end of file.
+  // If the system is not caching the file contents, then this is a noop.
+  virtual Status InvalidateCache(size_t offset, size_t length) override;
+};
+
+// A file abstraction for sequential writing.  The implementation
+// must provide buffering since callers may append small fragments
+// at a time to the file.
+class EncryptedWritableFile : public WritableFileWrapper {
+ protected:
+  std::unique_ptr<WritableFile> file_;
+  std::unique_ptr<BlockAccessCipherStream> stream_;
+  size_t prefixLength_;
+
+ public:
+  // Default ctor. Prefix is assumed to be written already.
+  EncryptedWritableFile(std::unique_ptr<WritableFile>&& f,
+                        std::unique_ptr<BlockAccessCipherStream>&& s,
+                        size_t prefixLength)
+      : WritableFileWrapper(f.get()),
+        file_(std::move(f)),
+        stream_(std::move(s)),
+        prefixLength_(prefixLength) {}
+
+  Status Append(const Slice& data) override;
+
+  Status PositionedAppend(const Slice& data, uint64_t offset) override;
+
+  // Indicates the upper layers if the current WritableFile implementation
+  // uses direct IO.
+  virtual bool use_direct_io() const override;
+
+  // Use the returned alignment value to allocate
+  // aligned buffer for Direct I/O
+  virtual size_t GetRequiredBufferAlignment() const override;
+
+  /*
+   * Get the size of valid data in the file.
+   */
+  virtual uint64_t GetFileSize() override;
+
+  // Truncate is necessary to trim the file to the correct size
+  // before closing. It is not always possible to keep track of the file
+  // size due to whole pages writes. The behavior is undefined if called
+  // with other writes to follow.
+  virtual Status Truncate(uint64_t size) override;
+
+  // Remove any kind of caching of data from the offset to offset+length
+  // of this file. If the length is 0, then it refers to the end of file.
+  // If the system is not caching the file contents, then this is a noop.
+  // This call has no effect on dirty pages in the cache.
+  virtual Status InvalidateCache(size_t offset, size_t length) override;
+
+  // Sync a file range with disk.
+  // offset is the starting byte of the file range to be synchronized.
+  // nbytes specifies the length of the range to be synchronized.
+  // This asks the OS to initiate flushing the cached data to disk,
+  // without waiting for completion.
+  // Default implementation does nothing.
+  virtual Status RangeSync(uint64_t offset, uint64_t nbytes) override;
+
+  // PrepareWrite performs any necessary preparation for a write
+  // before the write actually occurs.  This allows for pre-allocation
+  // of space on devices where it can result in less file
+  // fragmentation and/or less waste from over-zealous filesystem
+  // pre-allocation.
+  virtual void PrepareWrite(size_t offset, size_t len) override;
+
+  // Pre-allocates space for a file.
+  virtual Status Allocate(uint64_t offset, uint64_t len) override;
+};
+
+// A file abstraction for random reading and writing.
+class EncryptedRandomRWFile : public RandomRWFile {
+ protected:
+  std::unique_ptr<RandomRWFile> file_;
+  std::unique_ptr<BlockAccessCipherStream> stream_;
+  size_t prefixLength_;
+
+ public:
+  EncryptedRandomRWFile(std::unique_ptr<RandomRWFile>&& f,
+                        std::unique_ptr<BlockAccessCipherStream>&& s,
+                        size_t prefixLength)
+      : file_(std::move(f)),
+        stream_(std::move(s)),
+        prefixLength_(prefixLength) {}
+
+  // Indicates if the class makes use of direct I/O
+  // If false you must pass aligned buffer to Write()
+  virtual bool use_direct_io() const override;
+
+  // Use the returned alignment value to allocate
+  // aligned buffer for Direct I/O
+  virtual size_t GetRequiredBufferAlignment() const override;
+
+  // Write bytes in `data` at  offset `offset`, Returns Status::OK() on success.
+  // Pass aligned buffer when use_direct_io() returns true.
+  virtual Status Write(uint64_t offset, const Slice& data) override;
+
+  // Read up to `n` bytes starting from offset `offset` and store them in
+  // result, provided `scratch` size should be at least `n`.
+  // Returns Status::OK() on success.
+  virtual Status Read(uint64_t offset, size_t n, Slice* result,
+                      char* scratch) const override;
+
+  virtual Status Flush() override;
+
+  virtual Status Sync() override;
+
+  virtual Status Fsync() override;
+
+  virtual Status Close() override;
+};
+
 }  // namespace ROCKSDB_NAMESPACE

 #endif  // !defined(ROCKSDB_LITE)
@@ -18,6 +18,11 @@

 namespace ROCKSDB_NAMESPACE {

+// The unknown file checksum.
+constexpr char kUnknownFileChecksum[] = "";
+// The unknown sst file checksum function name.
+constexpr char kUnknownFileChecksumFuncName[] = "Unknown";
+
 struct FileChecksumGenContext {
  std::string file_name;
 };
@@ -51,13 +51,6 @@ class InternalKeyComparator;
 class WalFilter;
 class FileSystem;

-enum class CpuPriority {
-  kIdle = 0,
-  kLow = 1,
-  kNormal = 2,
-  kHigh = 3,
-};
-
 // DB contents are stored in a set of blocks, each of which holds a
 // sequence of key,value pairs.  Each block may be compressed before
 // being stored in a file.  The following enum describes which
@@ -1571,6 +1564,26 @@ struct IngestExternalFileOptions {
  // Using a large readahead size (> 2MB) can typically improve the performance
  // of forward iteration on spinning disks.
  size_t verify_checksums_readahead_size = 0;
+  // Set to TRUE if user wants to verify the sst file checksum of ingested
+  // files. The DB checksum function will generate the checksum of each
+  // ingested file (if file_checksum_gen_factory is set) and compare the
+  // checksum function name and checksum with the ingested checksum information.
+  //
+  // If this option is set to True: 1) if DB does not enable checksum
+  // (file_checksum_gen_factory == nullptr), the ingested checksum information
+  // will be ignored; 2) If DB enable the checksum function, we calculate the
+  // sst file checksum after the file is moved or copied and compare the
+  // checksum and checksum name. If checksum or checksum function name does
+  // not match, ingestion will be failed. If the verification is sucessful,
+  // checksum and checksum function name will be stored in Manifest.
+  // If this option is set to FALSE, 1) if DB does not enable checksum,
+  // the ingested checksum information will be ignored; 2) if DB enable the
+  // checksum, we only verify the ingested checksum function name and we
+  // trust the ingested checksum. If the checksum function name matches, we
+  // store the checksum in Manifest. DB does not calculate the checksum during
+  // ingestion. However, if no checksum information is provided with the
+  // ingested files, DB will generate the checksum and store in the Manifest.
+  bool verify_file_checksum = true;
 };

 enum TraceFilterType : uint64_t {
@@ -56,6 +56,12 @@ class PersistentCache {
  virtual StatsType Stats() = 0;

  virtual std::string GetPrintableOptions() const = 0;
+
+  // Return a new numeric id.  May be used by multiple clients who are
+  // sharding the same persistent cache to partition the key space.  Typically
+  // the client will allocate a new id at startup and prepend the id to its
+  // cache keys.
+  virtual uint64_t NewId() = 0;
 };

 // Factor method to create a new persistent cache
@@ -11,7 +11,7 @@ namespace ROCKSDB_NAMESPACE {

 class SSTDumpTool {
 public:
-  int Run(int argc, char** argv, Options options = Options());
+  int Run(int argc, char const* const* argv, Options options = Options());
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -11,6 +11,7 @@
 #include <vector>

 #include "rocksdb/file_system.h"
+#include "rocksdb/statistics.h"
 #include "rocksdb/status.h"

 namespace ROCKSDB_NAMESPACE {
@@ -80,6 +81,9 @@ class SstFileManager {
  // Return the total size of trash files
  // thread-safe
  virtual uint64_t GetTotalTrashSize() = 0;
+
+  // Set the statistics ptr to dump the stat information
+  virtual void SetStatisticsPtr(const std::shared_ptr<Statistics>& stats) = 0;
 };

 // Create a new SstFileManager that can be shared among multiple RocksDB
@@ -360,6 +360,12 @@ enum Tickers : uint32_t {
  //           <= BLOCK_CACHE_COMPRESSION_DICT_ADD
  BLOCK_CACHE_COMPRESSION_DICT_ADD_REDUNDANT,

+  // # of files marked as trash by sst file manager and will be deleted
+  // later by background thread.
+  FILES_MARKED_TRASH,
+  // # of files deleted immediately by sst file manger through delete scheduler.
+  FILES_DELETED_IMMEDIATELY,
+
  TICKER_ENUM_MAX
 };

@@ -200,6 +200,40 @@ struct BlockBasedTableOptions {
  // incompatible with block-based filters.
  bool partition_filters = false;

+  // EXPERIMENTAL Option to generate Bloom filters that minimize memory
+  // internal fragmentation.
+  //
+  // When false, malloc_usable_size is not available, or format_version < 5,
+  // filters are generated without regard to internal fragmentation when
+  // loaded into memory (historical behavior). When true (and
+  // malloc_usable_size is available and format_version >= 5), then Bloom
+  // filters are generated to "round up" and "round down" their sizes to
+  // minimize internal fragmentation when loaded into memory, assuming the
+  // reading DB has the same memory allocation characteristics as the
+  // generating DB. This option does not break forward or backward
+  // compatibility.
+  //
+  // While individual filters will vary in bits/key and false positive rate
+  // when setting is true, the implementation attempts to maintain a weighted
+  // average FP rate for filters consistent with this option set to false.
+  //
+  // With Jemalloc for example, this setting is expected to save about 10% of
+  // the memory footprint and block cache charge of filters, while increasing
+  // disk usage of filters by about 1-2% due to encoding efficiency losses
+  // with variance in bits/key.
+  //
+  // NOTE: Because some memory counted by block cache might be unmapped pages
+  // within internal fragmentation, this option can increase observed RSS
+  // memory usage. With cache_index_and_filter_blocks=true, this option makes
+  // the block cache better at using space it is allowed.
+  //
+  // NOTE: Do not set to true if you do not trust malloc_usable_size. With
+  // this option, RocksDB might access an allocated memory object beyond its
+  // original size if malloc_usable_size says it is safe to do so. While this
+  // can be considered bad practice, it should not produce undefined behavior
+  // unless malloc_usable_size is buggy or broken.
+  bool optimize_filters_for_memory = false;
+
  // Use delta encoding to compress keys in blocks.
  // ReadOptions::pin_data requires this option to be disabled.
  //