Fix the logic of setting read_amp_bytes_per_bit from OPTIONS file (#7680)

Summary:
Instead of using `EncodeFixed32` which always serialize a integer to
little endian, we should use the local machine's endianness when
populating a native data structure during options parsing.
Without this fix, `read_amp_bytes_per_bit` may be populated incorrectly
on big-endian machines.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7680

Test Plan: make check

Reviewed By: pdillinger

Differential Revision: D24999166

Pulled By: riversand963

fbshipit-source-id: dc603cff6e17f8fa32479ce6df93b93082e6b0c4

.circleci/config.yml

@@ -6,64 +6,12 @@ orbs:
 executors:
   windows-2xlarge:
     machine:
-      image: 'windows-server-2019-vs2019:201908-06'
+      image: 'windows-server-2019-vs2019:stable'
       resource_class: windows.2xlarge
       shell: bash.exe
 
 jobs:
-  build-linux:
-    machine:
-      image: ubuntu-1604:201903-01
-    resource_class: 2xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: pyenv global 3.5.2
-      - run: sudo apt-get update -y
-      - run: sudo apt-get install -y libgflags-dev
-      - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 make J=32 all check -j32
-
-  build-linux-release:
-    machine:
-      image: ubuntu-1604:201903-01
-    resource_class: 2xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: make release -j32
-
-  build-linux-lite:
-    machine:
-      image: ubuntu-1604:201903-01 
-    resource_class: 2xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: pyenv global 3.5.2
-      - run: SKIP_FORMAT_BUCK_CHECKS=1 PRINT_PARALLEL_OUTPUTS=1 LITE=1 make J=32 all check -j32
-
-  build-linux-lite-release:
-    machine:
-      image: ubuntu-1604:201903-01
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: make release -j32
-
-  build-linux-clang-no-test:
-    machine:
-      image: ubuntu-1604:201903-01
-    resource_class: 2xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: USE_CLANG=1 make all -j32
-
-  build-linux-cmake:
-    machine:
-      image: ubuntu-1604:201903-01
-    resource_class: 2xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make -j32
-
-  build-windows:
+  build:
     executor: windows-2xlarge
     
     environment:
@@ -106,26 +54,3 @@ jobs:
           shell: powershell.exe
           command: |
             build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,env_basic_test,env_test,db_merge_operand_test -Concurrency 16
-
-workflows:
-  build-linux:
-    jobs:
-      - build-linux
-  build-linux-lite:
-    jobs:
-      - build-linux-lite
-  build-linux-release:
-    jobs:
-      - build-linux-release
-  build-linux-lite-release:
-    jobs:
-      - build-linux-lite-release
-  build-linux-clang-no-test:
-    jobs:
-      - build-linux-clang-no-test
-  build-linux-cmake:
-    jobs:
-      - build-linux-cmake
-  build-windows:
-    jobs:
-      - build-windows

.github/workflows/sanity_check.yml

@@ -35,7 +35,7 @@ jobs:
         args: https://raw.githubusercontent.com/llvm-mirror/clang/master/tools/clang-format/clang-format-diff.py
 
     - name: Check format
-      run: VERBOSE_CHECK=1 make check-format
+      run: make check-format
 
     - name: Compare buckify output
       run: make check-buck-targets

.gitignore

@@ -85,5 +85,3 @@ buckifier/*.pyc
 buckifier/__pycache__
 
 compile_commands.json
-clang-format-diff.py
-.py3/

.travis.yml

@@ -53,8 +53,6 @@ env:
   - JOB_NAME=cmake-gcc9 # 3-5 minutes
   - JOB_NAME=cmake-gcc9-c++20 # 3-5 minutes
   - JOB_NAME=cmake-mingw # 3 minutes
-  - JOB_NAME=make-gcc4.8
-  - JOB_NAME=status_checked
 
 matrix:
   exclude:
@@ -66,8 +64,6 @@ matrix:
     env: JOB_NAME=cmake-gcc9-c++20
   - os: osx
     env: JOB_NAME=cmake-mingw
-  - os: osx
-    env: JOB_NAME=make-gcc4.8
   - os: osx
     arch: ppc64le
   - os: osx
@@ -75,15 +71,9 @@ matrix:
   - os : linux
     arch: arm64
     env: JOB_NAME=cmake-mingw
-  - os : linux
-    arch: arm64
-    env: JOB_NAME=make-gcc4.8
   - os: linux
     arch: ppc64le
     env: JOB_NAME=cmake-mingw
-  - os: linux
-    arch: ppc64le
-    env: JOB_NAME=make-gcc4.8
   - os: linux
     compiler: clang
   # Exclude all but most unique cmake variants for pull requests, but build all in branches
@@ -207,17 +197,6 @@ matrix:
     os: linux
     arch: ppc64le
     env: JOB_NAME=cmake-gcc9-c++20
-  - if: type = pull_request
-    os : osx
-    env: JOB_NAME=status_checked
-  - if: type = pull_request
-    os : linux
-    arch: arm64
-    env: JOB_NAME=status_checked
-  - if: type = pull_request
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=status_checked
 
 install:
   - if [ "${TRAVIS_OS_NAME}" == osx ]; then
@@ -234,10 +213,6 @@ install:
   - if [ "${JOB_NAME}" == cmake-mingw ]; then
       sudo apt-get install -y mingw-w64 ;
     fi
-  - if [ "${JOB_NAME}" == make-gcc4.8 ]; then
-      sudo apt-get install -y g++-4.8 ;
-      CC=gcc-4.8 && CXX=g++-4.8;
-    fi
   - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then
       CMAKE_DIST_URL="https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/cmake-3.14.5-Linux-$(uname -m).tar.bz2";
       TAR_OPT="--strip-components=1 -xj";
@@ -294,7 +269,7 @@ script:
       OPT=-DTRAVIS V=1 make rocksdbjava jtest
       ;;
     lite_build)
-      OPT='-DTRAVIS -DROCKSDB_LITE' V=1 make -j4 all
+      OPT='-DTRAVIS -DROCKSDB_LITE' V=1 make -j4 static_lib tools
       ;;
     examples)
       OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4
@@ -310,10 +285,7 @@ script:
           ;;
       esac
 
-      mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=0 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. && make -j4 && cd  .. && rm -rf build && mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release $OPT && make -j4 rocksdb rocksdbjni
-      ;;
-    make-gcc4.8)
-      OPT=-DTRAVIS V=1 SKIP_LINK=1 make -j4 all && [ "Linking broken because libgflags compiled with newer ABI" ]
+      mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release $OPT && make -j4 rocksdb rocksdbjni
       ;;
     status_checked)
       OPT=-DTRAVIS V=1 ASSERT_STATUS_CHECKED=1 make -j4 check_some

CMakeLists.txt

@@ -107,32 +107,18 @@ else()
     endif()
   endif()
 
+  # No config file for this
   if(WITH_GFLAGS)
-    # Config with namespace available since gflags 2.2.2
-    option(GFLAGS_USE_TARGET_NAMESPACE "Use gflags import target with namespace." ON)
-    find_package(gflags CONFIG)
-    if(gflags_FOUND)
-      if(TARGET ${GFLAGS_TARGET})
-        # Config with GFLAGS_TARGET available since gflags 2.2.0
-        list(APPEND THIRDPARTY_LIBS ${GFLAGS_TARGET})
-      else()
-        # Config with GFLAGS_LIBRARIES available since gflags 2.1.0
-        list(APPEND THIRDPARTY_LIBS ${GFLAGS_LIBRARIES})
-      endif()
-    else()
-      find_package(gflags REQUIRED)
-      list(APPEND THIRDPARTY_LIBS gflags::gflags)
-    endif()
+    find_package(gflags REQUIRED)
     add_definitions(-DGFLAGS=1)
+    include_directories(${gflags_INCLUDE_DIR})
+    list(APPEND THIRDPARTY_LIBS gflags::gflags)
   endif()
 
   if(WITH_SNAPPY)
-    find_package(Snappy CONFIG)
-    if(NOT Snappy_FOUND)
-      find_package(Snappy REQUIRED)
-    endif()
+    find_package(snappy REQUIRED)
     add_definitions(-DSNAPPY)
-    list(APPEND THIRDPARTY_LIBS Snappy::snappy)
+    list(APPEND THIRDPARTY_LIBS snappy::snappy)
   endif()
 
   if(WITH_ZLIB)
@@ -523,6 +509,7 @@ endif()
 
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/include)
+include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.8.1/fused-src)
 if(WITH_FOLLY_DISTRIBUTED_MUTEX)
   include_directories(${PROJECT_SOURCE_DIR}/third-party/folly)
 endif()
@@ -539,9 +526,6 @@ set(SOURCES
         db/blob/blob_file_addition.cc
         db/blob/blob_file_garbage.cc
         db/blob/blob_file_meta.cc
-        db/blob/blob_log_format.cc
-        db/blob/blob_log_reader.cc
-        db/blob/blob_log_writer.cc
         db/builder.cc
         db/c.cc
         db/column_family.cc
@@ -733,6 +717,9 @@ set(SOURCES
         utilities/blob_db/blob_db_impl_filesnapshot.cc
         utilities/blob_db/blob_dump_tool.cc
         utilities/blob_db/blob_file.cc
+        utilities/blob_db/blob_log_reader.cc
+        utilities/blob_db/blob_log_writer.cc
+        utilities/blob_db/blob_log_format.cc
         utilities/cassandra/cassandra_compaction_filter.cc
         utilities/cassandra/format.cc
         utilities/cassandra/merge_operator.cc
@@ -803,13 +790,9 @@ if(WIN32)
     port/win/env_win.cc
     port/win/env_default.cc
     port/win/port_win.cc
-    port/win/win_logger.cc)
-  if(NOT MINGW)
-    # Mingw only supports std::thread when using
-    # posix threads.
-    list(APPEND SOURCES
-      port/win/win_thread.cc)
-  endif()
+    port/win/win_logger.cc
+    port/win/win_thread.cc)
+
 if(WITH_XPRESS)
   list(APPEND SOURCES
     port/win/xpress_win.cc)
@@ -856,12 +839,12 @@ else()
 endif()
 
 add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
-target_link_libraries(${ROCKSDB_STATIC_LIB} PRIVATE
+target_link_libraries(${ROCKSDB_STATIC_LIB}
   ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
 if(ROCKSDB_BUILD_SHARED)
   add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
-  target_link_libraries(${ROCKSDB_SHARED_LIB} PRIVATE
+  target_link_libraries(${ROCKSDB_SHARED_LIB}
     ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
   if(WIN32)
@@ -889,16 +872,6 @@ else()
 endif()
 
 option(WITH_JNI "build with JNI" OFF)
-# Tests are excluded from Release builds
-CMAKE_DEPENDENT_OPTION(WITH_TESTS "build with tests" ON
-  "CMAKE_BUILD_TYPE STREQUAL Debug" OFF)
-option(WITH_BENCHMARK_TOOLS "build with benchmarks" ON)
-option(WITH_CORE_TOOLS "build with ldb and sst_dump" ON)
-option(WITH_TOOLS "build with tools" ON)
-
-if(WITH_TESTS OR WITH_BENCHMARK_TOOLS OR WITH_TOOLS OR WITH_JNI OR JNI)
-  include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.8.1/fused-src)
-endif()
 if(WITH_JNI OR JNI)
   message(STATUS "JNI library is enabled")
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
@@ -936,8 +909,6 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
 
   install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
 
-  install(DIRECTORY "${PROJECT_SOURCE_DIR}/cmake/modules" COMPONENT devel DESTINATION ${package_config_destination})
-
   install(
     TARGETS ${ROCKSDB_STATIC_LIB}
     EXPORT RocksDBTargets
@@ -974,11 +945,15 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
   )
 endif()
 
+# Tests are excluded from Release builds
+CMAKE_DEPENDENT_OPTION(WITH_TESTS "build with tests" ON
+  "CMAKE_BUILD_TYPE STREQUAL Debug" OFF)
 if(WITH_TESTS)
   add_subdirectory(third-party/gtest-1.8.1/fused-src/gtest)
   add_library(testharness STATIC
   test_util/testharness.cc)
   target_link_libraries(testharness gtest)
+
   set(TESTS
         cache/cache_test.cc
         cache/lru_cache_test.cc
@@ -1078,7 +1053,6 @@ if(WITH_TESTS)
         options/options_settable_test.cc
         options/options_test.cc
         table/block_based/block_based_filter_block_test.cc
-        table/block_based/block_based_table_reader_test.cc
         table/block_based/block_test.cc
         table/block_based/data_block_hash_index_test.cc
         table/block_based/full_filter_block_test.cc
@@ -1090,7 +1064,6 @@ if(WITH_TESTS)
         table/sst_file_reader_test.cc
         table/table_test.cc
         table/block_fetcher_test.cc
-        test_util/testutil_test.cc
         tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc
         tools/ldb_cmd_test.cc
         tools/reduce_levels_test.cc
@@ -1185,10 +1158,6 @@ if(WITH_TESTS)
         add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
         add_dependencies(check ${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX})
       endif()
-      if("${exename}" MATCHES "env_librados_test")
-        # env_librados_test.cc uses librados directly
-        target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} rados)
-      endif()
   endforeach(sourcefile ${TESTS})
 
   if(WIN32)
@@ -1212,6 +1181,7 @@ if(WITH_TESTS)
   endif()
 endif()
 
+option(WITH_BENCHMARK_TOOLS "build with benchmarks" ON)
 if(WITH_BENCHMARK_TOOLS)
   add_executable(db_bench
     tools/db_bench.cc
@@ -1250,6 +1220,8 @@ if(WITH_BENCHMARK_TOOLS)
     ${ROCKSDB_LIB})
 endif()
 
+option(WITH_CORE_TOOLS "build with ldb and sst_dump" ON)
+option(WITH_TOOLS "build with tools" ON)
 if(WITH_CORE_TOOLS OR WITH_TOOLS)
   add_subdirectory(tools)
   add_custom_target(core_tools

HISTORY.md

@@ -1,73 +1,30 @@
 # Rocksdb Change Log
-## 6.11.7 (11/15/2020)
+## 6.10.4 (11/15/2020)
 ### Bug Fixes
-* Fix a bug of encoding and parsing BlockBasedTableOptions::read_amp_bytes_per_bit as a 64-bit integer.
+* Fixed a bug of encoding and parsing BlockBasedTableOptions::read_amp_bytes_per_bit as a 64-bit integer.
 * Fixed the logic of populating native data structure for `read_amp_bytes_per_bit` during OPTIONS file parsing on big-endian architecture. Without this fix, original code introduced in PR7659, when running on big-endian machine, can mistakenly store read_amp_bytes_per_bit (an uint32) in little endian format. Future access to `read_amp_bytes_per_bit` will give wrong values. Little endian architecture is not affected.
 
-## 6.11.6 (10/12/2020)
-### Bug Fixes
-* Fixed a bug in the following combination of features: indexes with user keys (`format_version >= 3`), indexes are partitioned (`index_type == kTwoLevelIndexSearch`), and some index partitions are pinned in memory (`BlockBasedTableOptions::pin_l0_filter_and_index_blocks_in_cache`). The bug could cause keys to be truncated when read from the index leading to wrong read results or other unexpected behavior.
-* Fixed a bug when indexes are partitioned (`index_type == kTwoLevelIndexSearch`), some index partitions are pinned in memory (`BlockBasedTableOptions::pin_l0_filter_and_index_blocks_in_cache`), and partitions reads could be mixed between block cache and directly from the file (e.g., with `enable_index_compression == 1` and `mmap_read == 1`, partitions that were stored uncompressed due to poor compression ratio would be read directly from the file via mmap, while partitions that were stored compressed would be read from block cache). The bug could cause index partitions to be mistakenly considered empty during reads leading to wrong read results.
-
-## 6.11.5 (7/23/2020)
-### Bug Fixes
-* Memtable lookup should report unrecognized value_type as corruption (#7121).
-
-## 6.11.4 (7/15/2020)
-### Bug Fixes
-* Make compaction report InternalKey corruption while iterating over the input.
-
-## 6.11.3 (7/9/2020)
-### Bug Fixes
-* Fix a bug when index_type == kTwoLevelIndexSearch in PartitionedIndexBuilder to update FlushPolicy to point to internal key partitioner when it changes from user-key mode to internal-key mode in index partition.
-* Disable file deletion after MANIFEST write/sync failure until db re-open or Resume() so that subsequent re-open will not see MANIFEST referencing deleted SSTs.
-
-## 6.11.1 (6/23/2020)
-### Bug Fixes
+## 6.10.3 (6/16/2020)
+### Bug fix
+* Fix potential file descriptor leakage in PosixEnv's IsDirectory() and NewRandomAccessFile().
 * Best-efforts recovery ignores CURRENT file completely. If CURRENT file is missing during recovery, best-efforts recovery still proceeds with MANIFEST file(s).
 * In best-efforts recovery, an error that is not Corruption or IOError::kNotFound or IOError::kPathNotFound will be overwritten silently. Fix this by checking all non-ok cases and return early.
-* Compressed block cache was automatically disabled with read-only DBs by mistake. Now it is fixed: compressed block cache will be in effective with read-only DB too.
 * Fail recovery and report once hitting a physical log record checksum mismatch, while reading MANIFEST. RocksDB should not continue processing the MANIFEST any further.
 * Fix a bug of wrong iterator result if another thread finishes an update and a DB flush between two statement.
 
-### Public API Change
-* `DB::OpenForReadOnly()` now returns `Status::NotFound` when the specified DB directory does not exist. Previously the error returned depended on the underlying `Env`.
-
-## 6.11 (6/12/2020)
-### Bug Fixes
-* Fix consistency checking error swallowing in some cases when options.force_consistency_checks = true.
-* Fix possible false NotFound status from batched MultiGet using index type kHashSearch.
-* Fix corruption caused by enabling delete triggered compaction (NewCompactOnDeletionCollectorFactory) in universal compaction mode, along with parallel compactions. The bug can result in two parallel compactions picking the same input files, resulting in the DB resurrecting older and deleted versions of some keys.
-* Fix a use-after-free bug in best-efforts recovery. column_family_memtables_ needs to point to valid ColumnFamilySet.
-* Let best-efforts recovery ignore corrupted files during table loading.
-* Fix corrupt key read from ingested file when iterator direction switches from reverse to forward at a key that is a prefix of another key in the same file. It is only possible in files with a non-zero global seqno.
-* Fix abnormally large estimate from GetApproximateSizes when a range starts near the end of one SST file and near the beginning of another. Now GetApproximateSizes consistently and fairly includes the size of SST metadata in addition to data blocks, attributing metadata proportionally among the data blocks based on their size.
-* Fix potential file descriptor leakage in PosixEnv's IsDirectory() and NewRandomAccessFile().
+## 6.10.2 (6/5/2020)
+### Bug fix
 * Fix false negative from the VerifyChecksum() API when there is a checksum mismatch in an index partition block in a BlockBasedTable format table file (index_type is kTwoLevelIndexSearch).
-* Fix sst_dump to return non-zero exit code if the specified file is not a recognized SST file or fails requested checks.
-* Fix incorrect results from batched MultiGet for duplicate keys, when the duplicate key matches the largest key of an SST file and the value type for the key in the file is a merge value.
 
-### Public API Change
-* Flush(..., column_family) may return Status::ColumnFamilyDropped() instead of Status::InvalidArgument() if column_family is dropped while processing the flush request.
-* BlobDB now explicitly disallows using the default column family's storage directories as blob directory.
-* DeleteRange now returns `Status::InvalidArgument` if the range's end key comes before its start key according to the user comparator. Previously the behavior was undefined.
-* ldb now uses options.force_consistency_checks = true by default and "--disable_consistency_checks" is added to disable it.
-* DB::OpenForReadOnly no longer creates files or directories if the named DB does not exist, unless create_if_missing is set to true.
-* The consistency checks that validate LSM state changes (table file additions/deletions during flushes and compactions) are now stricter, more efficient, and no longer optional, i.e. they are performed even if `force_consistency_checks` is `false`.
-* Disable delete triggered compaction (NewCompactOnDeletionCollectorFactory) in universal compaction mode and num_levels = 1 in order to avoid a corruption bug.
-* `pin_l0_filter_and_index_blocks_in_cache` no longer applies to L0 files larger than `1.5 * write_buffer_size` to give more predictable memory usage. Such L0 files may exist due to intra-L0 compaction, external file ingestion, or user dynamically changing `write_buffer_size` (note, however, that files that are already pinned will continue being pinned, even after such a dynamic change).
-* In point-in-time wal recovery mode, fail database recovery in case of IOError while reading the WAL to avoid data loss.
-
-### New Features
-* sst_dump to add a new --readahead_size argument. Users can specify read size when scanning the data. Sst_dump also tries to prefetch tail part of the SST files so usually some number of I/Os are saved there too.
-* Generate file checksum in SstFileWriter if Options.file_checksum_gen_factory is set. The checksum and checksum function name are stored in ExternalSstFileInfo after the sst file write is finished.
-* Add a value_size_soft_limit in read options which limits the cumulative value size of keys read in batches in MultiGet. Once the cumulative value size of found keys exceeds read_options.value_size_soft_limit, all the remaining keys are returned with status Abort without further finding their values. By default the value_size_soft_limit is std::numeric_limits<uint64_t>::max().
-* Enable SST file ingestion with file checksum information when calling IngestExternalFiles(const std::vector<IngestExternalFileArg>& args). Added files_checksums and files_checksum_func_names to IngestExternalFileArg such that user can ingest the sst files with their file checksum information. Added verify_file_checksum to IngestExternalFileOptions (default is True). To be backward compatible, if DB does not enable file checksum or user does not provide checksum information (vectors of files_checksums and files_checksum_func_names are both empty), verification of file checksum is always sucessful. If DB enables file checksum, DB will always generate the checksum for each ingested SST file during Prepare stage of ingestion and store the checksum in Manifest, unless verify_file_checksum is False and checksum information is provided by the application. In this case, we only verify the checksum function name and directly store the ingested checksum in Manifest. If verify_file_checksum is set to True, DB will verify the ingested checksum and function name with the genrated ones. Any mismatch will fail the ingestion. Note that, if IngestExternalFileOptions::write_global_seqno is True, the seqno will be changed in the ingested file. Therefore, the checksum of the file will be changed. In this case, a new checksum will be generated after the seqno is updated and be stored in the Manifest.
-
-### Performance Improvements
-* Eliminate redundant key comparisons during random access in block-based tables.
+## 6.10.1 (5/27/2020)
+### Bug fix
+* Remove "u'<string>'" in TARGETS file.
+* Fix db_stress_lib target in buck.
 
 ## 6.10 (5/2/2020)
+### Behavior Changes
+* Disable delete triggered compaction (NewCompactOnDeletionCollectorFactory) in universal compaction mode and num_levels = 1 in order to avoid a corruption bug.
+
 ### Bug Fixes
 * Fix wrong result being read from ingested file. May happen when a key in the file happen to be prefix of another key also in the file. The issue can further cause more data corruption. The issue exists with rocksdb >= 5.0.0 since DB::IngestExternalFile() was introduced.
 * Finish implementation of BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey. It's now ready for use. Significantly reduces read amplification in some setups, especially for iterator seeks.
@@ -77,11 +34,22 @@
 * Fix a bug caused by not including user timestamp in MultiGet LookupKey construction. This can lead to wrong query result since the trailing bytes of a user key, if not shorter than timestamp, will be mistaken for user timestamp.
 * Fix a bug caused by using wrong compare function when sorting the input keys of MultiGet with timestamps.
 * Upgraded version of bzip library (1.0.6 -> 1.0.8) used with RocksJava to address potential vulnerabilities if an attacker can manipulate compressed data saved and loaded by RocksDB (not normal). See issue #6703.
+* Fix consistency checking error swallowing in some cases when options.force_consistency_checks = true.
+* Fix possible false NotFound status from batched MultiGet using index type kHashSearch.
+* Fix corruption caused by enabling delete triggered compaction (NewCompactOnDeletionCollectorFactory) in universal compaction mode, along with parallel compactions. The bug can result in two parallel compactions picking the same input files, resulting in the DB resurrecting older and deleted versions of some keys.
+* Fix a use-after-free bug in best-efforts recovery. column_family_memtables_ needs to point to valid ColumnFamilySet.
+* Let best-efforts recovery ignore corrupted files during table loading.
+* Fix a bug when making options.bottommost_compression, options.compression_opts and options.bottommost_compression_opts dynamically changeable: the modified values are not written to option files or returned back to users when being queried.
+* Fix a bug where index key comparisons were unaccounted in `PerfContext::user_key_comparison_count` for lookups in files written with `format_version >= 3`.
+* Fix many bloom.filter statistics not being updated in batch MultiGet.
+
 
 ### Public API Change
 * Add a ConfigOptions argument to the APIs dealing with converting options to and from strings and files.  The ConfigOptions is meant to replace some of the options (such as input_strings_escaped and ignore_unknown_options) and allow for more parameters to be passed in the future without changing the function signature.
 * Add NewFileChecksumGenCrc32cFactory to the file checksum public API, such that the builtin Crc32c based file checksum generator factory can be used by applications.
 * Add IsDirectory to Env and FS to indicate if a path is a directory.
+* ldb now uses options.force_consistency_checks = true by default and "--disable_consistency_checks" is added to disable it.
+* Add ReadOptions::deadline to allow users to specify a deadline for MultiGet requests
 
 ### New Features
 * Added support for pipelined & parallel compression optimization for `BlockBasedTableBuilder`. This optimization makes block building, block compression and block appending a pipeline, and uses multiple threads to accelerate block compression. Users can set `CompressionOptions::parallel_threads` greater than 1 to enable compression parallelism. This feature is experimental for now.
@@ -90,11 +58,6 @@
 * Added functionality in sst_dump tool to check the compressed file size for different compression levels and print the time spent on compressing files with each compression type. Added arguments `--compression_level_from` and `--compression_level_to` to report size of all compression levels and one compression_type must be specified with it so that it will report compressed sizes of one compression type with different levels.
 * Added statistics for redundant insertions into block cache: rocksdb.block.cache.*add.redundant. (There is currently no coordination to ensure that only one thread loads a table block when many threads are trying to access that same table block.)
 
-### Bug Fixes
-* Fix a bug when making options.bottommost_compression, options.compression_opts and options.bottommost_compression_opts dynamically changeable: the modified values are not written to option files or returned back to users when being queried.
-* Fix a bug where index key comparisons were unaccounted in `PerfContext::user_key_comparison_count` for lookups in files written with `format_version >= 3`.
-* Fix many bloom.filter statistics not being updated in batch MultiGet.
-
 ### Performance Improvements
 * Improve performance of batch MultiGet with partitioned filters, by sharing block cache lookups to applicable filter blocks.
 * Reduced memory copies when fetching and uncompressing compressed blocks from sst files.

Makefile

@@ -8,11 +8,7 @@
 
 BASH_EXISTS := $(shell which bash)
 SHELL := $(shell which bash)
-# Default to python3. Some distros like CentOS 8 do not have `python`.
-ifeq ($(origin PYTHON), undefined)
-	PYTHON := $(shell which python3 || which python || echo python3)
-endif
-export PYTHON
+PYTHON?=$(shell which python)
 
 CLEAN_FILES = # deliberately empty, so we can append below.
 CFLAGS += ${EXTRA_CFLAGS}
@@ -170,12 +166,6 @@ else
 	CXXFLAGS += -fno-rtti
 endif
 
-ifdef ASSERT_STATUS_CHECKED
-ifeq ($(filter -DROCKSDB_ASSERT_STATUS_CHECKED,$(OPT)),)
-	OPT += -DROCKSDB_ASSERT_STATUS_CHECKED
-endif
-endif
-
 $(warning Warning: Compiling in debug mode. Don't use the resulting binary in production)
 endif
 
@@ -197,16 +187,12 @@ AM_V_CC = $(am__v_CC_$(V))
 am__v_CC_ = $(am__v_CC_$(AM_DEFAULT_VERBOSITY))
 am__v_CC_0 = @echo "  CC      " $@;
 am__v_CC_1 =
-
+CCLD = $(CC)
+LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@
 AM_V_CCLD = $(am__v_CCLD_$(V))
 am__v_CCLD_ = $(am__v_CCLD_$(AM_DEFAULT_VERBOSITY))
-ifneq ($(SKIP_LINK), 1)
 am__v_CCLD_0 = @echo "  CCLD    " $@;
 am__v_CCLD_1 =
-else
-am__v_CCLD_0 = @echo "  !CCLD   " $@; true skip
-am__v_CCLD_1 = true skip
-endif
 AM_V_AR = $(am__v_AR_$(V))
 am__v_AR_ = $(am__v_AR_$(AM_DEFAULT_VERBOSITY))
 am__v_AR_0 = @echo "  AR      " $@;
@@ -218,7 +204,6 @@ LDFLAGS += -lrados
 endif
 
 AM_LINK = $(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-
 # Detect what platform we're building on.
 # Export some common variables that might have been passed as Make variables
 # instead of environment variables.
@@ -336,7 +321,7 @@ endif
 
 export GTEST_THROW_ON_FAILURE=1
 export GTEST_HAS_EXCEPTIONS=1
-GTEST_DIR = third-party/gtest-1.8.1/fused-src
+GTEST_DIR = ./third-party/gtest-1.8.1/fused-src
 # AIX: pre-defined system headers are surrounded by an extern "C" block
 ifeq ($(PLATFORM), OS_AIX)
 	PLATFORM_CCFLAGS += -I$(GTEST_DIR)
@@ -548,7 +533,6 @@ TESTS = \
 	random_access_file_reader_test \
 	file_reader_writer_test \
 	block_based_filter_block_test \
-	block_based_table_reader_test \
 	full_filter_block_test \
 	partitioned_filter_block_test \
 	hash_table_test \
@@ -641,7 +625,6 @@ TESTS = \
 	blob_file_garbage_test \
 	timer_test \
 	db_with_timestamp_compaction_test \
-	testutil_test \
 
 PARALLEL_TEST = \
 	backupable_db_test \
@@ -675,58 +658,6 @@ endif
 ifdef COMPILE_WITH_UBSAN
         TESTS := $(shell echo $(TESTS) | sed 's/\boptions_settable_test\b//g')
 endif
-ifdef ASSERT_STATUS_CHECKED
-	# This is a new check for which we will add support incrementally. The
-	# whitelist can be removed once support is fully added.
-	TESTS_WHITELIST = \
-		arena_test \
-		autovector_test \
-		blob_file_addition_test \
-		blob_file_garbage_test \
-		bloom_test \
-		cassandra_format_test \
-		cassandra_row_merge_test \
-		cassandra_serialize_test \
-		cleanable_test \
-		coding_test \
-		crc32c_test \
-		dbformat_test \
-		defer_test \
-		dynamic_bloom_test \
-		event_logger_test \
-		file_indexer_test \
-		folly_synchronization_distributed_mutex_test \
-		hash_table_test \
-		hash_test \
-		heap_test \
-		histogram_test \
-		inlineskiplist_test \
-		io_posix_test \
-		iostats_context_test \
-		memkind_kmem_allocator_test \
-		merger_test \
-		mock_env_test \
-		object_registry_test \
-		options_settable_test \
-		options_test \
-		random_test \
-		range_del_aggregator_test \
-		range_tombstone_fragmenter_test \
-		repeatable_thread_test \
-		skiplist_test \
-		slice_test \
-		statistics_test \
-		thread_local_test \
-		timer_queue_test \
-		timer_test \
-		util_merge_operators_test \
-		version_edit_test \
-		work_queue_test \
-		write_controller_test \
-
-	TESTS := $(filter $(TESTS_WHITELIST),$(TESTS))
-	PARALLEL_TEST := $(filter $(TESTS_WHITELIST),$(PARALLEL_TEST))
-endif
 SUBSET := $(TESTS)
 ifdef ROCKSDBTESTS_START
         SUBSET := $(shell echo $(SUBSET) | sed 's/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/')
@@ -978,12 +909,6 @@ J ?= 100%
 # Use this regexp to select the subset of tests whose names match.
 tests-regexp = .
 
-ifeq ($(PRINT_PARALLEL_OUTPUTS), 1)	
-	parallel_com = '{}'
-else
-	parallel_com = '{} >& t/log-{/}'
-endif
-
 .PHONY: check_0
 check_0:
 	$(AM_V_GEN)export TEST_TMPDIR=$(TMPD); \
@@ -997,10 +922,7 @@ check_0:
 	} \
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
-	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu  $(parallel_com) ; \
-	parallel_retcode=$$? ; \
-	awk '{ if ($$7 != 0 || $$8 != 0) { if ($$7 == "Exitval") { h = $$0; } else { if (!f) print h; print; f = 1 } } } END { if(f) exit 1; }' < LOG ; \
-	if [ $$parallel_retcode -ne 0 ] ; then exit 1 ; fi
+	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu '{} >& t/log-{/}'
 
 valgrind-blacklist-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized|MySQLStyleTransactionTest.TransactionStressTest
 
@@ -1034,9 +956,6 @@ CLEAN_FILES += t LOG $(TMPD)
 watch-log:
 	$(WATCH) --interval=0 'sort -k7,7nr -k4,4gr LOG|$(quoted_perl_command)'
 
-dump-log:
-	bash -c '$(quoted_perl_command)' < LOG
-
 # If J != 1 and GNU parallel is installed, run the tests in parallel,
 # via the check_0 rule above.  Otherwise, run them sequentially.
 check: all
@@ -1059,11 +978,9 @@ ifndef ASSERT_STATUS_CHECKED # not yet working with these tests
 	sh tools/rocksdb_dump_test.sh
 endif
 endif
-endif	
-ifndef SKIP_FORMAT_BUCK_CHECKS
+endif
 	$(MAKE) check-format
 	$(MAKE) check-buck-targets
-endif
 
 # TODO add ldb_tests
 check_some: $(SUBSET)
@@ -1352,9 +1269,9 @@ db_repl_stress: tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL)
 
 arena_test: memory/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
-
+	
 memkind_kmem_allocator_test: memory/memkind_kmem_allocator_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
+	$(AM_LINK)	
 
 autovector_test: util/autovector_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
@@ -1640,9 +1557,6 @@ file_reader_writer_test: util/file_reader_writer_test.o $(LIBOBJECTS) $(TESTHARN
 block_based_filter_block_test: table/block_based/block_based_filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-block_based_table_reader_test: table/block_based/block_based_table_reader_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 full_filter_block_test: table/block_based/full_filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1880,9 +1794,6 @@ blob_file_garbage_test: db/blob/blob_file_garbage_test.o $(LIBOBJECTS) $(TESTHAR
 timer_test: util/timer_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-testutil_test: test_util/testutil_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 #-------------------------------------------------
 # make install related stuff
 INSTALL_PATH ?= /usr/local
@@ -2291,7 +2202,6 @@ endif
 #  	Source files dependencies detection
 # ---------------------------------------------------------------------------
 
-# FIXME: nothing checks that entries in MAIN_SOURCES actually exist
 all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(ANALYZER_LIB_SOURCES) $(STRESS_LIB_SOURCES)
 DEPFILES = $(all_sources:.cc=.cc.d)
 

TARGETS

@@ -1,4 +1,4 @@
-# This file @generated by `python3 buckifier/buckify_rocksdb.py`
+# This file @generated by `python buckifier/buckify_rocksdb.py`
 # --> DO NOT EDIT MANUALLY <--
 # This file is a Facebook-specific integration for buck builds, so can
 # only be validated by Facebook employees.
@@ -26,6 +26,7 @@ ROCKSDB_EXTERNAL_DEPS = [
     ("lz4", None, "lz4"),
     ("zstd", None),
     ("tbb", None),
+    ("googletest", None, "gtest"),
 ]
 
 ROCKSDB_OS_DEPS = [
@@ -124,9 +125,6 @@ cpp_library(
         "db/blob/blob_file_addition.cc",
         "db/blob/blob_file_garbage.cc",
         "db/blob/blob_file_meta.cc",
-        "db/blob/blob_log_format.cc",
-        "db/blob/blob_log_reader.cc",
-        "db/blob/blob_log_writer.cc",
         "db/builder.cc",
         "db/c.cc",
         "db/column_family.cc",
@@ -321,6 +319,9 @@ cpp_library(
         "utilities/blob_db/blob_db_impl_filesnapshot.cc",
         "utilities/blob_db/blob_dump_tool.cc",
         "utilities/blob_db/blob_file.cc",
+        "utilities/blob_db/blob_log_format.cc",
+        "utilities/blob_db/blob_log_reader.cc",
+        "utilities/blob_db/blob_log_writer.cc",
         "utilities/cassandra/cassandra_compaction_filter.cc",
         "utilities/cassandra/format.cc",
         "utilities/cassandra/merge_operator.cc",
@@ -398,9 +399,7 @@ cpp_library(
     os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
     preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
     deps = [":rocksdb_lib"],
-    external_deps = ROCKSDB_EXTERNAL_DEPS + [
-        ("googletest", None, "gtest"),
-    ],
+    external_deps = ROCKSDB_EXTERNAL_DEPS,
 )
 
 cpp_library(
@@ -447,27 +446,6 @@ cpp_library(
     external_deps = ROCKSDB_EXTERNAL_DEPS,
 )
 
-if not is_opt_mode:
-    cpp_binary(
-        name = "c_test_bin",
-        srcs = ["db/c_test.c"],
-        arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-        os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-        compiler_flags = ROCKSDB_COMPILER_FLAGS,
-        preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-        deps = [":rocksdb_test_lib"],
-    )
-
-if not is_opt_mode:
-    custom_unittest(
-        "c_test",
-        command = [
-            native.package_name() + "/buckifier/rocks_test_runner.sh",
-            "$(location :{})".format("c_test_bin"),
-        ],
-        type = "simple",
-    )
-
 cpp_library(
     name = "env_basic_test_lib",
     srcs = ["env/env_basic_test.cc"],
@@ -539,13 +517,6 @@ ROCKS_TESTS = [
         [],
         [],
     ],
-    [
-        "block_based_table_reader_test",
-        "table/block_based/block_based_table_reader_test.cc",
-        "serial",
-        [],
-        [],
-    ],
     [
         "block_cache_trace_analyzer_test",
         "tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc",
@@ -581,6 +552,13 @@ ROCKS_TESTS = [
         [],
         [],
     ],
+    [
+        "c_test",
+        "db/c_test.c",
+        "serial",
+        [],
+        [],
+    ],
     [
         "cache_simulator_test",
         "utilities/simulator_cache/cache_simulator_test.cc",
@@ -1484,13 +1462,6 @@ ROCKS_TESTS = [
         [],
         [],
     ],
-    [
-        "testutil_test",
-        "test_util/testutil_test.cc",
-        "serial",
-        [],
-        [],
-    ],
     [
         "thread_list_test",
         "util/thread_list_test.cc",
@@ -1644,17 +1615,18 @@ ROCKS_TESTS = [
 # Do not build the tests in opt mode, since SyncPoint and other test code
 # will not be included.
 [
-    cpp_unittest(
-        name = test_name,
-        srcs = [test_cc],
-        arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-        os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-        compiler_flags = ROCKSDB_COMPILER_FLAGS,
-        preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-        deps = [":rocksdb_test_lib"] + extra_deps,
-        external_deps = ROCKSDB_EXTERNAL_DEPS + [
-            ("googletest", None, "gtest"),
-        ],
+    test_binary(
+        extra_compiler_flags = extra_compiler_flags,
+        extra_deps = extra_deps,
+        parallelism = parallelism,
+        rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
+        rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
+        rocksdb_external_deps = ROCKSDB_EXTERNAL_DEPS,
+        rocksdb_os_deps = ROCKSDB_OS_DEPS,
+        rocksdb_os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
+        rocksdb_preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
+        test_cc = test_cc,
+        test_name = test_name,
     )
     for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
     if not is_opt_mode

USERS.md

@@ -106,6 +106,3 @@ LzLabs is using RocksDB as a storage engine in their multi-database distributed
 ## Nebula Graph
 
 [Nebula Graph](https://github.com/vesoft-inc/nebula) is a distributed, scalable, lightning-fast, open source graph database capable of hosting super large scale graphs with dozens of billions of vertices (nodes) and trillions of edges, with milliseconds of latency.
-
-## YugabyteDB
-[YugabyteDB](https://www.yugabyte.com/) is an open source, high performance, distributed SQL database that uses RocksDB as its storage layer. For more information, please see https://github.com/yugabyte/yugabyte-db/.

buckifier/buckify_rocksdb.py

@@ -20,10 +20,10 @@ from util import ColorString
 # User can pass extra dependencies as a JSON object via command line, and this
 # script can include these dependencies in the generate TARGETS file.
 # Usage:
-# $python3 buckifier/buckify_rocksdb.py
+# $python buckifier/buckify_rocksdb.py
 # (This generates a TARGET file without user-specified dependency for unit
 # tests.)
-# $python3 buckifier/buckify_rocksdb.py \
+# $python buckifier/buckify_rocksdb.py \
 #        '{"fake": { \
 #                      "extra_deps": [":test_dep", "//fakes/module:mock1"],  \
 #                      "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"], \
@@ -64,6 +64,8 @@ def get_cc_files(repo_path):
             continue
         for filename in fnmatch.filter(filenames, '*.cc'):
             cc_files.append(os.path.join(root, filename))
+        for filename in fnmatch.filter(filenames, '*.c'):
+            cc_files.append(os.path.join(root, filename))
     return cc_files
 
 
@@ -106,6 +108,7 @@ def get_tests(repo_path):
 
     return tests
 
+
 # Parse extra dependencies passed by user from command line
 def get_dependencies():
     deps_map = {
@@ -144,7 +147,6 @@ def generate_targets(repo_path, deps_map):
         return False
 
     TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
-
     # rocksdb_lib
     TARGETS.add_library(
         "rocksdb_lib",
@@ -157,10 +159,7 @@ def generate_targets(repo_path, deps_map):
         src_mk.get("TEST_LIB_SOURCES", []) +
         src_mk.get("EXP_LIB_SOURCES", []) +
         src_mk.get("ANALYZER_LIB_SOURCES", []),
-        [":rocksdb_lib"],
-        extra_external_deps=""" + [
-        ("googletest", None, "gtest"),
-    ]""")
+        [":rocksdb_lib"])
     # rocksdb_tools_lib
     TARGETS.add_library(
         "rocksdb_tools_lib",
@@ -176,18 +175,10 @@ def generate_targets(repo_path, deps_map):
         + ["test_util/testutil.cc"])
 
     print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
-
-    # c_test.c is added through TARGETS.add_c_test(). If there
-    # are more than one .c test file, we need to extend
-    # TARGETS.add_c_test() to include other C tests too.
-    TARGETS.add_c_test()
-
-    # test for every .cc test we found in the Makefile
+    # test for every test we found in the Makefile
     for target_alias, deps in deps_map.items():
         for test in sorted(tests):
-            if test == 'c_test':
-                continue
-            match_src = [src for src in cc_files if ("/%s.cc" % test) in src]
+            match_src = [src for src in cc_files if ("/%s.c" % test) in src]
             if len(match_src) == 0:
                 print(ColorString.warning("Cannot find .cc file for %s" % test))
                 continue
@@ -228,7 +219,6 @@ def get_rocksdb_path():
 
     return rocksdb_path
 
-
 def exit_with_error(msg):
     print(ColorString.error(msg))
     sys.exit(1)

buckifier/check_buck_targets.sh

@@ -15,7 +15,7 @@ echo Backup original TARGETS file.
 
 cp TARGETS TARGETS.bkp
 
-${PYTHON:-python3} buckifier/buckify_rocksdb.py
+python buckifier/buckify_rocksdb.py
 
 TGT_DIFF=`git diff TARGETS | head -n 1`
 
@@ -24,9 +24,8 @@ then
   mv TARGETS.bkp TARGETS
   exit 0
 else
-  echo "Please run '${PYTHON:-python3} buckifier/buckify_rocksdb.py' to update TARGETS file."
+  echo "Please run 'python buckifier/buckify_rocksdb.py' to update TARGETS file."
   echo "Do not manually update TARGETS file."
-  ${PYTHON:-python3} --version
   mv TARGETS.bkp TARGETS
   exit 1
 fi

buckifier/targets_builder.py

@@ -28,8 +28,7 @@ class TARGETSBuilder(object):
     def __init__(self, path):
         self.path = path
         self.targets_file = open(path, 'w')
-        header = targets_cfg.rocksdb_target_header_template
-        self.targets_file.write(header)
+        self.targets_file.write(targets_cfg.rocksdb_target_header)
         self.total_lib = 0
         self.total_bin = 0
         self.total_test = 0
@@ -38,21 +37,17 @@ class TARGETSBuilder(object):
     def __del__(self):
         self.targets_file.close()
 
-    def add_library(self, name, srcs, deps=None, headers=None,
-                    extra_external_deps=""):
+    def add_library(self, name, srcs, deps=None, headers=None):
         headers_attr_prefix = ""
         if headers is None:
             headers_attr_prefix = "auto_"
             headers = "AutoHeaders.RECURSIVE_GLOB"
-        else:
-            headers = "[" + pretty_list(headers) + "]"
         self.targets_file.write(targets_cfg.library_template.format(
             name=name,
             srcs=pretty_list(srcs),
             headers_attr_prefix=headers_attr_prefix,
             headers=headers,
-            deps=pretty_list(deps),
-            extra_external_deps=extra_external_deps))
+            deps=pretty_list(deps)))
         self.total_lib = self.total_lib + 1
 
     def add_rocksdb_library(self, name, srcs, headers=None):
@@ -60,8 +55,6 @@ class TARGETSBuilder(object):
         if headers is None:
             headers_attr_prefix = "auto_"
             headers = "AutoHeaders.RECURSIVE_GLOB"
-        else:
-            headers = "[" + pretty_list(headers) + "]"
         self.targets_file.write(targets_cfg.rocksdb_library_template.format(
             name=name,
             srcs=pretty_list(srcs),
@@ -76,30 +69,6 @@ class TARGETSBuilder(object):
             pretty_list(deps)))
         self.total_bin = self.total_bin + 1
 
-    def add_c_test(self):
-        self.targets_file.write("""
-if not is_opt_mode:
-    cpp_binary(
-        name = "c_test_bin",
-        srcs = ["db/c_test.c"],
-        arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-        os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-        compiler_flags = ROCKSDB_COMPILER_FLAGS,
-        preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-        deps = [":rocksdb_test_lib"],
-    )
-
-if not is_opt_mode:
-    custom_unittest(
-        "c_test",
-        command = [
-            native.package_name() + "/buckifier/rocks_test_runner.sh",
-            "$(location :{})".format("c_test_bin"),
-        ],
-        type = "simple",
-    )
-""")
-
     def register_test(self,
                       test_name,
                       src,

buckifier/targets_cfg.py

@@ -4,8 +4,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-rocksdb_target_header_template = \
-    """# This file \100generated by `python3 buckifier/buckify_rocksdb.py`
+rocksdb_target_header = """# This file \100generated by `python buckifier/buckify_rocksdb.py`
 # --> DO NOT EDIT MANUALLY <--
 # This file is a Facebook-specific integration for buck builds, so can
 # only be validated by Facebook employees.
@@ -33,6 +32,7 @@ ROCKSDB_EXTERNAL_DEPS = [
     ("lz4", None, "lz4"),
     ("zstd", None),
     ("tbb", None),
+    ("googletest", None, "gtest"),
 ]
 
 ROCKSDB_OS_DEPS = [
@@ -133,7 +133,7 @@ cpp_library(
     os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
     preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
     deps = [{deps}],
-    external_deps = ROCKSDB_EXTERNAL_DEPS{extra_external_deps},
+    external_deps = ROCKSDB_EXTERNAL_DEPS,
 )
 """
 
@@ -182,17 +182,18 @@ ROCKS_TESTS = [
 # Do not build the tests in opt mode, since SyncPoint and other test code
 # will not be included.
 [
-    cpp_unittest(
-        name = test_name,
-        srcs = [test_cc],
-        arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-        os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-        compiler_flags = ROCKSDB_COMPILER_FLAGS,
-        preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-        deps = [":rocksdb_test_lib"] + extra_deps,
-        external_deps = ROCKSDB_EXTERNAL_DEPS + [
-            ("googletest", None, "gtest"),
-        ],
+    test_binary(
+        extra_compiler_flags = extra_compiler_flags,
+        extra_deps = extra_deps,
+        parallelism = parallelism,
+        rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
+        rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
+        rocksdb_external_deps = ROCKSDB_EXTERNAL_DEPS,
+        rocksdb_os_deps = ROCKSDB_OS_DEPS,
+        rocksdb_os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
+        rocksdb_preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
+        test_cc = test_cc,
+        test_name = test_name,
     )
     for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
     if not is_opt_mode

build_tools/format-diff.sh

@@ -26,77 +26,53 @@ while getopts ':ch' OPTION; do
   esac
 done
 
-REPO_ROOT="$(git rev-parse --show-toplevel)"
+if [ -z $CLANG_FORMAT_DIFF ]
+then
+CLANG_FORMAT_DIFF="clang-format-diff.py"
+fi
 
-if [ "$CLANG_FORMAT_DIFF" ]; then
-  echo "Note: CLANG_FORMAT_DIFF='$CLANG_FORMAT_DIFF'"
-  # Dry run to confirm dependencies like argparse
-  if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then
-    true #Good
-  else
+# Check clang-format-diff.py
+if ! which $CLANG_FORMAT_DIFF &> /dev/null
+then
+  if [ ! -f ./clang-format-diff.py ]
+  then
+    echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
+    echo "You can download clang-format-diff.py by running: "
+    echo "    curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}"
+    echo "You can download clang-format by running:"
+    echo "    brew install clang-format"
+    echo "  Or"
+    echo "    apt install clang-format"
+    echo "  This might work too:"
+    echo "    yum install git-clang-format"
+    echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}"
+    echo "and make sure ${CLANG_FORMAT_DIFF} is executable."
     exit 128
-  fi
-else
-  # First try directly executing the possibilities
-  if clang-format-diff.py --help &> /dev/null < /dev/null; then
-    CLANG_FORMAT_DIFF=clang-format-diff.py
-  elif $REPO_ROOT/clang-format-diff.py --help &> /dev/null < /dev/null; then
-    CLANG_FORMAT_DIFF=$REPO_ROOT/clang-format-diff.py
   else
-    # This probably means we need to directly invoke the interpreter.
-    # But first find clang-format-diff.py
-    if [ -f "$REPO_ROOT/clang-format-diff.py" ]; then
-      CFD_PATH="$REPO_ROOT/clang-format-diff.py"
-    elif which clang-format-diff.py &> /dev/null; then
-      CFD_PATH="$(which clang-format-diff.py)"
+    if [ -x ./clang-format-diff.py ]
+    then
+      PATH=$PATH:.
     else
-      echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
-      echo "You can download clang-format-diff.py by running: "
-      echo "    curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}"
-      echo "You can download clang-format by running:"
-      echo "    brew install clang-format"
-      echo "  Or"
-      echo "    apt install clang-format"
-      echo "  This might work too:"
-      echo "    yum install git-clang-format"
-      echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}"
-      echo "and make sure ${CLANG_FORMAT_DIFF} is executable."
-      exit 128
-    fi
-    # Check argparse pre-req on interpreter, or it will fail
-    if echo import argparse | ${PYTHON:-python3}; then
-      true # Good
-    else
-      echo "To run clang-format-diff.py, we'll need the library "argparse" to be"
-      echo "installed. You can try either of the follow ways to install it:"
-      echo "  1. Manually download argparse: https://pypi.python.org/pypi/argparse"
-      echo "  2. easy_install argparse (if you have easy_install)"
-      echo "  3. pip install argparse (if you have pip)"
-      exit 129
-    fi
-    # Unfortunately, some machines have a Python2 clang-format-diff.py
-    # installed but only a Python3 interpreter installed. Rather than trying
-    # different Python versions that might be installed, we can try migrating
-    # the code to Python3 if it looks like Python2
-    if grep -q "print '" "$CFD_PATH" && \
-       ${PYTHON:-python3} --version | grep -q 'ython 3'; then
-      if [ ! -f "$REPO_ROOT/.py3/clang-format-diff.py" ]; then
-        echo "Migrating $CFD_PATH to Python3 in a hidden file"
-        mkdir -p "$REPO_ROOT/.py3"
-        ${PYTHON:-python3} -m lib2to3 -w -n -o "$REPO_ROOT/.py3" "$CFD_PATH" > /dev/null || exit 128
-      fi
-      CFD_PATH="$REPO_ROOT/.py3/clang-format-diff.py"
-    fi
-    CLANG_FORMAT_DIFF="${PYTHON:-python3} $CFD_PATH"
-    # This had better work after all those checks
-    if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then
-      true #Good
-    else
-      exit 128
+      CLANG_FORMAT_DIFF="python ./clang-format-diff.py"
     fi
   fi
 fi
 
+# Check argparse, a library that clang-format-diff.py requires.
+python 2>/dev/null << EOF
+import argparse
+EOF
+
+if [ "$?" != 0 ]
+then
+  echo "To run clang-format-diff.py, we'll need the library "argparse" to be"
+  echo "installed. You can try either of the follow ways to install it:"
+  echo "  1. Manually download argparse: https://pypi.python.org/pypi/argparse"
+  echo "  2. easy_install argparse (if you have easy_install)"
+  echo "  3. pip install argparse (if you have pip)"
+  exit 129
+fi
+
 # TODO(kailiu) following work is not complete since we still need to figure
 # out how to add the modified files done pre-commit hook to git's commit index.
 #
@@ -148,10 +124,6 @@ then
 elif [ $CHECK_ONLY ]
 then
   echo "Your change has unformatted code. Please run make format!"
-  if [ $VERBOSE_CHECK ]; then
-    clang-format --version
-    echo "$diffs"
-  fi
   exit 1
 fi
 

cache/cache.cc

@@ -14,30 +14,7 @@
 #include "util/string_util.h"
 
 namespace ROCKSDB_NAMESPACE {
-#ifndef ROCKSDB_LITE
-static std::unordered_map<std::string, OptionTypeInfo>
-    lru_cache_options_type_info = {
-        {"capacity",
-         {offsetof(struct LRUCacheOptions, capacity), OptionType::kSizeT,
-          OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
-          offsetof(struct LRUCacheOptions, capacity)}},
-        {"num_shard_bits",
-         {offsetof(struct LRUCacheOptions, num_shard_bits), OptionType::kInt,
-          OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
-          offsetof(struct LRUCacheOptions, num_shard_bits)}},
-        {"strict_capacity_limit",
-         {offsetof(struct LRUCacheOptions, strict_capacity_limit),
-          OptionType::kBoolean, OptionVerificationType::kNormal,
-          OptionTypeFlags::kMutable,
-          offsetof(struct LRUCacheOptions, strict_capacity_limit)}},
-        {"high_pri_pool_ratio",
-         {offsetof(struct LRUCacheOptions, high_pri_pool_ratio),
-          OptionType::kDouble, OptionVerificationType::kNormal,
-          OptionTypeFlags::kMutable,
-          offsetof(struct LRUCacheOptions, high_pri_pool_ratio)}}};
-#endif  // ROCKSDB_LITE
-
-Status Cache::CreateFromString(const ConfigOptions& config_options,
+Status Cache::CreateFromString(const ConfigOptions& /*opts*/,
                                const std::string& value,
                                std::shared_ptr<Cache>* result) {
   Status status;
@@ -47,14 +24,12 @@ Status Cache::CreateFromString(const ConfigOptions& config_options,
   } else {
 #ifndef ROCKSDB_LITE
     LRUCacheOptions cache_opts;
-    status = OptionTypeInfo::ParseStruct(
-        config_options, "", &lru_cache_options_type_info, "", value,
-        reinterpret_cast<char*>(&cache_opts));
-    if (status.ok()) {
-      cache = NewLRUCache(cache_opts);
+    if (!ParseOptionHelper(reinterpret_cast<char*>(&cache_opts),
+                           OptionType::kLRUCacheOptions, value)) {
+      status = Status::InvalidArgument("Invalid cache options");
     }
+    cache = NewLRUCache(cache_opts);
 #else
-    (void)config_options;
     status = Status::NotSupported("Cannot load cache in LITE mode ", value);
 #endif  //! ROCKSDB_LITE
   }

cache/clock_cache.cc

@@ -182,7 +182,7 @@ struct CacheHandle {
   void (*deleter)(const Slice&, void* value);
 
   // Flags and counters associated with the cache handle:
-  //   lowest bit: in-cache bit
+  //   lowest bit: n-cache bit
   //   second lowest bit: usage bit
   //   the rest bits: reference count
   // The handle is unused when flags equals to 0. The thread decreases the count

cmake/RocksDBConfig.cmake.in

@@ -1,54 +1,3 @@
 @PACKAGE_INIT@
-
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules")
-
-include(CMakeFindDependencyMacro)
-
-set(GFLAGS_USE_TARGET_NAMESPACE @GFLAGS_USE_TARGET_NAMESPACE@)
-
-if(@WITH_JEMALLOC@)
-  find_dependency(JeMalloc)
-endif()
-
-if(@WITH_GFLAGS@)
-  find_dependency(gflags CONFIG)
-  if(NOT gflags_FOUND)
-    find_dependency(gflags)
-  endif()
-endif()
-
-if(@WITH_SNAPPY@)
-  find_dependency(Snappy CONFIG)
-  if(NOT Snappy_FOUND)
-    find_dependency(Snappy)
-  endif()
-endif()
-
-if(@WITH_ZLIB@)
-  find_dependency(ZLIB)
-endif()
-
-if(@WITH_BZ2@)
-  find_dependency(BZip2)
-endif()
-
-if(@WITH_LZ4@)
-  find_dependency(lz4)
-endif()
-
-if(@WITH_ZSTD@)
-  find_dependency(zstd)
-endif()
-
-if(@WITH_NUMA@)
-  find_dependency(NUMA)
-endif()
-
-if(@WITH_TBB@)
-  find_dependency(TBB)
-endif()
-
-find_dependency(Threads)
-
 include("${CMAKE_CURRENT_LIST_DIR}/RocksDBTargets.cmake")
 check_required_components(RocksDB)

cmake/modules/FindSnappy.cmake

@@ -1,29 +0,0 @@
-# - Find Snappy
-# Find the snappy compression library and includes
-#
-# Snappy_INCLUDE_DIRS - where to find snappy.h, etc.
-# Snappy_LIBRARIES - List of libraries when using snappy.
-# Snappy_FOUND - True if snappy found.
-
-find_path(Snappy_INCLUDE_DIRS
-  NAMES snappy.h
-  HINTS ${snappy_ROOT_DIR}/include)
-
-find_library(Snappy_LIBRARIES
-  NAMES snappy
-  HINTS ${snappy_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(Snappy DEFAULT_MSG Snappy_LIBRARIES Snappy_INCLUDE_DIRS)
-
-mark_as_advanced(
-  Snappy_LIBRARIES
-  Snappy_INCLUDE_DIRS)
-
-if(Snappy_FOUND AND NOT (TARGET Snappy::snappy))
-  add_library (Snappy::snappy UNKNOWN IMPORTED)
-  set_target_properties(Snappy::snappy
-    PROPERTIES
-      IMPORTED_LOCATION ${Snappy_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${Snappy_INCLUDE_DIRS})
-endif()

cmake/modules/Findgflags.cmake

@@ -1,8 +1,8 @@
 # - Find gflags library
 # Find the gflags includes and library
 #
-# GFLAGS_INCLUDE_DIR - where to find gflags.h.
-# GFLAGS_LIBRARIES - List of libraries when using gflags.
+# gflags_INCLUDE_DIR - where to find gflags.h.
+# gflags_LIBRARIES - List of libraries when using gflags.
 # gflags_FOUND - True if gflags found.
 
 find_path(GFLAGS_INCLUDE_DIR

cmake/modules/Findsnappy.cmake

@@ -0,0 +1,29 @@
+# - Find Snappy
+# Find the snappy compression library and includes
+#
+# snappy_INCLUDE_DIRS - where to find snappy.h, etc.
+# snappy_LIBRARIES - List of libraries when using snappy.
+# snappy_FOUND - True if snappy found.
+
+find_path(snappy_INCLUDE_DIRS
+  NAMES snappy.h
+  HINTS ${snappy_ROOT_DIR}/include)
+
+find_library(snappy_LIBRARIES
+  NAMES snappy
+  HINTS ${snappy_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(snappy DEFAULT_MSG snappy_LIBRARIES snappy_INCLUDE_DIRS)
+
+mark_as_advanced(
+  snappy_LIBRARIES
+  snappy_INCLUDE_DIRS)
+
+if(snappy_FOUND AND NOT (TARGET snappy::snappy))
+  add_library (snappy::snappy UNKNOWN IMPORTED)
+  set_target_properties(snappy::snappy
+    PROPERTIES
+      IMPORTED_LOCATION ${snappy_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${snappy_INCLUDE_DIRS})
+endif()

coverage/coverage_test.sh

@@ -12,7 +12,7 @@ fi
 ROOT=".."
 # Fetch right version of gcov
 if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
-  source $ROOT/build_tools/fbcode_config_platform007.sh
+  source $ROOT/build_tools/fbcode_config.sh
   GCOV=$GCC_BASE/bin/gcov
 else
   GCOV=$(which gcov)

db/blob/blob_file_meta.cc

@@ -38,15 +38,8 @@ std::string BlobFileMetaData::DebugString() const {
 std::ostream& operator<<(std::ostream& os, const BlobFileMetaData& meta) {
   const auto& shared_meta = meta.GetSharedMeta();
   assert(shared_meta);
-  os << (*shared_meta);
 
-  os << " linked_ssts: {";
-  for (uint64_t file_number : meta.GetLinkedSsts()) {
-    os << ' ' << file_number;
-  }
-  os << " }";
-
-  os << " garbage_blob_count: " << meta.GetGarbageBlobCount()
+  os << (*shared_meta) << " garbage_blob_count: " << meta.GetGarbageBlobCount()
      << " garbage_blob_bytes: " << meta.GetGarbageBlobBytes();
 
   return os;

db/blob/blob_file_meta.h

@@ -5,13 +5,12 @@
 
 #pragma once
 
+#include "rocksdb/rocksdb_namespace.h"
+
 #include <cassert>
 #include <iosfwd>
 #include <memory>
 #include <string>
-#include <unordered_set>
-
-#include "rocksdb/rocksdb_namespace.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -91,15 +90,11 @@ std::ostream& operator<<(std::ostream& os,
 
 class BlobFileMetaData {
  public:
-  using LinkedSsts = std::unordered_set<uint64_t>;
-
   static std::shared_ptr<BlobFileMetaData> Create(
       std::shared_ptr<SharedBlobFileMetaData> shared_meta,
-      LinkedSsts linked_ssts, uint64_t garbage_blob_count,
-      uint64_t garbage_blob_bytes) {
-    return std::shared_ptr<BlobFileMetaData>(
-        new BlobFileMetaData(std::move(shared_meta), std::move(linked_ssts),
-                             garbage_blob_count, garbage_blob_bytes));
+      uint64_t garbage_blob_count, uint64_t garbage_blob_bytes) {
+    return std::shared_ptr<BlobFileMetaData>(new BlobFileMetaData(
+        std::move(shared_meta), garbage_blob_count, garbage_blob_bytes));
   }
 
   BlobFileMetaData(const BlobFileMetaData&) = delete;
@@ -133,8 +128,6 @@ class BlobFileMetaData {
     return shared_meta_->GetChecksumValue();
   }
 
-  const LinkedSsts& GetLinkedSsts() const { return linked_ssts_; }
-
   uint64_t GetGarbageBlobCount() const { return garbage_blob_count_; }
   uint64_t GetGarbageBlobBytes() const { return garbage_blob_bytes_; }
 
@@ -142,10 +135,8 @@ class BlobFileMetaData {
 
  private:
   BlobFileMetaData(std::shared_ptr<SharedBlobFileMetaData> shared_meta,
-                   LinkedSsts linked_ssts, uint64_t garbage_blob_count,
-                   uint64_t garbage_blob_bytes)
+                   uint64_t garbage_blob_count, uint64_t garbage_blob_bytes)
       : shared_meta_(std::move(shared_meta)),
-        linked_ssts_(std::move(linked_ssts)),
         garbage_blob_count_(garbage_blob_count),
         garbage_blob_bytes_(garbage_blob_bytes) {
     assert(shared_meta_);
@@ -154,7 +145,6 @@ class BlobFileMetaData {
   }
 
   std::shared_ptr<SharedBlobFileMetaData> shared_meta_;
-  LinkedSsts linked_ssts_;
   uint64_t garbage_blob_count_;
   uint64_t garbage_blob_bytes_;
 };

db/builder.cc

@@ -239,9 +239,7 @@ Status BuildTable(
           (internal_stats == nullptr) ? nullptr
                                       : internal_stats->GetFileReadHist(0),
           TableReaderCaller::kFlush, /*arena=*/nullptr,
-          /*skip_filter=*/false, level,
-          MaxFileSizeForL0MetaPin(mutable_cf_options),
-          /*smallest_compaction_key=*/nullptr,
+          /*skip_filter=*/false, level, /*smallest_compaction_key=*/nullptr,
           /*largest_compaction_key*/ nullptr,
           /*allow_unprepared_value*/ false));
       s = it->status();

db/c.cc

@@ -1466,11 +1466,6 @@ void rocksdb_writebatch_delete(
   b->rep.Delete(Slice(key, klen));
 }
 
-void rocksdb_writebatch_singledelete(rocksdb_writebatch_t* b, const char* key,
-                                     size_t klen) {
-  b->rep.SingleDelete(Slice(key, klen));
-}
-
 void rocksdb_writebatch_delete_cf(
     rocksdb_writebatch_t* b,
     rocksdb_column_family_handle_t* column_family,
@@ -1478,12 +1473,6 @@ void rocksdb_writebatch_delete_cf(
   b->rep.Delete(column_family->rep, Slice(key, klen));
 }
 
-void rocksdb_writebatch_singledelete_cf(
-    rocksdb_writebatch_t* b, rocksdb_column_family_handle_t* column_family,
-    const char* key, size_t klen) {
-  b->rep.SingleDelete(column_family->rep, Slice(key, klen));
-}
-
 void rocksdb_writebatch_deletev(
     rocksdb_writebatch_t* b,
     int num_keys, const char* const* keys_list,
@@ -1734,11 +1723,6 @@ void rocksdb_writebatch_wi_delete(
   b->rep->Delete(Slice(key, klen));
 }
 
-void rocksdb_writebatch_wi_singledelete(rocksdb_writebatch_wi_t* b,
-                                        const char* key, size_t klen) {
-  b->rep->SingleDelete(Slice(key, klen));
-}
-
 void rocksdb_writebatch_wi_delete_cf(
     rocksdb_writebatch_wi_t* b,
     rocksdb_column_family_handle_t* column_family,
@@ -1746,12 +1730,6 @@ void rocksdb_writebatch_wi_delete_cf(
   b->rep->Delete(column_family->rep, Slice(key, klen));
 }
 
-void rocksdb_writebatch_wi_singledelete_cf(
-    rocksdb_writebatch_wi_t* b, rocksdb_column_family_handle_t* column_family,
-    const char* key, size_t klen) {
-  b->rep->SingleDelete(column_family->rep, Slice(key, klen));
-}
-
 void rocksdb_writebatch_wi_deletev(
     rocksdb_writebatch_wi_t* b,
     int num_keys, const char* const* keys_list,
@@ -2176,10 +2154,6 @@ void rocksdb_options_destroy(rocksdb_options_t* options) {
   delete options;
 }
 
-rocksdb_options_t* rocksdb_options_create_copy(rocksdb_options_t* options) {
-  return new rocksdb_options_t(*options);
-}
-
 void rocksdb_options_increase_parallelism(
     rocksdb_options_t* opt, int total_threads) {
   opt->rep.IncreaseParallelism(total_threads);
@@ -2205,10 +2179,6 @@ void rocksdb_options_set_allow_ingest_behind(
   opt->rep.allow_ingest_behind = v;
 }
 
-unsigned char rocksdb_options_get_allow_ingest_behind(rocksdb_options_t* opt) {
-  return opt->rep.allow_ingest_behind;
-}
-
 void rocksdb_options_set_compaction_filter(
     rocksdb_options_t* opt,
     rocksdb_compactionfilter_t* filter) {
@@ -2226,10 +2196,6 @@ void rocksdb_options_compaction_readahead_size(
   opt->rep.compaction_readahead_size = s;
 }
 
-size_t rocksdb_options_get_compaction_readahead_size(rocksdb_options_t* opt) {
-  return opt->rep.compaction_readahead_size;
-}
-
 void rocksdb_options_set_comparator(
     rocksdb_options_t* opt,
     rocksdb_comparator_t* cmp) {
@@ -2242,43 +2208,27 @@ void rocksdb_options_set_merge_operator(
   opt->rep.merge_operator = std::shared_ptr<MergeOperator>(merge_operator);
 }
 
+
 void rocksdb_options_set_create_if_missing(
     rocksdb_options_t* opt, unsigned char v) {
   opt->rep.create_if_missing = v;
 }
 
-unsigned char rocksdb_options_get_create_if_missing(rocksdb_options_t* opt) {
-  return opt->rep.create_if_missing;
-}
-
 void rocksdb_options_set_create_missing_column_families(
     rocksdb_options_t* opt, unsigned char v) {
   opt->rep.create_missing_column_families = v;
 }
 
-unsigned char rocksdb_options_get_create_missing_column_families(
-    rocksdb_options_t* opt) {
-  return opt->rep.create_missing_column_families;
-}
-
 void rocksdb_options_set_error_if_exists(
     rocksdb_options_t* opt, unsigned char v) {
   opt->rep.error_if_exists = v;
 }
 
-unsigned char rocksdb_options_get_error_if_exists(rocksdb_options_t* opt) {
-  return opt->rep.error_if_exists;
-}
-
 void rocksdb_options_set_paranoid_checks(
     rocksdb_options_t* opt, unsigned char v) {
   opt->rep.paranoid_checks = v;
 }
 
-unsigned char rocksdb_options_get_paranoid_checks(rocksdb_options_t* opt) {
-  return opt->rep.paranoid_checks;
-}
-
 void rocksdb_options_set_db_paths(rocksdb_options_t* opt,
                                   const rocksdb_dbpath_t** dbpath_values,
                                   size_t num_paths) {
@@ -2304,98 +2254,52 @@ void rocksdb_options_set_info_log_level(
   opt->rep.info_log_level = static_cast<InfoLogLevel>(v);
 }
 
-int rocksdb_options_get_info_log_level(rocksdb_options_t* opt) {
-  return static_cast<int>(opt->rep.info_log_level);
-}
-
 void rocksdb_options_set_db_write_buffer_size(rocksdb_options_t* opt,
                                               size_t s) {
   opt->rep.db_write_buffer_size = s;
 }
 
-size_t rocksdb_options_get_db_write_buffer_size(rocksdb_options_t* opt) {
-  return opt->rep.db_write_buffer_size;
-}
-
 void rocksdb_options_set_write_buffer_size(rocksdb_options_t* opt, size_t s) {
   opt->rep.write_buffer_size = s;
 }
 
-size_t rocksdb_options_get_write_buffer_size(rocksdb_options_t* opt) {
-  return opt->rep.write_buffer_size;
-}
-
 void rocksdb_options_set_max_open_files(rocksdb_options_t* opt, int n) {
   opt->rep.max_open_files = n;
 }
 
-int rocksdb_options_get_max_open_files(rocksdb_options_t* opt) {
-  return opt->rep.max_open_files;
-}
-
 void rocksdb_options_set_max_file_opening_threads(rocksdb_options_t* opt, int n) {
   opt->rep.max_file_opening_threads = n;
 }
 
-int rocksdb_options_get_max_file_opening_threads(rocksdb_options_t* opt) {
-  return opt->rep.max_file_opening_threads;
-}
-
 void rocksdb_options_set_max_total_wal_size(rocksdb_options_t* opt, uint64_t n) {
   opt->rep.max_total_wal_size = n;
 }
 
-uint64_t rocksdb_options_get_max_total_wal_size(rocksdb_options_t* opt) {
-  return opt->rep.max_total_wal_size;
-}
-
 void rocksdb_options_set_target_file_size_base(
     rocksdb_options_t* opt, uint64_t n) {
   opt->rep.target_file_size_base = n;
 }
 
-uint64_t rocksdb_options_get_target_file_size_base(rocksdb_options_t* opt) {
-  return opt->rep.target_file_size_base;
-}
-
 void rocksdb_options_set_target_file_size_multiplier(
     rocksdb_options_t* opt, int n) {
   opt->rep.target_file_size_multiplier = n;
 }
 
-int rocksdb_options_get_target_file_size_multiplier(rocksdb_options_t* opt) {
-  return opt->rep.target_file_size_multiplier;
-}
-
 void rocksdb_options_set_max_bytes_for_level_base(
     rocksdb_options_t* opt, uint64_t n) {
   opt->rep.max_bytes_for_level_base = n;
 }
 
-uint64_t rocksdb_options_get_max_bytes_for_level_base(rocksdb_options_t* opt) {
-  return opt->rep.max_bytes_for_level_base;
-}
-
 void rocksdb_options_set_level_compaction_dynamic_level_bytes(
     rocksdb_options_t* opt, unsigned char v) {
   opt->rep.level_compaction_dynamic_level_bytes = v;
 }
 
-unsigned char rocksdb_options_get_level_compaction_dynamic_level_bytes(
-    rocksdb_options_t* opt) {
-  return opt->rep.level_compaction_dynamic_level_bytes;
-}
-
 void rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t* opt,
                                                         double n) {
   opt->rep.max_bytes_for_level_multiplier = n;
 }
 
-double rocksdb_options_get_max_bytes_for_level_multiplier(
-    rocksdb_options_t* opt) {
-  return opt->rep.max_bytes_for_level_multiplier;
-}
-
 void rocksdb_options_set_max_compaction_bytes(rocksdb_options_t* opt,
                                               uint64_t n) {
   opt->rep.max_compaction_bytes = n;
@@ -2418,57 +2322,30 @@ void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt,
   opt->rep.skip_stats_update_on_db_open = val;
 }
 
-unsigned char rocksdb_options_get_skip_stats_update_on_db_open(
-    rocksdb_options_t* opt) {
-  return opt->rep.skip_stats_update_on_db_open;
-}
-
 void rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open(
     rocksdb_options_t* opt, unsigned char val) {
   opt->rep.skip_checking_sst_file_sizes_on_db_open = val;
 }
 
-unsigned char rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(
-    rocksdb_options_t* opt) {
-  return opt->rep.skip_checking_sst_file_sizes_on_db_open;
-}
-
 void rocksdb_options_set_num_levels(rocksdb_options_t* opt, int n) {
   opt->rep.num_levels = n;
 }
 
-int rocksdb_options_get_num_levels(rocksdb_options_t* opt) {
-  return opt->rep.num_levels;
-}
-
 void rocksdb_options_set_level0_file_num_compaction_trigger(
     rocksdb_options_t* opt, int n) {
   opt->rep.level0_file_num_compaction_trigger = n;
 }
 
-int rocksdb_options_get_level0_file_num_compaction_trigger(
-    rocksdb_options_t* opt) {
-  return opt->rep.level0_file_num_compaction_trigger;
-}
-
 void rocksdb_options_set_level0_slowdown_writes_trigger(
     rocksdb_options_t* opt, int n) {
   opt->rep.level0_slowdown_writes_trigger = n;
 }
 
-int rocksdb_options_get_level0_slowdown_writes_trigger(rocksdb_options_t* opt) {
-  return opt->rep.level0_slowdown_writes_trigger;
-}
-
 void rocksdb_options_set_level0_stop_writes_trigger(
     rocksdb_options_t* opt, int n) {
   opt->rep.level0_stop_writes_trigger = n;
 }
 
-int rocksdb_options_get_level0_stop_writes_trigger(rocksdb_options_t* opt) {
-  return opt->rep.level0_stop_writes_trigger;
-}
-
 void rocksdb_options_set_max_mem_compaction_level(rocksdb_options_t* /*opt*/,
                                                   int /*n*/) {}
 
@@ -2480,10 +2357,6 @@ void rocksdb_options_set_compression(rocksdb_options_t* opt, int t) {
   opt->rep.compression = static_cast<CompressionType>(t);
 }
 
-void rocksdb_options_set_bottommost_compression(rocksdb_options_t* opt, int t) {
-  opt->rep.bottommost_compression = static_cast<CompressionType>(t);
-}
-
 void rocksdb_options_set_compression_per_level(rocksdb_options_t* opt,
                                                int* level_values,
                                                size_t num_levels) {
@@ -2498,7 +2371,7 @@ void rocksdb_options_set_bottommost_compression_options(rocksdb_options_t* opt,
                                                         int w_bits, int level,
                                                         int strategy,
                                                         int max_dict_bytes,
-                                                        unsigned char enabled) {
+                                                        bool enabled) {
   opt->rep.bottommost_compression_opts.window_bits = w_bits;
   opt->rep.bottommost_compression_opts.level = level;
   opt->rep.bottommost_compression_opts.strategy = strategy;
@@ -2506,13 +2379,6 @@ void rocksdb_options_set_bottommost_compression_options(rocksdb_options_t* opt,
   opt->rep.bottommost_compression_opts.enabled = enabled;
 }
 
-void rocksdb_options_set_bottommost_compression_options_zstd_max_train_bytes(
-    rocksdb_options_t* opt, int zstd_max_train_bytes, unsigned char enabled) {
-  opt->rep.bottommost_compression_opts.zstd_max_train_bytes =
-      zstd_max_train_bytes;
-  opt->rep.bottommost_compression_opts.enabled = enabled;
-}
-
 void rocksdb_options_set_compression_options(rocksdb_options_t* opt, int w_bits,
                                              int level, int strategy,
                                              int max_dict_bytes) {
@@ -2522,11 +2388,6 @@ void rocksdb_options_set_compression_options(rocksdb_options_t* opt, int w_bits,
   opt->rep.compression_opts.max_dict_bytes = max_dict_bytes;
 }
 
-void rocksdb_options_set_compression_options_zstd_max_train_bytes(
-    rocksdb_options_t* opt, int zstd_max_train_bytes) {
-  opt->rep.compression_opts.zstd_max_train_bytes = zstd_max_train_bytes;
-}
-
 void rocksdb_options_set_prefix_extractor(
     rocksdb_options_t* opt, rocksdb_slicetransform_t* prefix_extractor) {
   opt->rep.prefix_extractor.reset(prefix_extractor);
@@ -2666,67 +2527,35 @@ void rocksdb_options_set_max_write_buffer_number(rocksdb_options_t* opt, int n)
   opt->rep.max_write_buffer_number = n;
 }
 
-int rocksdb_options_get_max_write_buffer_number(rocksdb_options_t* opt) {
-  return opt->rep.max_write_buffer_number;
-}
-
 void rocksdb_options_set_min_write_buffer_number_to_merge(rocksdb_options_t* opt, int n) {
   opt->rep.min_write_buffer_number_to_merge = n;
 }
 
-int rocksdb_options_get_min_write_buffer_number_to_merge(
-    rocksdb_options_t* opt) {
-  return opt->rep.min_write_buffer_number_to_merge;
-}
-
 void rocksdb_options_set_max_write_buffer_number_to_maintain(
     rocksdb_options_t* opt, int n) {
   opt->rep.max_write_buffer_number_to_maintain = n;
 }
 
-int rocksdb_options_get_max_write_buffer_number_to_maintain(
-    rocksdb_options_t* opt) {
-  return opt->rep.max_write_buffer_number_to_maintain;
-}
-
 void rocksdb_options_set_max_write_buffer_size_to_maintain(
     rocksdb_options_t* opt, int64_t n) {
   opt->rep.max_write_buffer_size_to_maintain = n;
 }
 
-int64_t rocksdb_options_get_max_write_buffer_size_to_maintain(
-    rocksdb_options_t* opt) {
-  return opt->rep.max_write_buffer_size_to_maintain;
-}
-
 void rocksdb_options_set_enable_pipelined_write(rocksdb_options_t* opt,
                                                 unsigned char v) {
   opt->rep.enable_pipelined_write = v;
 }
 
-unsigned char rocksdb_options_get_enable_pipelined_write(
-    rocksdb_options_t* opt) {
-  return opt->rep.enable_pipelined_write;
-}
-
 void rocksdb_options_set_unordered_write(rocksdb_options_t* opt,
                                          unsigned char v) {
   opt->rep.unordered_write = v;
 }
 
-unsigned char rocksdb_options_get_unordered_write(rocksdb_options_t* opt) {
-  return opt->rep.unordered_write;
-}
-
 void rocksdb_options_set_max_subcompactions(rocksdb_options_t* opt,
                                             uint32_t n) {
   opt->rep.max_subcompactions = n;
 }
 
-uint32_t rocksdb_options_get_max_subcompactions(rocksdb_options_t* opt) {
-  return opt->rep.max_subcompactions;
-}
-
 void rocksdb_options_set_max_background_jobs(rocksdb_options_t* opt, int n) {
   opt->rep.max_background_jobs = n;
 }
@@ -4627,10 +4456,6 @@ void rocksdb_approximate_memory_usage_destroy(rocksdb_memory_usage_t* usage) {
   delete usage;
 }
 
-void rocksdb_cancel_all_background_work(rocksdb_t* db, unsigned char wait) {
-  CancelAllBackgroundWork(db->rep, wait);
-}
-
 }  // end extern "C"
 
 #endif  // !ROCKSDB_LITE

db/c_test.c

@@ -1461,294 +1461,6 @@ int main(int argc, char** argv) {
     rocksdb_cuckoo_options_destroy(cuckoo_options);
   }
 
-  StartPhase("options");
-  {
-    rocksdb_options_t* o;
-    o = rocksdb_options_create();
-
-    // Set and check options.
-    rocksdb_options_set_allow_ingest_behind(o, 1);
-    CheckCondition(1 == rocksdb_options_get_allow_ingest_behind(o));
-
-    rocksdb_options_compaction_readahead_size(o, 10);
-    CheckCondition(10 == rocksdb_options_get_compaction_readahead_size(o));
-
-    rocksdb_options_set_create_if_missing(o, 1);
-    CheckCondition(1 == rocksdb_options_get_create_if_missing(o));
-
-    rocksdb_options_set_create_missing_column_families(o, 1);
-    CheckCondition(1 == rocksdb_options_get_create_missing_column_families(o));
-
-    rocksdb_options_set_error_if_exists(o, 1);
-    CheckCondition(1 == rocksdb_options_get_error_if_exists(o));
-
-    rocksdb_options_set_paranoid_checks(o, 1);
-    CheckCondition(1 == rocksdb_options_get_paranoid_checks(o));
-
-    rocksdb_options_set_info_log_level(o, 3);
-    CheckCondition(3 == rocksdb_options_get_info_log_level(o));
-
-    rocksdb_options_set_write_buffer_size(o, 100);
-    CheckCondition(100 == rocksdb_options_get_write_buffer_size(o));
-
-    rocksdb_options_set_db_write_buffer_size(o, 1000);
-    CheckCondition(1000 == rocksdb_options_get_db_write_buffer_size(o));
-
-    rocksdb_options_set_max_open_files(o, 21);
-    CheckCondition(21 == rocksdb_options_get_max_open_files(o));
-
-    rocksdb_options_set_max_file_opening_threads(o, 5);
-    CheckCondition(5 == rocksdb_options_get_max_file_opening_threads(o));
-
-    rocksdb_options_set_max_total_wal_size(o, 400);
-    CheckCondition(400 == rocksdb_options_get_max_total_wal_size(o));
-
-    rocksdb_options_set_num_levels(o, 7);
-    CheckCondition(7 == rocksdb_options_get_num_levels(o));
-
-    rocksdb_options_set_level0_file_num_compaction_trigger(o, 4);
-    CheckCondition(4 ==
-                   rocksdb_options_get_level0_file_num_compaction_trigger(o));
-
-    rocksdb_options_set_level0_slowdown_writes_trigger(o, 6);
-    CheckCondition(6 == rocksdb_options_get_level0_slowdown_writes_trigger(o));
-
-    rocksdb_options_set_level0_stop_writes_trigger(o, 8);
-    CheckCondition(8 == rocksdb_options_get_level0_stop_writes_trigger(o));
-
-    rocksdb_options_set_target_file_size_base(o, 256);
-    CheckCondition(256 == rocksdb_options_get_target_file_size_base(o));
-
-    rocksdb_options_set_target_file_size_multiplier(o, 3);
-    CheckCondition(3 == rocksdb_options_get_target_file_size_multiplier(o));
-
-    rocksdb_options_set_max_bytes_for_level_base(o, 1024);
-    CheckCondition(1024 == rocksdb_options_get_max_bytes_for_level_base(o));
-
-    rocksdb_options_set_level_compaction_dynamic_level_bytes(o, 1);
-    CheckCondition(1 ==
-                   rocksdb_options_get_level_compaction_dynamic_level_bytes(o));
-
-    rocksdb_options_set_max_bytes_for_level_multiplier(o, 2.0);
-    CheckCondition(2.0 ==
-                   rocksdb_options_get_max_bytes_for_level_multiplier(o));
-
-    rocksdb_options_set_skip_stats_update_on_db_open(o, 1);
-    CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o));
-
-    rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open(o, 1);
-    CheckCondition(
-        1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(o));
-
-    rocksdb_options_set_max_write_buffer_number(o, 97);
-    CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(o));
-
-    rocksdb_options_set_min_write_buffer_number_to_merge(o, 23);
-    CheckCondition(23 ==
-                   rocksdb_options_get_min_write_buffer_number_to_merge(o));
-
-    rocksdb_options_set_max_write_buffer_number_to_maintain(o, 64);
-    CheckCondition(64 ==
-                   rocksdb_options_get_max_write_buffer_number_to_maintain(o));
-
-    rocksdb_options_set_max_write_buffer_size_to_maintain(o, 50000);
-    CheckCondition(50000 ==
-                   rocksdb_options_get_max_write_buffer_size_to_maintain(o));
-
-    rocksdb_options_set_enable_pipelined_write(o, 1);
-    CheckCondition(1 == rocksdb_options_get_enable_pipelined_write(o));
-
-    rocksdb_options_set_unordered_write(o, 1);
-    CheckCondition(1 == rocksdb_options_get_unordered_write(o));
-
-    rocksdb_options_set_max_subcompactions(o, 123456);
-    CheckCondition(123456 == rocksdb_options_get_max_subcompactions(o));
-
-    // Create a copy that should be equal to the original.
-    rocksdb_options_t* copy;
-    copy = rocksdb_options_create_copy(o);
-
-    CheckCondition(1 == rocksdb_options_get_allow_ingest_behind(copy));
-    CheckCondition(10 == rocksdb_options_get_compaction_readahead_size(copy));
-    CheckCondition(1 == rocksdb_options_get_create_if_missing(copy));
-    CheckCondition(1 ==
-                   rocksdb_options_get_create_missing_column_families(copy));
-    CheckCondition(1 == rocksdb_options_get_error_if_exists(copy));
-    CheckCondition(1 == rocksdb_options_get_paranoid_checks(copy));
-    CheckCondition(3 == rocksdb_options_get_info_log_level(copy));
-    CheckCondition(100 == rocksdb_options_get_write_buffer_size(copy));
-    CheckCondition(1000 == rocksdb_options_get_db_write_buffer_size(copy));
-    CheckCondition(21 == rocksdb_options_get_max_open_files(copy));
-    CheckCondition(5 == rocksdb_options_get_max_file_opening_threads(copy));
-    CheckCondition(400 == rocksdb_options_get_max_total_wal_size(copy));
-    CheckCondition(7 == rocksdb_options_get_num_levels(copy));
-    CheckCondition(
-        4 == rocksdb_options_get_level0_file_num_compaction_trigger(copy));
-    CheckCondition(6 ==
-                   rocksdb_options_get_level0_slowdown_writes_trigger(copy));
-    CheckCondition(8 == rocksdb_options_get_level0_stop_writes_trigger(copy));
-    CheckCondition(256 == rocksdb_options_get_target_file_size_base(copy));
-    CheckCondition(3 == rocksdb_options_get_target_file_size_multiplier(copy));
-    CheckCondition(1024 == rocksdb_options_get_max_bytes_for_level_base(copy));
-    CheckCondition(
-        1 == rocksdb_options_get_level_compaction_dynamic_level_bytes(copy));
-    CheckCondition(2.0 ==
-                   rocksdb_options_get_max_bytes_for_level_multiplier(copy));
-    CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(copy));
-    CheckCondition(
-        1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(copy));
-    CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(copy));
-    CheckCondition(23 ==
-                   rocksdb_options_get_min_write_buffer_number_to_merge(copy));
-    CheckCondition(
-        64 == rocksdb_options_get_max_write_buffer_number_to_maintain(copy));
-    CheckCondition(50000 ==
-                   rocksdb_options_get_max_write_buffer_size_to_maintain(copy));
-    CheckCondition(1 == rocksdb_options_get_enable_pipelined_write(copy));
-    CheckCondition(1 == rocksdb_options_get_unordered_write(copy));
-    CheckCondition(123456 == rocksdb_options_get_max_subcompactions(copy));
-
-    // Copies should be independent.
-    rocksdb_options_set_allow_ingest_behind(copy, 0);
-    CheckCondition(0 == rocksdb_options_get_allow_ingest_behind(copy));
-    CheckCondition(1 == rocksdb_options_get_allow_ingest_behind(o));
-
-    rocksdb_options_compaction_readahead_size(copy, 20);
-    CheckCondition(20 == rocksdb_options_get_compaction_readahead_size(copy));
-    CheckCondition(10 == rocksdb_options_get_compaction_readahead_size(o));
-
-    rocksdb_options_set_create_if_missing(copy, 0);
-    CheckCondition(0 == rocksdb_options_get_create_if_missing(copy));
-    CheckCondition(1 == rocksdb_options_get_create_if_missing(o));
-
-    rocksdb_options_set_create_missing_column_families(copy, 0);
-    CheckCondition(0 ==
-                   rocksdb_options_get_create_missing_column_families(copy));
-    CheckCondition(1 == rocksdb_options_get_create_missing_column_families(o));
-
-    rocksdb_options_set_error_if_exists(copy, 0);
-    CheckCondition(0 == rocksdb_options_get_error_if_exists(copy));
-    CheckCondition(1 == rocksdb_options_get_error_if_exists(o));
-
-    rocksdb_options_set_paranoid_checks(copy, 0);
-    CheckCondition(0 == rocksdb_options_get_paranoid_checks(copy));
-    CheckCondition(1 == rocksdb_options_get_paranoid_checks(o));
-
-    rocksdb_options_set_info_log_level(copy, 2);
-    CheckCondition(2 == rocksdb_options_get_info_log_level(copy));
-    CheckCondition(3 == rocksdb_options_get_info_log_level(o));
-
-    rocksdb_options_set_write_buffer_size(copy, 200);
-    CheckCondition(200 == rocksdb_options_get_write_buffer_size(copy));
-    CheckCondition(100 == rocksdb_options_get_write_buffer_size(o));
-
-    rocksdb_options_set_db_write_buffer_size(copy, 2000);
-    CheckCondition(2000 == rocksdb_options_get_db_write_buffer_size(copy));
-    CheckCondition(1000 == rocksdb_options_get_db_write_buffer_size(o));
-
-    rocksdb_options_set_max_open_files(copy, 42);
-    CheckCondition(42 == rocksdb_options_get_max_open_files(copy));
-    CheckCondition(21 == rocksdb_options_get_max_open_files(o));
-
-    rocksdb_options_set_max_file_opening_threads(copy, 3);
-    CheckCondition(3 == rocksdb_options_get_max_file_opening_threads(copy));
-    CheckCondition(5 == rocksdb_options_get_max_file_opening_threads(o));
-
-    rocksdb_options_set_max_total_wal_size(copy, 4000);
-    CheckCondition(4000 == rocksdb_options_get_max_total_wal_size(copy));
-    CheckCondition(400 == rocksdb_options_get_max_total_wal_size(o));
-
-    rocksdb_options_set_num_levels(copy, 6);
-    CheckCondition(6 == rocksdb_options_get_num_levels(copy));
-    CheckCondition(7 == rocksdb_options_get_num_levels(o));
-
-    rocksdb_options_set_level0_file_num_compaction_trigger(copy, 14);
-    CheckCondition(
-        14 == rocksdb_options_get_level0_file_num_compaction_trigger(copy));
-    CheckCondition(4 ==
-                   rocksdb_options_get_level0_file_num_compaction_trigger(o));
-
-    rocksdb_options_set_level0_slowdown_writes_trigger(copy, 61);
-    CheckCondition(61 ==
-                   rocksdb_options_get_level0_slowdown_writes_trigger(copy));
-    CheckCondition(6 == rocksdb_options_get_level0_slowdown_writes_trigger(o));
-
-    rocksdb_options_set_level0_stop_writes_trigger(copy, 17);
-    CheckCondition(17 == rocksdb_options_get_level0_stop_writes_trigger(copy));
-    CheckCondition(8 == rocksdb_options_get_level0_stop_writes_trigger(o));
-
-    rocksdb_options_set_target_file_size_base(copy, 128);
-    CheckCondition(128 == rocksdb_options_get_target_file_size_base(copy));
-    CheckCondition(256 == rocksdb_options_get_target_file_size_base(o));
-
-    rocksdb_options_set_target_file_size_multiplier(copy, 13);
-    CheckCondition(13 == rocksdb_options_get_target_file_size_multiplier(copy));
-    CheckCondition(3 == rocksdb_options_get_target_file_size_multiplier(o));
-
-    rocksdb_options_set_max_bytes_for_level_base(copy, 900);
-    CheckCondition(900 == rocksdb_options_get_max_bytes_for_level_base(copy));
-    CheckCondition(1024 == rocksdb_options_get_max_bytes_for_level_base(o));
-
-    rocksdb_options_set_level_compaction_dynamic_level_bytes(copy, 0);
-    CheckCondition(
-        0 == rocksdb_options_get_level_compaction_dynamic_level_bytes(copy));
-    CheckCondition(1 ==
-                   rocksdb_options_get_level_compaction_dynamic_level_bytes(o));
-
-    rocksdb_options_set_max_bytes_for_level_multiplier(copy, 8.0);
-    CheckCondition(8.0 ==
-                   rocksdb_options_get_max_bytes_for_level_multiplier(copy));
-    CheckCondition(2.0 ==
-                   rocksdb_options_get_max_bytes_for_level_multiplier(o));
-
-    rocksdb_options_set_skip_stats_update_on_db_open(copy, 0);
-    CheckCondition(0 == rocksdb_options_get_skip_stats_update_on_db_open(copy));
-    CheckCondition(1 == rocksdb_options_get_skip_stats_update_on_db_open(o));
-
-    rocksdb_options_set_skip_checking_sst_file_sizes_on_db_open(copy, 0);
-    CheckCondition(
-        0 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(copy));
-    CheckCondition(
-        1 == rocksdb_options_get_skip_checking_sst_file_sizes_on_db_open(o));
-
-    rocksdb_options_set_max_write_buffer_number(copy, 2000);
-    CheckCondition(2000 == rocksdb_options_get_max_write_buffer_number(copy));
-    CheckCondition(97 == rocksdb_options_get_max_write_buffer_number(o));
-
-    rocksdb_options_set_min_write_buffer_number_to_merge(copy, 146);
-    CheckCondition(146 ==
-                   rocksdb_options_get_min_write_buffer_number_to_merge(copy));
-    CheckCondition(23 ==
-                   rocksdb_options_get_min_write_buffer_number_to_merge(o));
-
-    rocksdb_options_set_max_write_buffer_number_to_maintain(copy, 128);
-    CheckCondition(
-        128 == rocksdb_options_get_max_write_buffer_number_to_maintain(copy));
-    CheckCondition(64 ==
-                   rocksdb_options_get_max_write_buffer_number_to_maintain(o));
-
-    rocksdb_options_set_max_write_buffer_size_to_maintain(copy, 9000);
-    CheckCondition(9000 ==
-                   rocksdb_options_get_max_write_buffer_size_to_maintain(copy));
-    CheckCondition(50000 ==
-                   rocksdb_options_get_max_write_buffer_size_to_maintain(o));
-
-    rocksdb_options_set_enable_pipelined_write(copy, 0);
-    CheckCondition(0 == rocksdb_options_get_enable_pipelined_write(copy));
-    CheckCondition(1 == rocksdb_options_get_enable_pipelined_write(o));
-
-    rocksdb_options_set_unordered_write(copy, 0);
-    CheckCondition(0 == rocksdb_options_get_unordered_write(copy));
-    CheckCondition(1 == rocksdb_options_get_unordered_write(o));
-
-    rocksdb_options_set_max_subcompactions(copy, 90001);
-    CheckCondition(90001 == rocksdb_options_get_max_subcompactions(copy));
-    CheckCondition(123456 == rocksdb_options_get_max_subcompactions(o));
-
-    rocksdb_options_destroy(copy);
-    rocksdb_options_destroy(o);
-  }
-
   StartPhase("iterate_upper_bound");
   {
     // Create new empty database
@@ -2128,9 +1840,6 @@ int main(int argc, char** argv) {
     CheckNoError(err);
   }
 
-  StartPhase("cancel_all_background_work");
-  rocksdb_cancel_all_background_work(db, 1);
-
   StartPhase("cleanup");
   rocksdb_close(db);
   rocksdb_options_destroy(options);

db/compaction/compaction_iterator.cc

@@ -246,7 +246,6 @@ void CompactionIterator::NextFromInput() {
     iter_stats_.num_input_records++;
 
     if (!ParseInternalKey(key_, &ikey_)) {
-      iter_stats_.num_input_corrupt_records++;
       // If `expect_valid_internal_key_` is false, return the corrupted key
       // and let the caller decide what to do with it.
       // TODO(noetzli): We should have a more elegant solution for this.
@@ -259,6 +258,7 @@ void CompactionIterator::NextFromInput() {
       has_current_user_key_ = false;
       current_user_key_sequence_ = kMaxSequenceNumber;
       current_user_key_snapshot_ = 0;
+      iter_stats_.num_input_corrupt_records++;
       valid_ = true;
       break;
     }

db/compaction/compaction_job.cc

@@ -542,7 +542,7 @@ void CompactionJob::GenSubcompactionBoundaries() {
     // Greedily add ranges to the subcompaction until the sum of the ranges'
     // sizes becomes >= the expected mean size of a subcompaction
     sum = 0;
-    for (size_t i = 0; i + 1 < ranges.size(); i++) {
+    for (size_t i = 0; i < ranges.size() - 1; i++) {
       sum += ranges[i].size;
       if (subcompactions == 1) {
         // If there's only one left to schedule then it goes to the end so no
@@ -654,8 +654,6 @@ Status CompactionJob::Run() {
                 compact_->compaction->output_level()),
             TableReaderCaller::kCompactionRefill, /*arena=*/nullptr,
             /*skip_filters=*/false, compact_->compaction->output_level(),
-            MaxFileSizeForL0MetaPin(
-                *compact_->compaction->mutable_cf_options()),
             /*smallest_compaction_key=*/nullptr,
             /*largest_compaction_key=*/nullptr,
             /*allow_unprepared_value=*/false);
@@ -721,6 +719,7 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
   cfd->internal_stats()->AddCompactionStats(
       compact_->compaction->output_level(), thread_pri_, compaction_stats_);
 
+  versions_->SetIOStatusOK();
   if (status.ok()) {
     status = InstallCompactionResults(mutable_cf_options);
   }
@@ -895,10 +894,9 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   sub_compact->c_iter.reset(new CompactionIterator(
       input.get(), cfd->user_comparator(), &merge, versions_->LastSequence(),
       &existing_snapshots_, earliest_write_conflict_snapshot_,
-      snapshot_checker_, env_, ShouldReportDetailedTime(env_, stats_),
-      /*expect_valid_internal_key=*/true, &range_del_agg,
-      sub_compact->compaction, compaction_filter, shutting_down_,
-      preserve_deletes_seqnum_, manual_compaction_paused_,
+      snapshot_checker_, env_, ShouldReportDetailedTime(env_, stats_), false,
+      &range_del_agg, sub_compact->compaction, compaction_filter,
+      shutting_down_, preserve_deletes_seqnum_, manual_compaction_paused_,
       db_options_.info_log));
   auto c_iter = sub_compact->c_iter.get();
   c_iter->SeekToFirst();

db/compaction/compaction_job_test.cc

@@ -395,7 +395,7 @@ TEST_F(CompactionJobTest, Simple) {
   RunCompaction({ files }, expected_results);
 }
 
-TEST_F(CompactionJobTest, DISABLED_SimpleCorrupted) {
+TEST_F(CompactionJobTest, SimpleCorrupted) {
   NewDB();
 
   auto expected_results = CreateTwoFiles(true);
@@ -989,7 +989,7 @@ TEST_F(CompactionJobTest, MultiSingleDelete) {
 // single deletion and the (single) deletion gets removed while the corrupt key
 // gets written out. TODO(noetzli): We probably want a better way to treat
 // corrupt keys.
-TEST_F(CompactionJobTest, DISABLED_CorruptionAfterDeletion) {
+TEST_F(CompactionJobTest, CorruptionAfterDeletion) {
   NewDB();
 
   auto file1 =

db/compaction/compaction_picker_level.cc

@@ -93,13 +93,9 @@ class LevelCompactionBuilder {
   // otherwise, returns false.
   bool PickIntraL0Compaction();
 
-  // Picks a file from level_files to compact.
-  // level_files is a vector of (level, file metadata) in ascending order of
-  // level. If compact_to_next_level is true, compact the file to the next
-  // level, otherwise, compact to the same level as the input file.
-  void PickFileToCompact(
-      const autovector<std::pair<int, FileMetaData*>>& level_files,
-      bool compact_to_next_level);
+  void PickExpiredTtlFiles();
+
+  void PickFilesMarkedForPeriodicCompaction();
 
   const std::string& cf_name_;
   VersionStorageInfo* vstorage_;
@@ -128,34 +124,72 @@ class LevelCompactionBuilder {
   static const int kMinFilesForIntraL0Compaction = 4;
 };
 
-void LevelCompactionBuilder::PickFileToCompact(
-    const autovector<std::pair<int, FileMetaData*>>& level_files,
-    bool compact_to_next_level) {
-  for (auto& level_file : level_files) {
+void LevelCompactionBuilder::PickExpiredTtlFiles() {
+  if (vstorage_->ExpiredTtlFiles().empty()) {
+    return;
+  }
+
+  auto continuation = [&](std::pair<int, FileMetaData*> level_file) {
     // If it's being compacted it has nothing to do here.
     // If this assert() fails that means that some function marked some
     // files as being_compacted, but didn't call ComputeCompactionScore()
     assert(!level_file.second->being_compacted);
     start_level_ = level_file.first;
-    if ((compact_to_next_level &&
-         start_level_ == vstorage_->num_non_empty_levels() - 1) ||
+    output_level_ =
+        (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1;
+
+    if ((start_level_ == vstorage_->num_non_empty_levels() - 1) ||
         (start_level_ == 0 &&
          !compaction_picker_->level0_compactions_in_progress()->empty())) {
-      continue;
-    }
-    if (compact_to_next_level) {
-      output_level_ =
-          (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1;
-    } else {
-      output_level_ = start_level_;
+      return false;
     }
+
     start_level_inputs_.files = {level_file.second};
     start_level_inputs_.level = start_level_;
-    if (compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
-                                                   &start_level_inputs_)) {
+    return compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+                                                      &start_level_inputs_);
+  };
+
+  for (auto& level_file : vstorage_->ExpiredTtlFiles()) {
+    if (continuation(level_file)) {
+      // found the compaction!
       return;
     }
   }
+
+  start_level_inputs_.files.clear();
+}
+
+void LevelCompactionBuilder::PickFilesMarkedForPeriodicCompaction() {
+  if (vstorage_->FilesMarkedForPeriodicCompaction().empty()) {
+    return;
+  }
+
+  auto continuation = [&](std::pair<int, FileMetaData*> level_file) {
+    // If it's being compacted it has nothing to do here.
+    // If this assert() fails that means that some function marked some
+    // files as being_compacted, but didn't call ComputeCompactionScore()
+    assert(!level_file.second->being_compacted);
+    output_level_ = start_level_ = level_file.first;
+
+    if (start_level_ == 0 &&
+        !compaction_picker_->level0_compactions_in_progress()->empty()) {
+      return false;
+    }
+
+    start_level_inputs_.files = {level_file.second};
+    start_level_inputs_.level = start_level_;
+    return compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+                                                      &start_level_inputs_);
+  };
+
+  for (auto& level_file : vstorage_->FilesMarkedForPeriodicCompaction()) {
+    if (continuation(level_file)) {
+      // found the compaction!
+      return;
+    }
+  }
+
   start_level_inputs_.files.clear();
 }
 
@@ -204,46 +238,63 @@ void LevelCompactionBuilder::SetupInitialFiles() {
           }
         }
       }
-    } else {
-      // Compaction scores are sorted in descending order, no further scores
-      // will be >= 1.
-      break;
     }
   }
-  if (!start_level_inputs_.empty()) {
-    return;
-  }
 
   // if we didn't find a compaction, check if there are any files marked for
   // compaction
-  parent_index_ = base_index_ = -1;
+  if (start_level_inputs_.empty()) {
+    parent_index_ = base_index_ = -1;
 
-  compaction_picker_->PickFilesMarkedForCompaction(
-      cf_name_, vstorage_, &start_level_, &output_level_, &start_level_inputs_);
-  if (!start_level_inputs_.empty()) {
-    compaction_reason_ = CompactionReason::kFilesMarkedForCompaction;
-    return;
+    compaction_picker_->PickFilesMarkedForCompaction(
+        cf_name_, vstorage_, &start_level_, &output_level_,
+        &start_level_inputs_);
+    if (!start_level_inputs_.empty()) {
+      compaction_reason_ = CompactionReason::kFilesMarkedForCompaction;
+      return;
+    }
   }
 
   // Bottommost Files Compaction on deleting tombstones
-  PickFileToCompact(vstorage_->BottommostFilesMarkedForCompaction(), false);
-  if (!start_level_inputs_.empty()) {
-    compaction_reason_ = CompactionReason::kBottommostFiles;
-    return;
+  if (start_level_inputs_.empty()) {
+    size_t i;
+    for (i = 0; i < vstorage_->BottommostFilesMarkedForCompaction().size();
+         ++i) {
+      auto& level_and_file = vstorage_->BottommostFilesMarkedForCompaction()[i];
+      assert(!level_and_file.second->being_compacted);
+      start_level_inputs_.level = output_level_ = start_level_ =
+          level_and_file.first;
+      start_level_inputs_.files = {level_and_file.second};
+      if (compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+                                                     &start_level_inputs_)) {
+        break;
+      }
+    }
+    if (i == vstorage_->BottommostFilesMarkedForCompaction().size()) {
+      start_level_inputs_.clear();
+    } else {
+      assert(!start_level_inputs_.empty());
+      compaction_reason_ = CompactionReason::kBottommostFiles;
+      return;
+    }
   }
 
   // TTL Compaction
-  PickFileToCompact(vstorage_->ExpiredTtlFiles(), true);
-  if (!start_level_inputs_.empty()) {
-    compaction_reason_ = CompactionReason::kTtl;
-    return;
+  if (start_level_inputs_.empty()) {
+    PickExpiredTtlFiles();
+    if (!start_level_inputs_.empty()) {
+      compaction_reason_ = CompactionReason::kTtl;
+      return;
+    }
   }
 
   // Periodic Compaction
-  PickFileToCompact(vstorage_->FilesMarkedForPeriodicCompaction(), false);
-  if (!start_level_inputs_.empty()) {
-    compaction_reason_ = CompactionReason::kPeriodicCompaction;
-    return;
+  if (start_level_inputs_.empty()) {
+    PickFilesMarkedForPeriodicCompaction();
+    if (!start_level_inputs_.empty()) {
+      compaction_reason_ = CompactionReason::kPeriodicCompaction;
+      return;
+    }
   }
 }
 

db/compaction/compaction_picker_test.cc

@@ -414,8 +414,8 @@ TEST_F(CompactionPickerTest, LevelTriggerDynamic4) {
   mutable_cf_options_.max_bytes_for_level_multiplier = 10;
   NewVersionStorage(num_levels, kCompactionStyleLevel);
   Add(0, 1U, "150", "200");
-  Add(num_levels - 1, 2U, "200", "250", 300U);
-  Add(num_levels - 1, 3U, "300", "350", 3000U);
+  Add(num_levels - 1, 3U, "200", "250", 300U);
+  Add(num_levels - 1, 4U, "300", "350", 3000U);
   Add(num_levels - 1, 4U, "400", "450", 3U);
   Add(num_levels - 2, 5U, "150", "180", 300U);
   Add(num_levels - 2, 6U, "181", "350", 500U);
@@ -782,7 +782,7 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
   Add(2, 8U, "201", "300",
       60000000U);  // Overlaps with file 28, 29, total size 521M
 
-  Add(3, 25U, "100", "110", 261000000U);
+  Add(3, 26U, "100", "110", 261000000U);
   Add(3, 26U, "150", "170", 261000000U);
   Add(3, 27U, "171", "179", 260000000U);
   Add(3, 28U, "191", "220", 260000000U);
@@ -1298,7 +1298,7 @@ TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded1) {
   // Size ratio L4/L3 is 9.9
   // After merge from L3, L4 size is 1000900
   Add(4, 11U, "400", "500", 999900);
-  Add(5, 12U, "400", "500", 8007200);
+  Add(5, 11U, "400", "500", 8007200);
 
   UpdateVersionStorageInfo();
 
@@ -1611,8 +1611,8 @@ TEST_F(CompactionPickerTest, IsTrivialMoveOn) {
 
   Add(3, 5U, "120", "130", 7000U);
   Add(3, 6U, "170", "180", 7000U);
-  Add(3, 7U, "220", "230", 7000U);
-  Add(3, 8U, "270", "280", 7000U);
+  Add(3, 5U, "220", "230", 7000U);
+  Add(3, 5U, "270", "280", 7000U);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
@@ -1776,7 +1776,6 @@ TEST_F(CompactionPickerTest, IntraL0ForEarliestSeqno) {
   ASSERT_EQ(0, compaction->output_level());
 }
 
-#ifndef ROCKSDB_LITE
 TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
   const uint64_t kFileSize = 100000;
 
@@ -1811,7 +1810,7 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionFullOverlap) {
 
   AddVersionStorage();
   // Simulate a flush and mark the file for compaction
-  Add(0, 7U, "150", "200", kFileSize, 0, 551, 600, 0, true);
+  Add(0, 1U, "150", "200", kFileSize, 0, 551, 600, 0, true);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction2(
@@ -1933,7 +1932,6 @@ TEST_F(CompactionPickerTest, UniversalMarkedCompactionStartOutputOverlap) {
     DeleteVersionStorage();
   }
 }
-#endif  // ROCKSDB_LITE
 
 }  // namespace ROCKSDB_NAMESPACE
 

db/compaction/compaction_picker_universal.cc

@@ -765,7 +765,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() {
   }
 
   // Skip files that are already being compacted
-  for (size_t loop = 0; loop + 1 < sorted_runs_.size(); loop++) {
+  for (size_t loop = 0; loop < sorted_runs_.size() - 1; loop++) {
     sr = &sorted_runs_[loop];
     if (!sr->being_compacted) {
       start_index = loop;  // Consider this as the first candidate.
@@ -793,7 +793,7 @@ Compaction* UniversalCompactionBuilder::PickCompactionToReduceSizeAmp() {
   }
 
   // keep adding up all the remaining files
-  for (size_t loop = start_index; loop + 1 < sorted_runs_.size(); loop++) {
+  for (size_t loop = start_index; loop < sorted_runs_.size() - 1; loop++) {
     sr = &sorted_runs_[loop];
     if (sr->being_compacted) {
       char file_num_buf[kFormatFileNumberBufSize];

db/convenience.cc

@@ -61,8 +61,7 @@ Status VerifySstFileChecksum(const Options& options,
   s = ioptions.table_factory->NewTableReader(
       TableReaderOptions(ioptions, options.prefix_extractor.get(), env_options,
                          internal_comparator, false /* skip_filters */,
-                         !kImmortal, false /* force_direct_prefetch */,
-                         -1 /* level */),
+                         !kImmortal, -1 /* level */),
       std::move(file_reader), file_size, &table_reader,
       false /* prefetch_index_and_filter_in_cache */);
   if (!s.ok()) {

db/db_basic_test.cc

@@ -9,7 +9,6 @@
 
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
-#include "rocksdb/merge_operator.h"
 #include "rocksdb/perf_context.h"
 #include "rocksdb/utilities/debug.h"
 #include "table/block_based/block_based_table_reader.h"
@@ -18,8 +17,6 @@
 #if !defined(ROCKSDB_LITE)
 #include "test_util/sync_point.h"
 #endif
-#include "utilities/merge_operators.h"
-#include "utilities/merge_operators/string_append/stringappend.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -1343,57 +1340,6 @@ TEST_F(DBBasicTest, MultiGetBatchedSortedMultiFile) {
   } while (ChangeOptions());
 }
 
-TEST_F(DBBasicTest, MultiGetBatchedDuplicateKeys) {
-  Options opts = CurrentOptions();
-  opts.merge_operator = MergeOperators::CreateStringAppendOperator();
-  CreateAndReopenWithCF({"pikachu"}, opts);
-  SetPerfLevel(kEnableCount);
-  // To expand the power of this test, generate > 1 table file and
-  // mix with memtable
-  ASSERT_OK(Merge(1, "k1", "v1"));
-  ASSERT_OK(Merge(1, "k2", "v2"));
-  Flush(1);
-  MoveFilesToLevel(2, 1);
-  ASSERT_OK(Merge(1, "k3", "v3"));
-  ASSERT_OK(Merge(1, "k4", "v4"));
-  Flush(1);
-  MoveFilesToLevel(2, 1);
-  ASSERT_OK(Merge(1, "k4", "v4_2"));
-  ASSERT_OK(Merge(1, "k6", "v6"));
-  Flush(1);
-  MoveFilesToLevel(2, 1);
-  ASSERT_OK(Merge(1, "k7", "v7"));
-  ASSERT_OK(Merge(1, "k8", "v8"));
-  Flush(1);
-  MoveFilesToLevel(2, 1);
-
-  get_perf_context()->Reset();
-
-  std::vector<Slice> keys({"k8", "k8", "k8", "k4", "k4", "k1", "k3"});
-  std::vector<PinnableSlice> values(keys.size());
-  std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);
-  std::vector<Status> s(keys.size());
-
-  db_->MultiGet(ReadOptions(), handles_[1], keys.size(), keys.data(),
-                values.data(), s.data(), false);
-
-  ASSERT_EQ(values.size(), keys.size());
-  ASSERT_EQ(std::string(values[0].data(), values[0].size()), "v8");
-  ASSERT_EQ(std::string(values[1].data(), values[1].size()), "v8");
-  ASSERT_EQ(std::string(values[2].data(), values[2].size()), "v8");
-  ASSERT_EQ(std::string(values[3].data(), values[3].size()), "v4,v4_2");
-  ASSERT_EQ(std::string(values[4].data(), values[4].size()), "v4,v4_2");
-  ASSERT_EQ(std::string(values[5].data(), values[5].size()), "v1");
-  ASSERT_EQ(std::string(values[6].data(), values[6].size()), "v3");
-  ASSERT_EQ(24, (int)get_perf_context()->multiget_read_bytes);
-
-  for (Status& status : s) {
-    ASSERT_OK(status);
-  }
-
-  SetPerfLevel(kDisable);
-}
-
 TEST_F(DBBasicTest, MultiGetBatchedMultiLevel) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
@@ -1555,221 +1501,6 @@ TEST_F(DBBasicTest, MultiGetBatchedMultiLevelMerge) {
   }
 }
 
-TEST_F(DBBasicTest, MultiGetBatchedValueSizeInMemory) {
-  CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-  SetPerfLevel(kEnableCount);
-  ASSERT_OK(Put(1, "k1", "v_1"));
-  ASSERT_OK(Put(1, "k2", "v_2"));
-  ASSERT_OK(Put(1, "k3", "v_3"));
-  ASSERT_OK(Put(1, "k4", "v_4"));
-  ASSERT_OK(Put(1, "k5", "v_5"));
-  ASSERT_OK(Put(1, "k6", "v_6"));
-  std::vector<Slice> keys = {"k1", "k2", "k3", "k4", "k5", "k6"};
-  std::vector<PinnableSlice> values(keys.size());
-  std::vector<Status> s(keys.size());
-  std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);
-
-  get_perf_context()->Reset();
-  ReadOptions ro;
-  ro.value_size_soft_limit = 11;
-  db_->MultiGet(ro, handles_[1], keys.size(), keys.data(), values.data(),
-                s.data(), false);
-
-  ASSERT_EQ(values.size(), keys.size());
-  for (unsigned int i = 0; i < 4; i++) {
-    ASSERT_EQ(std::string(values[i].data(), values[i].size()),
-              "v_" + std::to_string(i + 1));
-  }
-
-  for (unsigned int i = 4; i < 6; i++) {
-    ASSERT_TRUE(s[i].IsAborted());
-  }
-
-  ASSERT_EQ(12, (int)get_perf_context()->multiget_read_bytes);
-  SetPerfLevel(kDisable);
-}
-
-TEST_F(DBBasicTest, MultiGetBatchedValueSize) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    SetPerfLevel(kEnableCount);
-
-    ASSERT_OK(Put(1, "k6", "v6"));
-    ASSERT_OK(Put(1, "k7", "v7_"));
-    ASSERT_OK(Put(1, "k3", "v3_"));
-    ASSERT_OK(Put(1, "k4", "v4"));
-    Flush(1);
-    ASSERT_OK(Delete(1, "k4"));
-    ASSERT_OK(Put(1, "k11", "v11"));
-    ASSERT_OK(Delete(1, "no_key"));
-    ASSERT_OK(Put(1, "k8", "v8_"));
-    ASSERT_OK(Put(1, "k13", "v13"));
-    ASSERT_OK(Put(1, "k14", "v14"));
-    ASSERT_OK(Put(1, "k15", "v15"));
-    ASSERT_OK(Put(1, "k16", "v16"));
-    ASSERT_OK(Put(1, "k17", "v17"));
-    Flush(1);
-
-    ASSERT_OK(Put(1, "k1", "v1_"));
-    ASSERT_OK(Put(1, "k2", "v2_"));
-    ASSERT_OK(Put(1, "k5", "v5_"));
-    ASSERT_OK(Put(1, "k9", "v9_"));
-    ASSERT_OK(Put(1, "k10", "v10"));
-    ASSERT_OK(Delete(1, "k2"));
-    ASSERT_OK(Delete(1, "k6"));
-
-    get_perf_context()->Reset();
-
-    std::vector<Slice> keys({"k1", "k10", "k11", "k12", "k13", "k14", "k15",
-                             "k16", "k17", "k2", "k3", "k4", "k5", "k6", "k7",
-                             "k8", "k9", "no_key"});
-    std::vector<PinnableSlice> values(keys.size());
-    std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);
-    std::vector<Status> s(keys.size());
-
-    ReadOptions ro;
-    ro.value_size_soft_limit = 20;
-    db_->MultiGet(ro, handles_[1], keys.size(), keys.data(), values.data(),
-                  s.data(), false);
-
-    ASSERT_EQ(values.size(), keys.size());
-
-    // In memory keys
-    ASSERT_EQ(std::string(values[0].data(), values[0].size()), "v1_");
-    ASSERT_EQ(std::string(values[1].data(), values[1].size()), "v10");
-    ASSERT_TRUE(s[9].IsNotFound());  // k2
-    ASSERT_EQ(std::string(values[12].data(), values[12].size()), "v5_");
-    ASSERT_TRUE(s[13].IsNotFound());  // k6
-    ASSERT_EQ(std::string(values[16].data(), values[16].size()), "v9_");
-
-    // In sst files
-    ASSERT_EQ(std::string(values[2].data(), values[1].size()), "v11");
-    ASSERT_EQ(std::string(values[4].data(), values[4].size()), "v13");
-    ASSERT_EQ(std::string(values[5].data(), values[5].size()), "v14");
-
-    // Remaining aborted after value_size exceeds.
-    ASSERT_TRUE(s[3].IsAborted());
-    ASSERT_TRUE(s[6].IsAborted());
-    ASSERT_TRUE(s[7].IsAborted());
-    ASSERT_TRUE(s[8].IsAborted());
-    ASSERT_TRUE(s[10].IsAborted());
-    ASSERT_TRUE(s[11].IsAborted());
-    ASSERT_TRUE(s[14].IsAborted());
-    ASSERT_TRUE(s[15].IsAborted());
-    ASSERT_TRUE(s[17].IsAborted());
-
-    // 6 kv pairs * 3 bytes per value (i.e. 18)
-    ASSERT_EQ(21, (int)get_perf_context()->multiget_read_bytes);
-    SetPerfLevel(kDisable);
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, MultiGetBatchedValueSizeMultiLevelMerge) {
-  Options options = CurrentOptions();
-  options.disable_auto_compactions = true;
-  options.merge_operator = MergeOperators::CreateStringAppendOperator();
-  BlockBasedTableOptions bbto;
-  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
-  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-  Reopen(options);
-  int num_keys = 0;
-
-  for (int i = 0; i < 64; ++i) {
-    ASSERT_OK(Put("key_" + std::to_string(i), "val_l2_" + std::to_string(i)));
-    num_keys++;
-    if (num_keys == 8) {
-      Flush();
-      num_keys = 0;
-    }
-  }
-  if (num_keys > 0) {
-    Flush();
-    num_keys = 0;
-  }
-  MoveFilesToLevel(2);
-
-  for (int i = 0; i < 64; i += 3) {
-    ASSERT_OK(Merge("key_" + std::to_string(i), "val_l1_" + std::to_string(i)));
-    num_keys++;
-    if (num_keys == 8) {
-      Flush();
-      num_keys = 0;
-    }
-  }
-  if (num_keys > 0) {
-    Flush();
-    num_keys = 0;
-  }
-  MoveFilesToLevel(1);
-
-  for (int i = 0; i < 64; i += 5) {
-    ASSERT_OK(Merge("key_" + std::to_string(i), "val_l0_" + std::to_string(i)));
-    num_keys++;
-    if (num_keys == 8) {
-      Flush();
-      num_keys = 0;
-    }
-  }
-  if (num_keys > 0) {
-    Flush();
-    num_keys = 0;
-  }
-  ASSERT_EQ(0, num_keys);
-
-  for (int i = 0; i < 64; i += 9) {
-    ASSERT_OK(
-        Merge("key_" + std::to_string(i), "val_mem_" + std::to_string(i)));
-  }
-
-  std::vector<std::string> keys_str;
-  for (int i = 10; i < 50; ++i) {
-    keys_str.push_back("key_" + std::to_string(i));
-  }
-
-  std::vector<Slice> keys(keys_str.size());
-  for (int i = 0; i < 40; i++) {
-    keys[i] = Slice(keys_str[i]);
-  }
-
-  std::vector<PinnableSlice> values(keys_str.size());
-  std::vector<Status> statuses(keys_str.size());
-  ReadOptions read_options;
-  read_options.verify_checksums = true;
-  read_options.value_size_soft_limit = 380;
-  db_->MultiGet(read_options, dbfull()->DefaultColumnFamily(), keys.size(),
-                keys.data(), values.data(), statuses.data());
-
-  ASSERT_EQ(values.size(), keys.size());
-
-  uint64_t curr_value_size = 0;
-  for (unsigned int j = 0; j < 26; ++j) {
-    int key = j + 10;
-    std::string value;
-    value.append("val_l2_" + std::to_string(key));
-    if (key % 3 == 0) {
-      value.append(",");
-      value.append("val_l1_" + std::to_string(key));
-    }
-    if (key % 5 == 0) {
-      value.append(",");
-      value.append("val_l0_" + std::to_string(key));
-    }
-    if (key % 9 == 0) {
-      value.append(",");
-      value.append("val_mem_" + std::to_string(key));
-    }
-    curr_value_size += value.size();
-    ASSERT_EQ(values[j], value);
-    ASSERT_OK(statuses[j]);
-  }
-  // ASSERT_TRUE(curr_value_size <= read_options.value_size_hard_limit);
-
-  // All remaning keys status is set Status::Abort
-  for (unsigned int j = 26; j < 40; j++) {
-    ASSERT_TRUE(statuses[j].IsAborted());
-  }
-}
-
 // Test class for batched MultiGet with prefix extractor
 // Param bool - If true, use partitioned filters
 //              If false, use full filter block
@@ -1954,13 +1685,13 @@ TEST_F(DBBasicTest, GetAllKeyVersions) {
   ASSERT_EQ(kNumInserts + kNumDeletes + kNumUpdates, key_versions.size());
 
   // Check non-default column family
-  for (size_t i = 0; i + 1 != kNumInserts; ++i) {
+  for (size_t i = 0; i != kNumInserts - 1; ++i) {
     ASSERT_OK(Put(1, std::to_string(i), "value"));
   }
-  for (size_t i = 0; i + 1 != kNumUpdates; ++i) {
+  for (size_t i = 0; i != kNumUpdates - 1; ++i) {
     ASSERT_OK(Put(1, std::to_string(i), "value1"));
   }
-  for (size_t i = 0; i + 1 != kNumDeletes; ++i) {
+  for (size_t i = 0; i != kNumDeletes - 1; ++i) {
     ASSERT_OK(Delete(1, std::to_string(i)));
   }
   ASSERT_OK(ROCKSDB_NAMESPACE::GetAllKeyVersions(
@@ -2298,11 +2029,11 @@ TEST_F(DBBasicTest, ManifestChecksumMismatch) {
 class DBBasicTestMultiGet : public DBTestBase {
  public:
   DBBasicTestMultiGet(std::string test_dir, int num_cfs, bool compressed_cache,
-                      bool uncompressed_cache, bool _compression_enabled,
-                      bool _fill_cache, uint32_t compression_parallel_threads)
+                      bool uncompressed_cache, bool compression_enabled,
+                      bool fill_cache, uint32_t compression_parallel_threads)
       : DBTestBase(test_dir) {
-    compression_enabled_ = _compression_enabled;
-    fill_cache_ = _fill_cache;
+    compression_enabled_ = compression_enabled;
+    fill_cache_ = fill_cache;
 
     if (compressed_cache) {
       std::shared_ptr<Cache> cache = NewLRUCache(1048576);
@@ -3023,33 +2754,6 @@ TEST_F(DBBasicTestMultiGetDeadline, MultiGetDeadlineExceeded) {
   Close();
 }
 
-TEST_F(DBBasicTest, ManifestWriteFailure) {
-  Options options = GetDefaultOptions();
-  options.create_if_missing = true;
-  options.disable_auto_compactions = true;
-  options.env = env_;
-  DestroyAndReopen(options);
-  ASSERT_OK(Put("foo", "bar"));
-  ASSERT_OK(Flush());
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-  SyncPoint::GetInstance()->SetCallBack(
-      "VersionSet::ProcessManifestWrites:AfterSyncManifest", [&](void* arg) {
-        ASSERT_NE(nullptr, arg);
-        auto* s = reinterpret_cast<Status*>(arg);
-        ASSERT_OK(*s);
-        // Manually overwrite return status
-        *s = Status::IOError();
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-  ASSERT_OK(Put("key", "value"));
-  ASSERT_NOK(Flush());
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-  SyncPoint::GetInstance()->EnableProcessing();
-  Reopen(options);
-}
-
 }  // namespace ROCKSDB_NAMESPACE
 
 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS

db/db_block_cache_test.cc

@@ -187,7 +187,7 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
   Iterator* iter = nullptr;
 
   // Load blocks into cache.
-  for (size_t i = 0; i + 1 < kNumBlocks; i++) {
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
     iter = db_->NewIterator(read_options);
     iter->Seek(ToString(i));
     ASSERT_OK(iter->status());
@@ -209,12 +209,12 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
   iter = nullptr;
 
   // Release iterators and access cache again.
-  for (size_t i = 0; i + 1 < kNumBlocks; i++) {
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
     iterators[i].reset();
     CheckCacheCounters(options, 0, 0, 0, 0);
   }
   ASSERT_EQ(0, cache->GetPinnedUsage());
-  for (size_t i = 0; i + 1 < kNumBlocks; i++) {
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
     iter = db_->NewIterator(read_options);
     iter->Seek(ToString(i));
     ASSERT_OK(iter->status());
@@ -243,7 +243,7 @@ TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
   Iterator* iter = nullptr;
 
   // Load blocks into cache.
-  for (size_t i = 0; i + 1 < kNumBlocks; i++) {
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
     iter = db_->NewIterator(read_options);
     iter->Seek(ToString(i));
     ASSERT_OK(iter->status());

db/db_bloom_filter_test.cc

@@ -1063,16 +1063,7 @@ TEST_P(DBBloomFilterTestVaryPrefixAndFormatVer, PartitionedMultiGet) {
   bbto.partition_filters = true;
   bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
   bbto.whole_key_filtering = !use_prefix_;
-  if (use_prefix_) {  // (not related to prefix, just alternating between)
-    // Make sure code appropriately deals with metadata block size setting
-    // that is "too small" (smaller than minimum size for filter builder)
-    bbto.metadata_block_size = 63;
-  } else {
-    // Make sure the test will work even on platforms with large minimum
-    // filter size, due to large cache line size.
-    // (Largest cache line size + 10+% overhead.)
-    bbto.metadata_block_size = 290;
-  }
+  bbto.metadata_block_size = 290;
   options.table_factory.reset(NewBlockBasedTableFactory(bbto));
   DestroyAndReopen(options);
   ReadOptions ropts;

db/db_compaction_test.cc

@@ -3184,15 +3184,6 @@ TEST_P(DBCompactionTestWithParam, IntraL0Compaction) {
   options.level0_file_num_compaction_trigger = 5;
   options.max_background_compactions = 2;
   options.max_subcompactions = max_subcompactions_;
-  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
-  options.write_buffer_size = 2 << 20;  // 2MB
-
-  BlockBasedTableOptions table_options;
-  table_options.block_cache = NewLRUCache(64 << 20);  // 64MB
-  table_options.cache_index_and_filter_blocks = true;
-  table_options.pin_l0_filter_and_index_blocks_in_cache = true;
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-
   DestroyAndReopen(options);
 
   const size_t kValueSize = 1 << 20;
@@ -3223,7 +3214,6 @@ TEST_P(DBCompactionTestWithParam, IntraL0Compaction) {
       ASSERT_OK(Put(Key(i + 1), value));
     }
     ASSERT_OK(Flush());
-    ASSERT_EQ(i + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
   }
   dbfull()->TEST_WaitForCompact();
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
@@ -3238,16 +3228,6 @@ TEST_P(DBCompactionTestWithParam, IntraL0Compaction) {
   for (int i = 0; i < 2; ++i) {
     ASSERT_GE(level_to_files[0][i].fd.file_size, 1 << 21);
   }
-
-  // The index/filter in the file produced by intra-L0 should not be pinned.
-  // That means clearing unref'd entries in block cache and re-accessing the
-  // file produced by intra-L0 should bump the index block miss count.
-  uint64_t prev_index_misses =
-      TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS);
-  table_options.block_cache->EraseUnRefEntries();
-  ASSERT_EQ("", Get(Key(0)));
-  ASSERT_EQ(prev_index_misses + 1,
-            TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
 }
 
 TEST_P(DBCompactionTestWithParam, IntraL0CompactionDoesNotObsoleteDeletions) {
@@ -5044,11 +5024,10 @@ TEST_P(DBCompactionTestWithParam, FixFileIngestionCompactionDeadlock) {
 
 TEST_F(DBCompactionTest, ConsistencyFailTest) {
   Options options = CurrentOptions();
-  options.force_consistency_checks = true;
   DestroyAndReopen(options);
 
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "VersionBuilder::CheckConsistency0", [&](void* arg) {
+      "VersionBuilder::CheckConsistency", [&](void* arg) {
         auto p =
             reinterpret_cast<std::pair<FileMetaData**, FileMetaData**>*>(arg);
         // just swap the two FileMetaData so that we hit error
@@ -5067,48 +5046,6 @@ TEST_F(DBCompactionTest, ConsistencyFailTest) {
 
   ASSERT_NOK(Put("foo", "bar"));
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-TEST_F(DBCompactionTest, ConsistencyFailTest2) {
-  Options options = CurrentOptions();
-  options.force_consistency_checks = true;
-  options.target_file_size_base = 1000;
-  options.level0_file_num_compaction_trigger = 2;
-  BlockBasedTableOptions bbto;
-  bbto.block_size = 400;  // small block size
-  options.table_factory.reset(new BlockBasedTableFactory(bbto));
-  DestroyAndReopen(options);
-
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "VersionBuilder::CheckConsistency1", [&](void* arg) {
-        auto p =
-            reinterpret_cast<std::pair<FileMetaData**, FileMetaData**>*>(arg);
-        // just swap the two FileMetaData so that we hit error
-        // in CheckConsistency funcion
-        FileMetaData* temp = *(p->first);
-        *(p->first) = *(p->second);
-        *(p->second) = temp;
-      });
-
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-  std::string value = RandomString(&rnd, 1000);
-
-  ASSERT_OK(Put("foo1", value));
-  ASSERT_OK(Put("z", ""));
-  Flush();
-  ASSERT_OK(Put("foo2", value));
-  ASSERT_OK(Put("z", ""));
-  Flush();
-
-  // This probably returns non-OK, but we rely on the next Put()
-  // to determine the DB is frozen.
-  dbfull()->TEST_WaitForCompact();
-  ASSERT_NOK(Put("foo", "bar"));
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
 void IngestOneKeyValue(DBImpl* db, const std::string& key,

db/db_filesnapshot.cc

@@ -23,6 +23,57 @@
 
 namespace ROCKSDB_NAMESPACE {
 
+Status DBImpl::DisableFileDeletions() {
+  InstrumentedMutexLock l(&mutex_);
+  ++disable_delete_obsolete_files_;
+  if (disable_delete_obsolete_files_ == 1) {
+    ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Disabled");
+  } else {
+    ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                   "File Deletions Disabled, but already disabled. Counter: %d",
+                   disable_delete_obsolete_files_);
+  }
+  return Status::OK();
+}
+
+Status DBImpl::EnableFileDeletions(bool force) {
+  // Job id == 0 means that this is not our background process, but rather
+  // user thread
+  JobContext job_context(0);
+  bool file_deletion_enabled = false;
+  {
+    InstrumentedMutexLock l(&mutex_);
+    if (force) {
+      // if force, we need to enable file deletions right away
+      disable_delete_obsolete_files_ = 0;
+    } else if (disable_delete_obsolete_files_ > 0) {
+      --disable_delete_obsolete_files_;
+    }
+    if (disable_delete_obsolete_files_ == 0)  {
+      file_deletion_enabled = true;
+      FindObsoleteFiles(&job_context, true);
+      bg_cv_.SignalAll();
+    }
+  }
+  if (file_deletion_enabled) {
+    ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Enabled");
+    if (job_context.HaveSomethingToDelete()) {
+      PurgeObsoleteFiles(job_context);
+    }
+  } else {
+    ROCKS_LOG_WARN(immutable_db_options_.info_log,
+                   "File Deletions Enable, but not really enabled. Counter: %d",
+                   disable_delete_obsolete_files_);
+  }
+  job_context.Clean();
+  LogFlush(immutable_db_options_.info_log);
+  return Status::OK();
+}
+
+int DBImpl::IsFileDeletionsEnabled() const {
+  return !disable_delete_obsolete_files_;
+}
+
 Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
                             uint64_t* manifest_file_size,
                             bool flush_memtable) {
@@ -39,9 +90,6 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
       mutex_.Unlock();
       status = AtomicFlushMemTables(cfds, FlushOptions(),
                                     FlushReason::kGetLiveFiles);
-      if (status.IsColumnFamilyDropped()) {
-        status = Status::OK();
-      }
       mutex_.Lock();
     } else {
       for (auto cfd : *versions_->GetColumnFamilySet()) {
@@ -55,10 +103,8 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
         TEST_SYNC_POINT("DBImpl::GetLiveFiles:2");
         mutex_.Lock();
         cfd->UnrefAndTryDelete();
-        if (!status.ok() && !status.IsColumnFamilyDropped()) {
+        if (!status.ok()) {
           break;
-        } else if (status.IsColumnFamilyDropped()) {
-          status = Status::OK();
         }
       }
     }
@@ -72,33 +118,27 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
     }
   }
 
-  // Make a set of all of the live table and blob files
-  std::vector<uint64_t> live_table_files;
-  std::vector<uint64_t> live_blob_files;
+  // Make a set of all of the live *.sst files
+  std::vector<FileDescriptor> live;
   for (auto cfd : *versions_->GetColumnFamilySet()) {
     if (cfd->IsDropped()) {
       continue;
     }
-    cfd->current()->AddLiveFiles(&live_table_files, &live_blob_files);
+    cfd->current()->AddLiveFiles(&live);
   }
 
   ret.clear();
-  ret.reserve(live_table_files.size() + live_blob_files.size() +
-              3);  // for CURRENT + MANIFEST + OPTIONS
+  ret.reserve(live.size() + 3);  // *.sst + CURRENT + MANIFEST + OPTIONS
 
   // create names of the live files. The names are not absolute
   // paths, instead they are relative to dbname_;
-  for (const auto& table_file_number : live_table_files) {
-    ret.emplace_back(MakeTableFileName("", table_file_number));
+  for (const auto& live_file : live) {
+    ret.push_back(MakeTableFileName("", live_file.GetNumber()));
   }
 
-  for (const auto& blob_file_number : live_blob_files) {
-    ret.emplace_back(BlobFileName("", blob_file_number));
-  }
-
-  ret.emplace_back(CurrentFileName(""));
-  ret.emplace_back(DescriptorFileName("", versions_->manifest_file_number()));
-  ret.emplace_back(OptionsFileName("", versions_->options_file_number()));
+  ret.push_back(CurrentFileName(""));
+  ret.push_back(DescriptorFileName("", versions_->manifest_file_number()));
+  ret.push_back(OptionsFileName("", versions_->options_file_number()));
 
   // find length of manifest file while holding the mutex lock
   *manifest_file_size = versions_->manifest_file_size();

db/db_flush_test.cc

@@ -499,13 +499,13 @@ TEST_P(DBAtomicFlushTest, AtomicFlushTriggeredByMemTableFull) {
   TEST_SYNC_POINT(
       "DBAtomicFlushTest::AtomicFlushTriggeredByMemTableFull:BeforeCheck");
   if (options.atomic_flush) {
-    for (size_t i = 0; i + 1 != num_cfs; ++i) {
+    for (size_t i = 0; i != num_cfs - 1; ++i) {
       auto cfh = static_cast<ColumnFamilyHandleImpl*>(handles_[i]);
       ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed());
       ASSERT_TRUE(cfh->cfd()->mem()->IsEmpty());
     }
   } else {
-    for (size_t i = 0; i + 1 != num_cfs; ++i) {
+    for (size_t i = 0; i != num_cfs - 1; ++i) {
       auto cfh = static_cast<ColumnFamilyHandleImpl*>(handles_[i]);
       ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed());
       ASSERT_FALSE(cfh->cfd()->mem()->IsEmpty());

db/db_impl/db_impl.cc

@@ -312,36 +312,8 @@ Status DBImpl::ResumeImpl() {
   }
 
   // Make sure the IO Status stored in version set is set to OK.
-  bool file_deletion_disabled = !IsFileDeletionsEnabled();
   if (s.ok()) {
-    IOStatus io_s = versions_->io_status();
-    if (io_s.IsIOError()) {
-      // If resuming from IOError resulted from MANIFEST write, then assert
-      // that we must have already set the MANIFEST writer to nullptr during
-      // clean-up phase MANIFEST writing. We must have also disabled file
-      // deletions.
-      assert(!versions_->descriptor_log_);
-      assert(file_deletion_disabled);
-      // Since we are trying to recover from MANIFEST write error, we need to
-      // switch to a new MANIFEST anyway. The old MANIFEST can be corrupted.
-      // Therefore, force writing a dummy version edit because we do not know
-      // whether there are flush jobs with non-empty data to flush, triggering
-      // appends to MANIFEST.
-      VersionEdit edit;
-      auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(default_cf_handle_);
-      assert(cfh);
-      ColumnFamilyData* cfd = cfh->cfd();
-      const MutableCFOptions& cf_opts = *cfd->GetLatestMutableCFOptions();
-      s = versions_->LogAndApply(cfd, cf_opts, &edit, &mutex_,
-                                 directories_.GetDbDir());
-      if (!s.ok()) {
-        io_s = versions_->io_status();
-        if (!io_s.ok()) {
-          s = error_handler_.SetBGError(io_s,
-                                        BackgroundErrorReason::kManifestWrite);
-        }
-      }
-    }
+    versions_->SetIOStatusOK();
   }
 
   // We cannot guarantee consistency of the WAL. So force flush Memtables of
@@ -392,13 +364,6 @@ Status DBImpl::ResumeImpl() {
   job_context.Clean();
 
   if (s.ok()) {
-    assert(versions_->io_status().ok());
-    // If we reach here, we should re-enable file deletions if it was disabled
-    // during previous error handling.
-    if (file_deletion_disabled) {
-      // Always return ok
-      EnableFileDeletions(/*force=*/true);
-    }
     ROCKS_LOG_INFO(immutable_db_options_.info_log, "Successfully resumed DB");
   }
   mutex_.Lock();
@@ -1822,7 +1787,6 @@ std::vector<Status> DBImpl::MultiGet(
   // merge_operands will contain the sequence of merges in the latter case.
   size_t num_found = 0;
   size_t keys_read;
-  uint64_t curr_value_size = 0;
   for (keys_read = 0; keys_read < num_keys; ++keys_read) {
     merge_context.Clear();
     Status& s = stat_list[keys_read];
@@ -1866,13 +1830,6 @@ std::vector<Status> DBImpl::MultiGet(
     if (s.ok()) {
       bytes_read += value->size();
       num_found++;
-      curr_value_size += value->size();
-      if (curr_value_size > read_options.value_size_soft_limit) {
-        while (++keys_read < num_keys) {
-          stat_list[keys_read] = Status::Aborted();
-        }
-        break;
-      }
     }
 
     if (read_options.deadline.count() &&
@@ -2127,11 +2084,11 @@ void DBImpl::MultiGet(const ReadOptions& read_options, const size_t num_keys,
     }
   }
   if (!s.ok()) {
-    assert(s.IsTimedOut() || s.IsAborted());
+    assert(s.IsTimedOut());
     for (++cf_iter; cf_iter != multiget_cf_data.end(); ++cf_iter) {
       for (size_t i = cf_iter->start; i < cf_iter->start + cf_iter->num_keys;
            ++i) {
-        *sorted_keys[i]->s = s;
+        *sorted_keys[i]->s = Status::TimedOut();
       }
     }
   }
@@ -2286,7 +2243,7 @@ void DBImpl::MultiGetWithCallback(
   Status s = MultiGetImpl(read_options, 0, num_keys, sorted_keys,
                           multiget_cf_data[0].super_version, consistent_seqnum,
                           nullptr, nullptr);
-  assert(s.ok() || s.IsTimedOut() || s.IsAborted());
+  assert(s.ok() || s.IsTimedOut());
   ReturnAndCleanupSuperVersion(multiget_cf_data[0].cfd,
                                multiget_cf_data[0].super_version);
 }
@@ -2314,7 +2271,6 @@ Status DBImpl::MultiGetImpl(
   // merge_operands will contain the sequence of merges in the latter case.
   size_t keys_left = num_keys;
   Status s;
-  uint64_t curr_value_size = 0;
   while (keys_left) {
     if (read_options.deadline.count() &&
         env_->NowMicros() >
@@ -2329,7 +2285,6 @@ Status DBImpl::MultiGetImpl(
     MultiGetContext ctx(sorted_keys, start_key + num_keys - keys_left,
                         batch_size, snapshot, read_options);
     MultiGetRange range = ctx.GetMultiGetRange();
-    range.AddValueSize(curr_value_size);
     bool lookup_current = false;
 
     keys_left -= batch_size;
@@ -2360,11 +2315,6 @@ Status DBImpl::MultiGetImpl(
       super_version->current->MultiGet(read_options, &range, callback,
                                        is_blob_index);
     }
-    curr_value_size = range.GetValueSize();
-    if (curr_value_size > read_options.value_size_soft_limit) {
-      s = Status::Aborted();
-      break;
-    }
   }
 
   // Post processing (decrement reference counts and record statistics)
@@ -2379,11 +2329,11 @@ Status DBImpl::MultiGetImpl(
     }
   }
   if (keys_left) {
-    assert(s.IsTimedOut() || s.IsAborted());
+    assert(s.IsTimedOut());
     for (size_t i = start_key + num_keys - keys_left; i < start_key + num_keys;
          ++i) {
       KeyContext* key = (*sorted_keys)[i];
-      *key->s = s;
+      *key->s = Status::TimedOut();
     }
   }
 
@@ -4235,8 +4185,7 @@ Status DBImpl::IngestExternalFiles(
         static_cast<ColumnFamilyHandleImpl*>(args[i].column_family)->cfd();
     SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
     exec_results[i].second = ingestion_jobs[i].Prepare(
-        args[i].external_files, args[i].files_checksums,
-        args[i].files_checksum_func_names, start_file_number, super_version);
+        args[i].external_files, start_file_number, super_version);
     exec_results[i].first = true;
     CleanupSuperVersion(super_version);
   }
@@ -4247,8 +4196,7 @@ Status DBImpl::IngestExternalFiles(
         static_cast<ColumnFamilyHandleImpl*>(args[0].column_family)->cfd();
     SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
     exec_results[0].second = ingestion_jobs[0].Prepare(
-        args[0].external_files, args[0].files_checksums,
-        args[0].files_checksum_func_names, next_file_number, super_version);
+        args[0].external_files, next_file_number, super_version);
     exec_results[0].first = true;
     CleanupSuperVersion(super_version);
   }
@@ -4424,14 +4372,6 @@ Status DBImpl::IngestExternalFiles(
 #endif  // !NDEBUG
         }
       }
-    } else if (versions_->io_status().IsIOError()) {
-      // Error while writing to MANIFEST.
-      // In fact, versions_->io_status() can also be the result of renaming
-      // CURRENT file. With current code, it's just difficult to tell. So just
-      // be pessimistic and try write to a new MANIFEST.
-      // TODO: distinguish between MANIFEST write and CURRENT renaming
-      const IOStatus& io_s = versions_->io_status();
-      error_handler_.SetBGError(io_s, BackgroundErrorReason::kManifestWrite);
     }
 
     // Resume writes to the DB

db/db_impl/db_impl.h

@@ -356,12 +356,6 @@ class DBImpl : public DB {
 
   virtual Status Close() override;
 
-  virtual Status DisableFileDeletions() override;
-
-  virtual Status EnableFileDeletions(bool force) override;
-
-  virtual bool IsFileDeletionsEnabled() const;
-
   Status GetStatsHistory(
       uint64_t start_time, uint64_t end_time,
       std::unique_ptr<StatsHistoryIterator>* stats_iterator) override;
@@ -369,6 +363,9 @@ class DBImpl : public DB {
 #ifndef ROCKSDB_LITE
   using DB::ResetStats;
   virtual Status ResetStats() override;
+  virtual Status DisableFileDeletions() override;
+  virtual Status EnableFileDeletions(bool force) override;
+  virtual int IsFileDeletionsEnabled() const;
   // All the returned filenames start with "/"
   virtual Status GetLiveFiles(std::vector<std::string>&,
                               uint64_t* manifest_file_size,
@@ -973,12 +970,6 @@ class DBImpl : public DB {
   void TEST_WaitForPersistStatsRun(std::function<void()> callback) const;
   bool TEST_IsPersistentStatsEnabled() const;
   size_t TEST_EstimateInMemoryStatsHistorySize() const;
-
-  VersionSet* TEST_GetVersionSet() const { return versions_.get(); }
-
-  const std::unordered_set<uint64_t>& TEST_GetFilesGrabbedForPurge() const {
-    return files_grabbed_for_purge_;
-  }
 #endif  // NDEBUG
 
  protected:
@@ -1783,8 +1774,6 @@ class DBImpl : public DB {
       SuperVersion* sv, SequenceNumber snap_seqnum, ReadCallback* callback,
       bool* is_blob_index);
 
-  Status DisableFileDeletionsWithLock();
-
   // table_cache_ provides its own synchronization
   std::shared_ptr<Cache> table_cache_;
 
@@ -1947,7 +1936,7 @@ class DBImpl : public DB {
   std::unordered_map<uint64_t, PurgeFileInfo> purge_files_;
 
   // A vector to store the file numbers that have been assigned to certain
-  // JobContext. Current implementation tracks table and blob files only.
+  // JobContext. Current implementation tracks ssts only.
   std::unordered_set<uint64_t> files_grabbed_for_purge_;
 
   // A queue to store log writers to close

db/db_impl/db_impl_compaction_flush.cc

@@ -210,15 +210,7 @@ Status DBImpl::FlushMemTableToOutputFile(
   if (!s.ok() && !s.IsShutdownInProgress() && !s.IsColumnFamilyDropped()) {
     if (!io_s.ok() && !io_s.IsShutdownInProgress() &&
         !io_s.IsColumnFamilyDropped()) {
-      // Error while writing to MANIFEST.
-      // In fact, versions_->io_status() can also be the result of renaming
-      // CURRENT file. With current code, it's just difficult to tell. So just
-      // be pessimistic and try write to a new MANIFEST.
-      // TODO: distinguish between MANIFEST write and CURRENT renaming
-      auto err_reason = versions_->io_status().ok()
-                            ? BackgroundErrorReason::kFlush
-                            : BackgroundErrorReason::kManifestWrite;
-      error_handler_.SetBGError(io_s, err_reason);
+      error_handler_.SetBGError(io_s, BackgroundErrorReason::kFlush);
     } else {
       Status new_bg_error = s;
       error_handler_.SetBGError(new_bg_error, BackgroundErrorReason::kFlush);
@@ -581,16 +573,8 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
   // Need to undo atomic flush if something went wrong, i.e. s is not OK and
   // it is not because of CF drop.
   if (!s.ok() && !s.IsColumnFamilyDropped()) {
-    if (!io_s.ok() && !io_s.IsColumnFamilyDropped()) {
-      // Error while writing to MANIFEST.
-      // In fact, versions_->io_status() can also be the result of renaming
-      // CURRENT file. With current code, it's just difficult to tell. So just
-      // be pessimistic and try write to a new MANIFEST.
-      // TODO: distinguish between MANIFEST write and CURRENT renaming
-      auto err_reason = versions_->io_status().ok()
-                            ? BackgroundErrorReason::kFlush
-                            : BackgroundErrorReason::kManifestWrite;
-      error_handler_.SetBGError(io_s, err_reason);
+    if (!io_s.ok() && io_s.IsColumnFamilyDropped()) {
+      error_handler_.SetBGError(io_s, BackgroundErrorReason::kFlush);
     } else {
       Status new_bg_error = s;
       error_handler_.SetBGError(new_bg_error, BackgroundErrorReason::kFlush);
@@ -1922,7 +1906,7 @@ Status DBImpl::WaitForFlushMemTables(
       }
     }
     if (1 == num_dropped && 1 == num) {
-      return Status::ColumnFamilyDropped();
+      return Status::InvalidArgument("Cannot flush a dropped CF");
     }
     // Column families involved in this flush request have either been dropped
     // or finished flush. Then it's time to finish waiting.
@@ -2703,6 +2687,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
     for (const auto& f : *c->inputs(0)) {
       c->edit()->DeleteFile(c->level(), f->fd.GetNumber());
     }
+    versions_->SetIOStatusOK();
     status = versions_->LogAndApply(c->column_family_data(),
                                     *c->mutable_cf_options(), c->edit(),
                                     &mutex_, directories_.GetDbDir());
@@ -2760,6 +2745,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
       }
     }
 
+    versions_->SetIOStatusOK();
     status = versions_->LogAndApply(c->column_family_data(),
                                     *c->mutable_cf_options(), c->edit(),
                                     &mutex_, directories_.GetDbDir());
@@ -2891,15 +2877,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
     ROCKS_LOG_WARN(immutable_db_options_.info_log, "Compaction error: %s",
                    status.ToString().c_str());
     if (!io_s.ok()) {
-      // Error while writing to MANIFEST.
-      // In fact, versions_->io_status() can also be the result of renaming
-      // CURRENT file. With current code, it's just difficult to tell. So just
-      // be pessimistic and try write to a new MANIFEST.
-      // TODO: distinguish between MANIFEST write and CURRENT renaming
-      auto err_reason = versions_->io_status().ok()
-                            ? BackgroundErrorReason::kCompaction
-                            : BackgroundErrorReason::kManifestWrite;
-      error_handler_.SetBGError(io_s, err_reason);
+      error_handler_.SetBGError(io_s, BackgroundErrorReason::kCompaction);
     } else {
       error_handler_.SetBGError(status, BackgroundErrorReason::kCompaction);
     }

db/db_impl/db_impl_files.cc

@@ -36,63 +36,7 @@ uint64_t DBImpl::MinObsoleteSstNumberToKeep() {
   return std::numeric_limits<uint64_t>::max();
 }
 
-Status DBImpl::DisableFileDeletions() {
-  InstrumentedMutexLock l(&mutex_);
-  return DisableFileDeletionsWithLock();
-}
-
-Status DBImpl::DisableFileDeletionsWithLock() {
-  mutex_.AssertHeld();
-  ++disable_delete_obsolete_files_;
-  if (disable_delete_obsolete_files_ == 1) {
-    ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Disabled");
-  } else {
-    ROCKS_LOG_WARN(immutable_db_options_.info_log,
-                   "File Deletions Disabled, but already disabled. Counter: %d",
-                   disable_delete_obsolete_files_);
-  }
-  return Status::OK();
-}
-
-Status DBImpl::EnableFileDeletions(bool force) {
-  // Job id == 0 means that this is not our background process, but rather
-  // user thread
-  JobContext job_context(0);
-  bool file_deletion_enabled = false;
-  {
-    InstrumentedMutexLock l(&mutex_);
-    if (force) {
-      // if force, we need to enable file deletions right away
-      disable_delete_obsolete_files_ = 0;
-    } else if (disable_delete_obsolete_files_ > 0) {
-      --disable_delete_obsolete_files_;
-    }
-    if (disable_delete_obsolete_files_ == 0) {
-      file_deletion_enabled = true;
-      FindObsoleteFiles(&job_context, true);
-      bg_cv_.SignalAll();
-    }
-  }
-  if (file_deletion_enabled) {
-    ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Enabled");
-    if (job_context.HaveSomethingToDelete()) {
-      PurgeObsoleteFiles(job_context);
-    }
-  } else {
-    ROCKS_LOG_WARN(immutable_db_options_.info_log,
-                   "File Deletions Enable, but not really enabled. Counter: %d",
-                   disable_delete_obsolete_files_);
-  }
-  job_context.Clean();
-  LogFlush(immutable_db_options_.info_log);
-  return Status::OK();
-}
-
-bool DBImpl::IsFileDeletionsEnabled() const {
-  return 0 == disable_delete_obsolete_files_;
-}
-
-// * Returns the list of live files in 'sst_live' and 'blob_live'.
+// * Returns the list of live files in 'sst_live'
 // If it's doing full scan:
 // * Returns the list of all files in the filesystem in
 // 'full_scan_candidate_files'.
@@ -133,7 +77,12 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
   // Since job_context->min_pending_output is set, until file scan finishes,
   // mutex_ cannot be released. Otherwise, we might see no min_pending_output
   // here but later find newer generated unfinalized files while scanning.
-  job_context->min_pending_output = MinObsoleteSstNumberToKeep();
+  if (!pending_outputs_.empty()) {
+    job_context->min_pending_output = *pending_outputs_.begin();
+  } else {
+    // delete all of them
+    job_context->min_pending_output = std::numeric_limits<uint64_t>::max();
+  }
 
   // Get obsolete files.  This function will also update the list of
   // pending files in VersionSet().
@@ -141,18 +90,13 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
       &job_context->sst_delete_files, &job_context->blob_delete_files,
       &job_context->manifest_delete_files, job_context->min_pending_output);
 
-  // Mark the elements in job_context->sst_delete_files and
-  // job_context->blob_delete_files as "grabbed for purge" so that other threads
-  // calling FindObsoleteFiles with full_scan=true will not add these files to
-  // candidate list for purge.
+  // Mark the elements in job_context->sst_delete_files as grabbedForPurge
+  // so that other threads calling FindObsoleteFiles with full_scan=true
+  // will not add these files to candidate list for purge.
   for (const auto& sst_to_del : job_context->sst_delete_files) {
     MarkAsGrabbedForPurge(sst_to_del.metadata->fd.GetNumber());
   }
 
-  for (const auto& blob_file : job_context->blob_delete_files) {
-    MarkAsGrabbedForPurge(blob_file.GetBlobFileNumber());
-  }
-
   // store the current filenum, lognum, etc
   job_context->manifest_file_number = versions_->manifest_file_number();
   job_context->pending_manifest_file_number =
@@ -160,7 +104,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
   job_context->log_number = MinLogNumberToKeep();
   job_context->prev_log_number = versions_->prev_log_number();
 
-  versions_->AddLiveFiles(&job_context->sst_live, &job_context->blob_live);
+  versions_->AddLiveFiles(&job_context->sst_live);
   if (doing_the_full_scan) {
     InfoLogPrefix info_log_prefix(!immutable_db_options_.db_log_dir.empty(),
                                   dbname_);
@@ -313,11 +257,8 @@ bool CompareCandidateFile(const JobContext::CandidateFileInfo& first,
 void DBImpl::DeleteObsoleteFileImpl(int job_id, const std::string& fname,
                                     const std::string& path_to_sync,
                                     FileType type, uint64_t number) {
-  TEST_SYNC_POINT_CALLBACK("DBImpl::DeleteObsoleteFileImpl::BeforeDeletion",
-                           const_cast<std::string*>(&fname));
-
   Status file_deletion_status;
-  if (type == kTableFile || type == kBlobFile || type == kLogFile) {
+  if (type == kTableFile || type == kLogFile) {
     file_deletion_status =
         DeleteDBFile(&immutable_db_options_, fname, path_to_sync,
                      /*force_bg=*/false, /*force_fg=*/!wal_in_db_path_);
@@ -363,19 +304,19 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
   // FindObsoleteFiles() should've populated this so nonzero
   assert(state.manifest_file_number != 0);
 
-  // Now, convert lists to unordered sets, WITHOUT mutex held; set is slow.
-  std::unordered_set<uint64_t> sst_live_set(state.sst_live.begin(),
-                                            state.sst_live.end());
-  std::unordered_set<uint64_t> blob_live_set(state.blob_live.begin(),
-                                             state.blob_live.end());
+  // Now, convert live list to an unordered map, WITHOUT mutex held;
+  // set is slow.
+  std::unordered_map<uint64_t, const FileDescriptor*> sst_live_map;
+  for (const FileDescriptor& fd : state.sst_live) {
+    sst_live_map[fd.GetNumber()] = &fd;
+  }
   std::unordered_set<uint64_t> log_recycle_files_set(
       state.log_recycle_files.begin(), state.log_recycle_files.end());
 
   auto candidate_files = state.full_scan_candidate_files;
   candidate_files.reserve(
       candidate_files.size() + state.sst_delete_files.size() +
-      state.blob_delete_files.size() + state.log_delete_files.size() +
-      state.manifest_delete_files.size());
+      state.log_delete_files.size() + state.manifest_delete_files.size());
   // We may ignore the dbname when generating the file names.
   for (auto& file : state.sst_delete_files) {
     candidate_files.emplace_back(
@@ -386,11 +327,6 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
     file.DeleteMetadata();
   }
 
-  for (const auto& blob_file : state.blob_delete_files) {
-    candidate_files.emplace_back(BlobFileName(blob_file.GetBlobFileNumber()),
-                                 blob_file.GetPath());
-  }
-
   for (auto file_num : state.log_delete_files) {
     if (file_num > 0) {
       candidate_files.emplace_back(LogFileName(file_num),
@@ -477,19 +413,12 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
       case kTableFile:
         // If the second condition is not there, this makes
         // DontDeletePendingOutputs fail
-        keep = (sst_live_set.find(number) != sst_live_set.end()) ||
+        keep = (sst_live_map.find(number) != sst_live_map.end()) ||
                number >= state.min_pending_output;
         if (!keep) {
           files_to_del.insert(number);
         }
         break;
-      case kBlobFile:
-        keep = number >= state.min_pending_output ||
-               (blob_live_set.find(number) != blob_live_set.end());
-        if (!keep) {
-          files_to_del.insert(number);
-        }
-        break;
       case kTempFile:
         // Any temp files that are currently being written to must
         // be recorded in pending_outputs_, which is inserted into "live".
@@ -499,8 +428,7 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
         //
         // TODO(yhchiang): carefully modify the third condition to safely
         //                 remove the temp options files.
-        keep = (sst_live_set.find(number) != sst_live_set.end()) ||
-               (blob_live_set.find(number) != blob_live_set.end()) ||
+        keep = (sst_live_map.find(number) != sst_live_map.end()) ||
                (number == state.pending_manifest_file_number) ||
                (to_delete.find(kOptionsFileNamePrefix) != std::string::npos);
         break;
@@ -523,6 +451,7 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
       case kDBLockFile:
       case kIdentityFile:
       case kMetaDatabase:
+      case kBlobFile:
         keep = true;
         break;
     }
@@ -538,9 +467,6 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
       TableCache::Evict(table_cache_.get(), number);
       fname = MakeTableFileName(candidate_file.file_path, number);
       dir_to_sync = candidate_file.file_path;
-    } else if (type == kBlobFile) {
-      fname = BlobFileName(candidate_file.file_path, number);
-      dir_to_sync = candidate_file.file_path;
     } else {
       dir_to_sync =
           (type == kLogFile) ? immutable_db_options_.wal_dir : dbname_;

db/db_impl/db_impl_open.cc

@@ -727,10 +727,10 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
     Status* status;  // nullptr if immutable_db_options_.paranoid_checks==false
     void Corruption(size_t bytes, const Status& s) override {
       ROCKS_LOG_WARN(info_log, "%s%s: dropping %d bytes; %s",
-                     (status == nullptr ? "(ignoring error) " : ""), fname,
-                     static_cast<int>(bytes), s.ToString().c_str());
-      if (status != nullptr && status->ok()) {
-        *status = s;
+                     (this->status == nullptr ? "(ignoring error) " : ""),
+                     fname, static_cast<int>(bytes), s.ToString().c_str());
+      if (this->status != nullptr && this->status->ok()) {
+        *this->status = s;
       }
     }
   };
@@ -853,8 +853,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
     Slice record;
     WriteBatch batch;
 
-    TEST_SYNC_POINT_CALLBACK("DBImpl::RecoverLogFiles:BeforeReadWal",
-                             /*arg=*/nullptr);
     while (!stop_replay_by_wal_filter &&
            reader.ReadRecord(&record, &scratch,
                              immutable_db_options_.wal_recovery_mode) &&
@@ -1019,16 +1017,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
         status = Status::OK();
       } else if (immutable_db_options_.wal_recovery_mode ==
                  WALRecoveryMode::kPointInTimeRecovery) {
-        if (status.IsIOError()) {
-          ROCKS_LOG_ERROR(immutable_db_options_.info_log,
-                          "IOError during point-in-time reading log #%" PRIu64
-                          " seq #%" PRIu64
-                          ". %s. This likely mean loss of synced WAL, "
-                          "thus recovery fails.",
-                          log_number, *next_sequence,
-                          status.ToString().c_str());
-          return status;
-        }
         // We should ignore the error but not continue replaying
         status = Status::OK();
         stop_replay_for_corruption = true;
@@ -1604,12 +1592,6 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
   auto sfm = static_cast<SstFileManagerImpl*>(
       impl->immutable_db_options_.sst_file_manager.get());
   if (s.ok() && sfm) {
-    // Set Statistics ptr for SstFileManager to dump the stats of
-    // DeleteScheduler.
-    sfm->SetStatisticsPtr(impl->immutable_db_options_.statistics);
-    ROCKS_LOG_INFO(impl->immutable_db_options_.info_log,
-                   "SstFileManager instance %p", sfm);
-
     // Notify SstFileManager about all sst files that already exist in
     // db_paths[0] and cf_paths[0] when the DB is opened.
 

db/db_impl/db_impl_readonly.cc

@@ -128,38 +128,12 @@ Status DBImplReadOnly::NewIterators(
   return Status::OK();
 }
 
-namespace {
-// Return OK if dbname exists in the file system
-// or create_if_missing is false
-Status OpenForReadOnlyCheckExistence(const DBOptions& db_options,
-                                     const std::string& dbname) {
-  Status s;
-  if (!db_options.create_if_missing) {
-    // Attempt to read "CURRENT" file
-    const std::shared_ptr<FileSystem>& fs = db_options.env->GetFileSystem();
-    std::string manifest_path;
-    uint64_t manifest_file_number;
-    s = VersionSet::GetCurrentManifestPath(dbname, fs.get(), &manifest_path,
-                                           &manifest_file_number);
-    if (!s.ok()) {
-      return Status::NotFound(CurrentFileName(dbname), "does not exist");
-    }
-  }
-  return s;
-}
-}  // namespace
-
 Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
                            DB** dbptr, bool /*error_if_log_file_exist*/) {
-  // If dbname does not exist in the file system, should not do anything
-  Status s = OpenForReadOnlyCheckExistence(options, dbname);
-  if (!s.ok()) {
-    return s;
-  }
-
   *dbptr = nullptr;
 
   // Try to first open DB as fully compacted DB
+  Status s;
   s = CompactedDBImpl::Open(options, dbname, dbptr);
   if (s.ok()) {
     return s;
@@ -172,8 +146,7 @@ Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
       ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options));
   std::vector<ColumnFamilyHandle*> handles;
 
-  s = DBImplReadOnly::OpenForReadOnlyWithoutCheck(
-      db_options, dbname, column_families, &handles, dbptr);
+  s = DB::OpenForReadOnly(db_options, dbname, column_families, &handles, dbptr);
   if (s.ok()) {
     assert(handles.size() == 1);
     // i can delete the handle since DBImpl is always holding a
@@ -188,22 +161,6 @@ Status DB::OpenForReadOnly(
     const std::vector<ColumnFamilyDescriptor>& column_families,
     std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
     bool error_if_log_file_exist) {
-  // If dbname does not exist in the file system, should not do anything
-  Status s = OpenForReadOnlyCheckExistence(db_options, dbname);
-  if (!s.ok()) {
-    return s;
-  }
-
-  return DBImplReadOnly::OpenForReadOnlyWithoutCheck(
-      db_options, dbname, column_families, handles, dbptr,
-      error_if_log_file_exist);
-}
-
-Status DBImplReadOnly::OpenForReadOnlyWithoutCheck(
-    const DBOptions& db_options, const std::string& dbname,
-    const std::vector<ColumnFamilyDescriptor>& column_families,
-    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
-    bool error_if_log_file_exist) {
   *dbptr = nullptr;
   handles->clear();
 

db/db_impl/db_impl_readonly.h

@@ -130,15 +130,6 @@ class DBImplReadOnly : public DBImpl {
   }
 
  private:
-  // A "helper" function for DB::OpenForReadOnly without column families
-  // to reduce unnecessary I/O
-  // It has the same functionality as DB::OpenForReadOnly with column families
-  // but does not check the existence of dbname in the file system
-  static Status OpenForReadOnlyWithoutCheck(
-      const DBOptions& db_options, const std::string& dbname,
-      const std::vector<ColumnFamilyDescriptor>& column_families,
-      std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
-      bool error_if_log_file_exist = false);
   friend class DB;
 };
 }  // namespace ROCKSDB_NAMESPACE

db/db_impl/db_impl_write.cc

@@ -1837,30 +1837,8 @@ Status DB::Put(const WriteOptions& opt, ColumnFamilyHandle* column_family,
 
 Status DB::Delete(const WriteOptions& opt, ColumnFamilyHandle* column_family,
                   const Slice& key) {
-  if (nullptr == opt.timestamp) {
-    WriteBatch batch;
-    Status s = batch.Delete(column_family, key);
-    if (!s.ok()) {
-      return s;
-    }
-    return Write(opt, &batch);
-  }
-  const Slice* ts = opt.timestamp;
-  assert(ts != nullptr);
-  const size_t ts_sz = ts->size();
-  constexpr size_t kKeyAndValueLenSize = 11;
-  constexpr size_t kWriteBatchOverhead =
-      WriteBatchInternal::kHeader + sizeof(ValueType) + kKeyAndValueLenSize;
-  WriteBatch batch(key.size() + ts_sz + kWriteBatchOverhead, /*max_bytes=*/0,
-                   ts_sz);
-  Status s = batch.Delete(column_family, key);
-  if (!s.ok()) {
-    return s;
-  }
-  s = batch.AssignTimestamp(*ts);
-  if (!s.ok()) {
-    return s;
-  }
+  WriteBatch batch;
+  batch.Delete(column_family, key);
   return Write(opt, &batch);
 }
 

db/db_iter.cc

@@ -258,7 +258,7 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
       // prone to bugs causing the same user key with the same sequence number.
       // Note that with current timestamp implementation, the same user key can
       // have different timestamps and zero sequence number on the bottommost
-      // level. This may change in the future.
+      // level. This will change in the future.
       if ((!is_prev_key_seqnum_zero || timestamp_size_ > 0) &&
           skipping_saved_key &&
           CompareKeyForSkip(ikey_.user_key, saved_key_.GetUserKey()) <= 0) {
@@ -276,7 +276,6 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
         reseek_done = false;
         switch (ikey_.type) {
           case kTypeDeletion:
-          case kTypeDeletionWithTimestamp:
           case kTypeSingleDeletion:
             // Arrange to skip all upcoming entries for this key since
             // they are hidden by this deletion.

db/db_range_del_test.cc

@@ -47,21 +47,6 @@ TEST_F(DBRangeDelTest, WriteBatchWithIndexNotSupported) {
   ASSERT_TRUE(indexedBatch.DeleteRange("dr1", "dr1").IsNotSupported());
 }
 
-TEST_F(DBRangeDelTest, EndSameAsStartCoversNothing) {
-  ASSERT_OK(db_->Put(WriteOptions(), "b", "val"));
-  ASSERT_OK(
-      db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "b", "b"));
-  ASSERT_EQ("val", Get("b"));
-}
-
-TEST_F(DBRangeDelTest, EndComesBeforeStartInvalidArgument) {
-  db_->Put(WriteOptions(), "b", "val");
-  ASSERT_TRUE(
-      db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "b", "a")
-          .IsInvalidArgument());
-  ASSERT_EQ("val", Get("b"));
-}
-
 TEST_F(DBRangeDelTest, FlushOutputHasOnlyRangeTombstones) {
   do {
     DestroyAndReopen(CurrentOptions());
@@ -189,7 +174,7 @@ TEST_F(DBRangeDelTest, MaxCompactionBytesCutsOutputFiles) {
   std::vector<std::vector<FileMetaData>> files;
   dbfull()->TEST_GetFilesMetaData(db_->DefaultColumnFamily(), &files);
 
-  for (size_t i = 0; i + 1 < files[1].size(); ++i) {
+  for (size_t i = 0; i < files[1].size() - 1; ++i) {
     ASSERT_TRUE(InternalKeyComparator(opts.comparator)
                     .Compare(files[1][i].largest, files[1][i + 1].smallest) <
                 0);

db/db_sst_test.cc

@@ -376,7 +376,6 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
   env_->SetTimeElapseOnlySleep(&options);
   options.disable_auto_compactions = true;
   options.env = env_;
-  options.statistics = CreateDBStatistics();
 
   int64_t rate_bytes_per_sec = 1024 * 10;  // 10 Kbs / Sec
   Status s;
@@ -426,9 +425,6 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
   }
   ASSERT_GT(time_spent_deleting, expected_penlty * 0.9);
   ASSERT_LT(time_spent_deleting, expected_penlty * 1.1);
-  ASSERT_EQ(4, options.statistics->getAndResetTickerCount(FILES_MARKED_TRASH));
-  ASSERT_EQ(
-      0, options.statistics->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
 
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
@@ -820,12 +816,6 @@ TEST_F(DBSSTTest, CancellingCompactionsWorks) {
   ASSERT_EQ(sfm->GetCompactionsReservedSize(), 0);
   // Make sure the stat is bumped
   ASSERT_GT(dbfull()->immutable_db_options().statistics.get()->getTickerCount(COMPACTION_CANCELLED), 0);
-  ASSERT_EQ(0,
-            dbfull()->immutable_db_options().statistics.get()->getTickerCount(
-                FILES_MARKED_TRASH));
-  ASSERT_EQ(4,
-            dbfull()->immutable_db_options().statistics.get()->getTickerCount(
-                FILES_DELETED_IMMEDIATELY));
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
 

db/db_test.cc

@@ -1489,24 +1489,18 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
 }
 
 TEST_F(DBTest, ApproximateSizesFilesWithErrorMargin) {
-  // Roughly 4 keys per data block, 1000 keys per file,
-  // with filter substantially larger than a data block
-  BlockBasedTableOptions table_options;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(16));
-  table_options.block_size = 100;
   Options options = CurrentOptions();
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  options.write_buffer_size = 24 * 1024;
+  options.write_buffer_size = 1024 * 1024;
   options.compression = kNoCompression;
   options.create_if_missing = true;
-  options.target_file_size_base = 24 * 1024;
+  options.target_file_size_base = 1024 * 1024;
   DestroyAndReopen(options);
   const auto default_cf = db_->DefaultColumnFamily();
 
   const int N = 64000;
   Random rnd(301);
   for (int i = 0; i < N; i++) {
-    ASSERT_OK(Put(Key(i), RandomString(&rnd, 24)));
+    ASSERT_OK(Put(Key(i), RandomString(&rnd, 1024)));
   }
   // Flush everything to files
   Flush();
@@ -1515,7 +1509,7 @@ TEST_F(DBTest, ApproximateSizesFilesWithErrorMargin) {
 
   // Write more keys
   for (int i = N; i < (N + N / 4); i++) {
-    ASSERT_OK(Put(Key(i), RandomString(&rnd, 24)));
+    ASSERT_OK(Put(Key(i), RandomString(&rnd, 1024)));
   }
   // Flush everything to files again
   Flush();
@@ -1523,45 +1517,27 @@ TEST_F(DBTest, ApproximateSizesFilesWithErrorMargin) {
   // Wait for compaction to finish
   ASSERT_OK(dbfull()->TEST_WaitForCompact());
 
-  {
-    const std::string start = Key(0);
-    const std::string end = Key(2 * N);
-    const Range r(start, end);
+  const std::string start = Key(0);
+  const std::string end = Key(2 * N);
+  const Range r(start, end);
 
-    SizeApproximationOptions size_approx_options;
-    size_approx_options.include_memtabtles = false;
-    size_approx_options.include_files = true;
-    size_approx_options.files_size_error_margin = -1.0;  // disabled
+  SizeApproximationOptions size_approx_options;
+  size_approx_options.include_memtabtles = false;
+  size_approx_options.include_files = true;
+  size_approx_options.files_size_error_margin = -1.0;  // disabled
 
-    // Get the precise size without any approximation heuristic
-    uint64_t size;
-    db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size);
-    ASSERT_NE(size, 0);
+  // Get the precise size without any approximation heuristic
+  uint64_t size;
+  db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size);
+  ASSERT_NE(size, 0);
 
-    // Get the size with an approximation heuristic
-    uint64_t size2;
-    const double error_margin = 0.2;
-    size_approx_options.files_size_error_margin = error_margin;
-    db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size2);
-    ASSERT_LT(size2, size * (1 + error_margin));
-    ASSERT_GT(size2, size * (1 - error_margin));
-  }
-
-  {
-    // Ensure that metadata is not falsely attributed only to the last data in
-    // the file. (In some applications, filters can be large portion of data
-    // size.)
-    // Perform many queries over small range, enough to ensure crossing file
-    // boundary, and make sure we never see a spike for large filter.
-    for (int i = 0; i < 3000; i += 10) {
-      const std::string start = Key(i);
-      const std::string end = Key(i + 11);  // overlap by 1 key
-      const Range r(start, end);
-      uint64_t size;
-      db_->GetApproximateSizes(&r, 1, &size);
-      ASSERT_LE(size, 11 * 100);
-    }
-  }
+  // Get the size with an approximation heuristic
+  uint64_t size2;
+  const double error_margin = 0.2;
+  size_approx_options.files_size_error_margin = error_margin;
+  db_->GetApproximateSizes(size_approx_options, default_cf, &r, 1, &size2);
+  ASSERT_LT(size2, size * (1 + error_margin));
+  ASSERT_GT(size2, size * (1 - error_margin));
 }
 
 TEST_F(DBTest, GetApproximateMemTableStats) {
@@ -1699,13 +1675,12 @@ TEST_F(DBTest, ApproximateSizes_MixOfSmallAndLarge) {
       ASSERT_TRUE(Between(Size("", Key(2), 1), 20000, 21000));
       ASSERT_TRUE(Between(Size("", Key(3), 1), 120000, 121000));
       ASSERT_TRUE(Between(Size("", Key(4), 1), 130000, 131000));
-      ASSERT_TRUE(Between(Size("", Key(5), 1), 230000, 232000));
-      ASSERT_TRUE(Between(Size("", Key(6), 1), 240000, 242000));
-      // Ensure some overhead is accounted for, even without including all
-      ASSERT_TRUE(Between(Size("", Key(7), 1), 540500, 545000));
-      ASSERT_TRUE(Between(Size("", Key(8), 1), 550500, 555000));
+      ASSERT_TRUE(Between(Size("", Key(5), 1), 230000, 231000));
+      ASSERT_TRUE(Between(Size("", Key(6), 1), 240000, 241000));
+      ASSERT_TRUE(Between(Size("", Key(7), 1), 540000, 541000));
+      ASSERT_TRUE(Between(Size("", Key(8), 1), 550000, 560000));
 
-      ASSERT_TRUE(Between(Size(Key(3), Key(5), 1), 110100, 111000));
+      ASSERT_TRUE(Between(Size(Key(3), Key(5), 1), 110000, 111000));
 
       dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
     }
@@ -2334,42 +2309,6 @@ TEST_F(DBTest, ReadonlyDBGetLiveManifestSize) {
     Close();
   } while (ChangeCompactOptions());
 }
-
-TEST_F(DBTest, GetLiveBlobFiles) {
-  VersionSet* const versions = dbfull()->TEST_GetVersionSet();
-  assert(versions);
-  assert(versions->GetColumnFamilySet());
-
-  ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault();
-  assert(cfd);
-
-  // Add a live blob file.
-  VersionEdit edit;
-
-  constexpr uint64_t blob_file_number = 234;
-  constexpr uint64_t total_blob_count = 555;
-  constexpr uint64_t total_blob_bytes = 66666;
-  constexpr char checksum_method[] = "CRC32";
-  constexpr char checksum_value[] = "3d87ff57";
-
-  edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
-                   checksum_method, checksum_value);
-
-  dbfull()->TEST_LockMutex();
-  Status s = versions->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
-                                   &edit, dbfull()->mutex());
-  dbfull()->TEST_UnlockMutex();
-
-  ASSERT_OK(s);
-
-  // Make sure it appears in the results returned by GetLiveFiles.
-  uint64_t manifest_size = 0;
-  std::vector<std::string> files;
-  ASSERT_OK(dbfull()->GetLiveFiles(files, &manifest_size));
-
-  ASSERT_FALSE(files.empty());
-  ASSERT_EQ(files[0], BlobFileName("", blob_file_number));
-}
 #endif
 
 TEST_F(DBTest, PurgeInfoLogs) {
@@ -2990,11 +2929,10 @@ class ModelDB : public DB {
 
   Status SyncWAL() override { return Status::OK(); }
 
+#ifndef ROCKSDB_LITE
   Status DisableFileDeletions() override { return Status::OK(); }
 
   Status EnableFileDeletions(bool /*force*/) override { return Status::OK(); }
-#ifndef ROCKSDB_LITE
-
   Status GetLiveFiles(std::vector<std::string>&, uint64_t* /*size*/,
                       bool /*flush_memtable*/ = true) override {
     return Status::OK();

db/db_test2.cc

@@ -25,159 +25,6 @@ class DBTest2 : public DBTestBase {
   DBTest2() : DBTestBase("/db_test2") {}
 };
 
-#ifndef ROCKSDB_LITE
-TEST_F(DBTest2, OpenForReadOnly) {
-  DB* db_ptr = nullptr;
-  std::string dbname = test::PerThreadDBPath("db_readonly");
-  Options options = CurrentOptions();
-  options.create_if_missing = true;
-  // OpenForReadOnly should fail but will create <dbname> in the file system
-  ASSERT_NOK(DB::OpenForReadOnly(options, dbname, &db_ptr));
-  // Since <dbname> is created, we should be able to delete the dir
-  // We first get the list files under <dbname>
-  // There should not be any subdirectories -- this is not checked here
-  std::vector<std::string> files;
-  ASSERT_OK(env_->GetChildren(dbname, &files));
-  for (auto& f : files) {
-    if (f != "." && f != "..") {
-      ASSERT_OK(env_->DeleteFile(dbname + "/" + f));
-    }
-  }
-  // <dbname> should be empty now and we should be able to delete it
-  ASSERT_OK(env_->DeleteDir(dbname));
-  options.create_if_missing = false;
-  // OpenForReadOnly should fail since <dbname> was successfully deleted
-  ASSERT_NOK(DB::OpenForReadOnly(options, dbname, &db_ptr));
-  // With create_if_missing false, there should not be a dir in the file system
-  ASSERT_NOK(env_->FileExists(dbname));
-}
-
-TEST_F(DBTest2, OpenForReadOnlyWithColumnFamilies) {
-  DB* db_ptr = nullptr;
-  std::string dbname = test::PerThreadDBPath("db_readonly");
-  Options options = CurrentOptions();
-  options.create_if_missing = true;
-
-  ColumnFamilyOptions cf_options(options);
-  std::vector<ColumnFamilyDescriptor> column_families;
-  column_families.push_back(
-      ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options));
-  column_families.push_back(ColumnFamilyDescriptor("goku", cf_options));
-  std::vector<ColumnFamilyHandle*> handles;
-  // OpenForReadOnly should fail but will create <dbname> in the file system
-  ASSERT_NOK(
-      DB::OpenForReadOnly(options, dbname, column_families, &handles, &db_ptr));
-  // Since <dbname> is created, we should be able to delete the dir
-  // We first get the list files under <dbname>
-  // There should not be any subdirectories -- this is not checked here
-  std::vector<std::string> files;
-  ASSERT_OK(env_->GetChildren(dbname, &files));
-  for (auto& f : files) {
-    if (f != "." && f != "..") {
-      ASSERT_OK(env_->DeleteFile(dbname + "/" + f));
-    }
-  }
-  // <dbname> should be empty now and we should be able to delete it
-  ASSERT_OK(env_->DeleteDir(dbname));
-  options.create_if_missing = false;
-  // OpenForReadOnly should fail since <dbname> was successfully deleted
-  ASSERT_NOK(
-      DB::OpenForReadOnly(options, dbname, column_families, &handles, &db_ptr));
-  // With create_if_missing false, there should not be a dir in the file system
-  ASSERT_NOK(env_->FileExists(dbname));
-}
-
-class TestReadOnlyWithCompressedCache
-    : public DBTestBase,
-      public testing::WithParamInterface<std::tuple<int, bool>> {
- public:
-  TestReadOnlyWithCompressedCache()
-      : DBTestBase("/test_readonly_with_compressed_cache") {
-    max_open_files_ = std::get<0>(GetParam());
-    use_mmap_ = std::get<1>(GetParam());
-  }
-  int max_open_files_;
-  bool use_mmap_;
-};
-
-TEST_P(TestReadOnlyWithCompressedCache, ReadOnlyWithCompressedCache) {
-  if (use_mmap_ && !IsMemoryMappedAccessSupported()) {
-    return;
-  }
-  ASSERT_OK(Put("foo", "bar"));
-  ASSERT_OK(Put("foo2", "barbarbarbarbarbarbarbar"));
-  ASSERT_OK(Flush());
-
-  DB* db_ptr = nullptr;
-  Options options = CurrentOptions();
-  options.allow_mmap_reads = use_mmap_;
-  options.max_open_files = max_open_files_;
-  options.compression = kSnappyCompression;
-  BlockBasedTableOptions table_options;
-  table_options.block_cache_compressed = NewLRUCache(8 * 1024 * 1024);
-  table_options.no_block_cache = true;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  options.statistics = CreateDBStatistics();
-
-  ASSERT_OK(DB::OpenForReadOnly(options, dbname_, &db_ptr));
-
-  std::string v;
-  ASSERT_OK(db_ptr->Get(ReadOptions(), "foo", &v));
-  ASSERT_EQ("bar", v);
-  ASSERT_EQ(0, options.statistics->getTickerCount(BLOCK_CACHE_COMPRESSED_HIT));
-  ASSERT_OK(db_ptr->Get(ReadOptions(), "foo", &v));
-  ASSERT_EQ("bar", v);
-  if (Snappy_Supported()) {
-    if (use_mmap_) {
-      ASSERT_EQ(0,
-                options.statistics->getTickerCount(BLOCK_CACHE_COMPRESSED_HIT));
-    } else {
-      ASSERT_EQ(1,
-                options.statistics->getTickerCount(BLOCK_CACHE_COMPRESSED_HIT));
-    }
-  }
-
-  delete db_ptr;
-}
-
-INSTANTIATE_TEST_CASE_P(TestReadOnlyWithCompressedCache,
-                        TestReadOnlyWithCompressedCache,
-                        ::testing::Combine(::testing::Values(-1, 100),
-                                           ::testing::Bool()));
-
-class PartitionedIndexTestListener : public EventListener {
- public:
-  void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
-    ASSERT_GT(info.table_properties.index_partitions, 1);
-    ASSERT_EQ(info.table_properties.index_key_is_user_key, 0);
-  }
-};
-
-TEST_F(DBTest2, PartitionedIndexUserToInternalKey) {
-  BlockBasedTableOptions table_options;
-  Options options = CurrentOptions();
-  table_options.index_type = BlockBasedTableOptions::kTwoLevelIndexSearch;
-  PartitionedIndexTestListener* listener = new PartitionedIndexTestListener();
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  options.listeners.emplace_back(listener);
-  std::vector<const Snapshot*> snapshots;
-  Reopen(options);
-  Random rnd(301);
-
-  for (int i = 0; i < 3000; i++) {
-    int j = i % 30;
-    std::string value = RandomString(&rnd, 10500);
-    ASSERT_OK(Put("keykey_" + std::to_string(j), value));
-    snapshots.push_back(db_->GetSnapshot());
-  }
-  Flush();
-  for (auto s : snapshots) {
-    db_->ReleaseSnapshot(s);
-  }
-}
-
-#endif  // ROCKSDB_LITE
-
 class PrefixFullBloomWithReverseComparator
     : public DBTestBase,
       public ::testing::WithParamInterface<bool> {
@@ -1421,152 +1268,6 @@ class CompactionCompressionListener : public EventListener {
   const Options* db_options_;
 };
 
-enum CompressionFailureType {
-  kTestCompressionFail,
-  kTestDecompressionFail,
-  kTestDecompressionCorruption
-};
-
-class CompressionFailuresTest
-    : public DBTest2,
-      public testing::WithParamInterface<std::tuple<
-          CompressionFailureType, CompressionType, uint32_t, uint32_t>> {
- public:
-  CompressionFailuresTest() {
-    std::tie(compression_failure_type_, compression_type_,
-             compression_max_dict_bytes_, compression_parallel_threads_) =
-        GetParam();
-  }
-
-  CompressionFailureType compression_failure_type_ = kTestCompressionFail;
-  CompressionType compression_type_ = kNoCompression;
-  uint32_t compression_max_dict_bytes_ = 0;
-  uint32_t compression_parallel_threads_ = 0;
-};
-
-INSTANTIATE_TEST_CASE_P(
-    DBTest2, CompressionFailuresTest,
-    ::testing::Combine(::testing::Values(kTestCompressionFail,
-                                         kTestDecompressionFail,
-                                         kTestDecompressionCorruption),
-                       ::testing::ValuesIn(GetSupportedCompressions()),
-                       ::testing::Values(0, 10), ::testing::Values(1, 4)));
-
-TEST_P(CompressionFailuresTest, CompressionFailures) {
-  if (compression_type_ == kNoCompression) {
-    return;
-  }
-
-  Options options = CurrentOptions();
-  options.level0_file_num_compaction_trigger = 2;
-  options.max_bytes_for_level_base = 1024;
-  options.max_bytes_for_level_multiplier = 2;
-  options.num_levels = 7;
-  options.max_background_compactions = 1;
-  options.target_file_size_base = 512;
-
-  BlockBasedTableOptions table_options;
-  table_options.block_size = 512;
-  table_options.verify_compression = true;
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-
-  options.compression = compression_type_;
-  options.compression_opts.parallel_threads = compression_parallel_threads_;
-  options.compression_opts.max_dict_bytes = compression_max_dict_bytes_;
-  options.bottommost_compression_opts.parallel_threads =
-      compression_parallel_threads_;
-  options.bottommost_compression_opts.max_dict_bytes =
-      compression_max_dict_bytes_;
-
-  if (compression_failure_type_ == kTestCompressionFail) {
-    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-        "BlockBasedTableBuilder::CompressBlockInternal:TamperWithReturnValue",
-        [](void* arg) {
-          bool* ret = static_cast<bool*>(arg);
-          *ret = false;
-        });
-  } else if (compression_failure_type_ == kTestDecompressionFail) {
-    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-        "UncompressBlockContentsForCompressionType:TamperWithReturnValue",
-        [](void* arg) {
-          Status* ret = static_cast<Status*>(arg);
-          ASSERT_OK(*ret);
-          *ret = Status::Corruption("kTestDecompressionFail");
-        });
-  } else if (compression_failure_type_ == kTestDecompressionCorruption) {
-    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-        "UncompressBlockContentsForCompressionType:"
-        "TamperWithDecompressionOutput",
-        [](void* arg) {
-          BlockContents* contents = static_cast<BlockContents*>(arg);
-          // Ensure uncompressed data != original data
-          const size_t len = contents->data.size() + 1;
-          std::unique_ptr<char[]> fake_data(new char[len]());
-          *contents = BlockContents(std::move(fake_data), len);
-        });
-  }
-
-  std::map<std::string, std::string> key_value_written;
-
-  const int kKeySize = 5;
-  const int kValUnitSize = 16;
-  const int kValSize = 256;
-  Random rnd(405);
-
-  Status s = Status::OK();
-
-  DestroyAndReopen(options);
-  // Write 10 random files
-  for (int i = 0; i < 10; i++) {
-    for (int j = 0; j < 5; j++) {
-      std::string key = RandomString(&rnd, kKeySize);
-      // Ensure good compression ratio
-      std::string valueUnit = RandomString(&rnd, kValUnitSize);
-      std::string value;
-      for (int k = 0; k < kValSize; k += kValUnitSize) {
-        value += valueUnit;
-      }
-      s = Put(key, value);
-      if (compression_failure_type_ == kTestCompressionFail) {
-        key_value_written[key] = value;
-        ASSERT_OK(s);
-      }
-    }
-    s = Flush();
-    if (compression_failure_type_ == kTestCompressionFail) {
-      ASSERT_OK(s);
-    }
-    s = dbfull()->TEST_WaitForCompact();
-    if (compression_failure_type_ == kTestCompressionFail) {
-      ASSERT_OK(s);
-    }
-    if (i == 4) {
-      // Make compression fail at the mid of table building
-      ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
-    }
-  }
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-
-  if (compression_failure_type_ == kTestCompressionFail) {
-    // Should be kNoCompression, check content consistency
-    std::unique_ptr<Iterator> db_iter(db_->NewIterator(ReadOptions()));
-    for (db_iter->SeekToFirst(); db_iter->Valid(); db_iter->Next()) {
-      std::string key = db_iter->key().ToString();
-      std::string value = db_iter->value().ToString();
-      ASSERT_NE(key_value_written.find(key), key_value_written.end());
-      ASSERT_EQ(key_value_written[key], value);
-      key_value_written.erase(key);
-    }
-    ASSERT_EQ(0, key_value_written.size());
-  } else if (compression_failure_type_ == kTestDecompressionFail) {
-    ASSERT_EQ(std::string(s.getState()),
-              "Could not decompress: kTestDecompressionFail");
-  } else if (compression_failure_type_ == kTestDecompressionCorruption) {
-    ASSERT_EQ(std::string(s.getState()),
-              "Decompressed block did not match raw block");
-  }
-}
-
 TEST_F(DBTest2, CompressionOptions) {
   if (!Zlib_Supported() || !Snappy_Supported()) {
     return;
@@ -5104,32 +4805,6 @@ TEST_F(DBTest2, AutoPrefixMode1) {
   }
 }
 #endif  // ROCKSDB_LITE
-
-// WAL recovery mode is WALRecoveryMode::kPointInTimeRecovery.
-TEST_F(DBTest2, PointInTimeRecoveryWithIOErrorWhileReadingWal) {
-  Options options = CurrentOptions();
-  DestroyAndReopen(options);
-  ASSERT_OK(Put("foo", "value0"));
-  Close();
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-  bool should_inject_error = false;
-  SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::RecoverLogFiles:BeforeReadWal",
-      [&](void* /*arg*/) { should_inject_error = true; });
-  SyncPoint::GetInstance()->SetCallBack(
-      "LogReader::ReadMore:AfterReadFile", [&](void* arg) {
-        if (should_inject_error) {
-          ASSERT_NE(nullptr, arg);
-          *reinterpret_cast<Status*>(arg) = Status::IOError("Injected IOError");
-        }
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-  options.avoid_flush_during_recovery = true;
-  options.wal_recovery_mode = WALRecoveryMode::kPointInTimeRecovery;
-  Status s = TryReopen(options);
-  ASSERT_TRUE(s.IsIOError());
-}
 }  // namespace ROCKSDB_NAMESPACE
 
 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS

db/db_wal_test.cc

@@ -1422,11 +1422,8 @@ TEST_F(DBWALTest, RecoverFromCorruptedWALWithoutFlush) {
     delete iter;
     return data;
   };
-  for (auto& mode : {WALRecoveryMode::kTolerateCorruptedTailRecords,
-                     WALRecoveryMode::kAbsoluteConsistency,
-                     WALRecoveryMode::kPointInTimeRecovery,
-                     WALRecoveryMode::kSkipAnyCorruptedRecords}) {
-    options.wal_recovery_mode = mode;
+  for (auto& mode : wal_recovery_mode_string_map) {
+    options.wal_recovery_mode = mode.second;
     for (auto trunc : {true, false}) {
       for (int i = 0; i < 4; i++) {
         for (int j = jstart; j < jend; j++) {

db/db_with_timestamp_basic_test.cc

@@ -449,65 +449,6 @@ TEST_F(DBBasicTestWithTimestamp, MaxKeysSkipped) {
   Close();
 }
 
-// Create two L0, and compact them to a new L1. In this test, L1 is L_bottom.
-// Two L0s:
-//       f1                                  f2
-// <a, 1, kTypeValue>    <a, 3, kTypeDeletionWithTimestamp>...<b, 2, kTypeValue>
-// Since f2.smallest < f1.largest < f2.largest
-// f1 and f2 will be the inputs of a real compaction instead of trivial move.
-TEST_F(DBBasicTestWithTimestamp, CompactDeletionWithTimestampMarkerToBottom) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  const size_t kTimestampSize = Timestamp(0, 0).size();
-  TestComparator test_cmp(kTimestampSize);
-  options.comparator = &test_cmp;
-  options.num_levels = 2;
-  options.level0_file_num_compaction_trigger = 2;
-  DestroyAndReopen(options);
-  WriteOptions write_opts;
-  std::string ts_str = Timestamp(1, 0);
-  Slice ts = ts_str;
-  write_opts.timestamp = &ts;
-  ASSERT_OK(db_->Put(write_opts, "a", "value0"));
-  ASSERT_OK(Flush());
-
-  ts_str = Timestamp(2, 0);
-  ts = ts_str;
-  write_opts.timestamp = &ts;
-  ASSERT_OK(db_->Put(write_opts, "b", "value0"));
-  ts_str = Timestamp(3, 0);
-  ts = ts_str;
-  write_opts.timestamp = &ts;
-  ASSERT_OK(db_->Delete(write_opts, "a"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-
-  ReadOptions read_opts;
-  ts_str = Timestamp(1, 0);
-  ts = ts_str;
-  read_opts.timestamp = &ts;
-  std::string value;
-  Status s = db_->Get(read_opts, "a", &value);
-  ASSERT_OK(s);
-  ASSERT_EQ("value0", value);
-
-  ts_str = Timestamp(3, 0);
-  ts = ts_str;
-  read_opts.timestamp = &ts;
-  s = db_->Get(read_opts, "a", &value);
-  ASSERT_TRUE(s.IsNotFound());
-
-  // Time-travel to the past before deletion
-  ts_str = Timestamp(2, 0);
-  ts = ts_str;
-  read_opts.timestamp = &ts;
-  s = db_->Get(read_opts, "a", &value);
-  ASSERT_OK(s);
-  ASSERT_EQ("value0", value);
-  Close();
-}
-
 class DBBasicTestWithTimestampCompressionSettings
     : public DBBasicTestWithTimestampBase,
       public testing::WithParamInterface<
@@ -595,104 +536,6 @@ TEST_P(DBBasicTestWithTimestampCompressionSettings, PutAndGet) {
   Close();
 }
 
-TEST_P(DBBasicTestWithTimestampCompressionSettings, PutDeleteGet) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  const size_t kTimestampSize = Timestamp(0, 0).size();
-  TestComparator test_cmp(kTimestampSize);
-  options.comparator = &test_cmp;
-  const int kNumKeysPerFile = 1024;
-  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
-  BlockBasedTableOptions bbto;
-  bbto.filter_policy = std::get<0>(GetParam());
-  bbto.whole_key_filtering = true;
-  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-
-  const CompressionType comp_type = std::get<1>(GetParam());
-#if LZ4_VERSION_NUMBER < 10400  // r124+
-  if (comp_type == kLZ4Compression || comp_type == kLZ4HCCompression) {
-    return;
-  }
-#endif  // LZ4_VERSION_NUMBER >= 10400
-  if (!ZSTD_Supported() && comp_type == kZSTD) {
-    return;
-  }
-  if (!Zlib_Supported() && comp_type == kZlibCompression) {
-    return;
-  }
-
-  options.compression = comp_type;
-  options.compression_opts.max_dict_bytes = std::get<2>(GetParam());
-  if (comp_type == kZSTD) {
-    options.compression_opts.zstd_max_train_bytes = std::get<2>(GetParam());
-  }
-  options.compression_opts.parallel_threads = std::get<3>(GetParam());
-  options.target_file_size_base = 1 << 26;  // 64MB
-
-  DestroyAndReopen(options);
-
-  const size_t kNumL0Files =
-      static_cast<size_t>(Options().level0_file_num_compaction_trigger);
-  {
-    // Generate enough L0 files with ts=1 to trigger compaction to L1
-    std::string ts_str = Timestamp(1, 0);
-    Slice ts = ts_str;
-    WriteOptions wopts;
-    wopts.timestamp = &ts;
-    for (size_t i = 0; i != kNumL0Files; ++i) {
-      for (int j = 0; j != kNumKeysPerFile; ++j) {
-        ASSERT_OK(db_->Put(wopts, Key1(j), "value" + std::to_string(i)));
-      }
-      ASSERT_OK(db_->Flush(FlushOptions()));
-    }
-    ASSERT_OK(dbfull()->TEST_WaitForCompact());
-    // Generate another L0 at ts=3
-    ts_str = Timestamp(3, 0);
-    ts = ts_str;
-    wopts.timestamp = &ts;
-    for (int i = 0; i != kNumKeysPerFile; ++i) {
-      std::string key_str = Key1(i);
-      Slice key(key_str);
-      if ((i % 3) == 0) {
-        ASSERT_OK(db_->Delete(wopts, key));
-      } else {
-        ASSERT_OK(db_->Put(wopts, key, "new_value"));
-      }
-    }
-    ASSERT_OK(db_->Flush(FlushOptions()));
-    // Populate memtable at ts=5
-    ts_str = Timestamp(5, 0);
-    ts = ts_str;
-    wopts.timestamp = &ts;
-    for (int i = 0; i != kNumKeysPerFile; ++i) {
-      std::string key_str = Key1(i);
-      Slice key(key_str);
-      if ((i % 3) == 1) {
-        ASSERT_OK(db_->Delete(wopts, key));
-      } else if ((i % 3) == 2) {
-        ASSERT_OK(db_->Put(wopts, key, "new_value_2"));
-      }
-    }
-  }
-  {
-    std::string ts_str = Timestamp(6, 0);
-    Slice ts = ts_str;
-    ReadOptions ropts;
-    ropts.timestamp = &ts;
-    for (uint64_t i = 0; i != static_cast<uint64_t>(kNumKeysPerFile); ++i) {
-      std::string value;
-      Status s = db_->Get(ropts, Key1(i), &value);
-      if ((i % 3) == 2) {
-        ASSERT_OK(s);
-        ASSERT_EQ("new_value_2", value);
-      } else {
-        ASSERT_TRUE(s.IsNotFound());
-      }
-    }
-  }
-}
-
 #ifndef ROCKSDB_LITE
 // A class which remembers the name of each flushed file.
 class FlushedFileCollector : public EventListener {
@@ -1027,10 +870,15 @@ TEST_P(DBBasicTestWithTimestampPrefixSeek, ForwardIterateWithPrefix) {
     for (size_t i = 0; i != write_ts_list.size(); ++i) {
       Slice write_ts = write_ts_list[i];
       write_opts.timestamp = &write_ts;
-      for (uint64_t key = kMaxKey; key >= kMinKey; --key) {
+      uint64_t key = kMinKey;
+      do {
         Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(i));
         ASSERT_OK(s);
-      }
+        if (key == kMaxKey) {
+          break;
+        }
+        ++key;
+      } while (true);
     }
   }
   const std::vector<std::string> read_ts_list = {Timestamp(5, 0xffffffff),
@@ -1114,90 +962,6 @@ INSTANTIATE_TEST_CASE_P(
                                                    false))),
         ::testing::Bool()));
 
-class DBBasicTestWithTsIterTombstones
-    : public DBBasicTestWithTimestampBase,
-      public testing::WithParamInterface<
-          std::tuple<std::shared_ptr<const SliceTransform>,
-                     std::shared_ptr<const FilterPolicy>, int>> {
- public:
-  DBBasicTestWithTsIterTombstones()
-      : DBBasicTestWithTimestampBase("/db_basic_ts_iter_tombstones") {}
-};
-
-TEST_P(DBBasicTestWithTsIterTombstones, ForwardIterDelete) {
-  constexpr size_t kNumKeysPerFile = 128;
-  Options options = CurrentOptions();
-  options.env = env_;
-  const size_t kTimestampSize = Timestamp(0, 0).size();
-  TestComparator test_cmp(kTimestampSize);
-  options.comparator = &test_cmp;
-  options.prefix_extractor = std::get<0>(GetParam());
-  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
-  BlockBasedTableOptions bbto;
-  bbto.filter_policy = std::get<1>(GetParam());
-  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-  options.num_levels = std::get<2>(GetParam());
-  DestroyAndReopen(options);
-  std::vector<std::string> write_ts_strs = {Timestamp(2, 0), Timestamp(4, 0)};
-  constexpr uint64_t kMaxKey = 0xffffffffffffffff;
-  constexpr uint64_t kMinKey = 0xfffffffffffff000;
-  // Insert kMinKey...kMaxKey
-  uint64_t key = kMinKey;
-  WriteOptions write_opts;
-  Slice ts = write_ts_strs[0];
-  write_opts.timestamp = &ts;
-  do {
-    Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(key));
-    ASSERT_OK(s);
-    if (kMaxKey == key) {
-      break;
-    }
-    ++key;
-  } while (true);
-  // Delete them all
-  ts = write_ts_strs[1];
-  write_opts.timestamp = &ts;
-  for (key = kMaxKey; key >= kMinKey; --key) {
-    Status s;
-    if (0 != (key % 2)) {
-      s = db_->Put(write_opts, Key1(key), "value1" + std::to_string(key));
-    } else {
-      s = db_->Delete(write_opts, Key1(key));
-    }
-    ASSERT_OK(s);
-  }
-  ASSERT_OK(dbfull()->TEST_WaitForCompact());
-  {
-    std::string read_ts = Timestamp(4, 0);
-    ts = read_ts;
-    ReadOptions read_opts;
-    read_opts.total_order_seek = true;
-    read_opts.timestamp = &ts;
-    std::unique_ptr<Iterator> iter(db_->NewIterator(read_opts));
-    size_t count = 0;
-    key = kMinKey + 1;
-    for (iter->SeekToFirst(); iter->Valid(); iter->Next(), ++count, key += 2) {
-      ASSERT_EQ(Key1(key), iter->key());
-      ASSERT_EQ("value1" + std::to_string(key), iter->value());
-    }
-    ASSERT_EQ((kMaxKey - kMinKey + 1) / 2, count);
-  }
-  Close();
-}
-
-INSTANTIATE_TEST_CASE_P(
-    Timestamp, DBBasicTestWithTsIterTombstones,
-    ::testing::Combine(
-        ::testing::Values(
-            std::shared_ptr<const SliceTransform>(NewFixedPrefixTransform(7)),
-            std::shared_ptr<const SliceTransform>(NewFixedPrefixTransform(8))),
-        ::testing::Values(std::shared_ptr<const FilterPolicy>(nullptr),
-                          std::shared_ptr<const FilterPolicy>(
-                              NewBloomFilterPolicy(10, false)),
-                          std::shared_ptr<const FilterPolicy>(
-                              NewBloomFilterPolicy(20, false))),
-        ::testing::Values(2, 6)));
-
 }  // namespace ROCKSDB_NAMESPACE
 
 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS

db/dbformat.cc

@@ -23,7 +23,7 @@ namespace ROCKSDB_NAMESPACE {
 // and the value type is embedded as the low 8 bits in the sequence
 // number in internal keys, we need to use the highest-numbered
 // ValueType, not the lowest).
-const ValueType kValueTypeForSeek = kTypeDeletionWithTimestamp;
+const ValueType kValueTypeForSeek = kTypeBlobIndex;
 const ValueType kValueTypeForSeekForPrev = kTypeDeletion;
 
 uint64_t PackSequenceAndType(uint64_t seq, ValueType t) {

db/dbformat.h

@@ -69,8 +69,7 @@ enum ValueType : unsigned char {
   // generated by WriteUnprepared write policy is not mistakenly read by
   // another.
   kTypeBeginUnprepareXID = 0x13,  // WAL only.
-  kTypeDeletionWithTimestamp = 0x14,
-  kMaxValue = 0x7F  // Not used for storing records.
+  kMaxValue = 0x7F                // Not used for storing records.
 };
 
 // Defined in dbformat.cc
@@ -80,8 +79,7 @@ extern const ValueType kValueTypeForSeekForPrev;
 // Checks whether a type is an inline value type
 // (i.e. a type used in memtable skiplist and sst file datablock).
 inline bool IsValueType(ValueType t) {
-  return t <= kTypeMerge || t == kTypeSingleDeletion || t == kTypeBlobIndex ||
-         kTypeDeletionWithTimestamp == t;
+  return t <= kTypeMerge || t == kTypeSingleDeletion || t == kTypeBlobIndex;
 }
 
 // Checks whether a type is from user operation

db/error_handler.cc

@@ -51,19 +51,9 @@ std::map<std::tuple<BackgroundErrorReason, Status::Code, Status::SubCode, bool>,
                          Status::Code::kIOError, Status::SubCode::kNoSpace,
                          false),
          Status::Severity::kHardError},
-        // Errors during MANIFEST write
-        {std::make_tuple(BackgroundErrorReason::kManifestWrite,
-                         Status::Code::kIOError, Status::SubCode::kNoSpace,
-                         true),
-         Status::Severity::kHardError},
-        {std::make_tuple(BackgroundErrorReason::kManifestWrite,
-                         Status::Code::kIOError, Status::SubCode::kNoSpace,
-                         false),
-         Status::Severity::kHardError},
 };
 
-std::map<std::tuple<BackgroundErrorReason, Status::Code, bool>,
-         Status::Severity>
+std::map<std::tuple<BackgroundErrorReason, Status::Code, bool>, Status::Severity>
     DefaultErrorSeverityMap = {
         // Errors during BG compaction
         {std::make_tuple(BackgroundErrorReason::kCompaction,
@@ -85,11 +75,11 @@ std::map<std::tuple<BackgroundErrorReason, Status::Code, bool>,
         {std::make_tuple(BackgroundErrorReason::kFlush,
                          Status::Code::kCorruption, false),
          Status::Severity::kNoError},
-        {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError,
-                         true),
+        {std::make_tuple(BackgroundErrorReason::kFlush,
+                         Status::Code::kIOError, true),
          Status::Severity::kFatalError},
-        {std::make_tuple(BackgroundErrorReason::kFlush, Status::Code::kIOError,
-                         false),
+        {std::make_tuple(BackgroundErrorReason::kFlush,
+                         Status::Code::kIOError, false),
          Status::Severity::kNoError},
         // Errors during Write
         {std::make_tuple(BackgroundErrorReason::kWriteCallback,
@@ -104,36 +94,30 @@ std::map<std::tuple<BackgroundErrorReason, Status::Code, bool>,
         {std::make_tuple(BackgroundErrorReason::kWriteCallback,
                          Status::Code::kIOError, false),
          Status::Severity::kNoError},
-        {std::make_tuple(BackgroundErrorReason::kManifestWrite,
-                         Status::Code::kIOError, true),
-         Status::Severity::kFatalError},
-        {std::make_tuple(BackgroundErrorReason::kManifestWrite,
-                         Status::Code::kIOError, false),
-         Status::Severity::kFatalError},
 };
 
 std::map<std::tuple<BackgroundErrorReason, bool>, Status::Severity>
     DefaultReasonMap = {
         // Errors during BG compaction
         {std::make_tuple(BackgroundErrorReason::kCompaction, true),
-         Status::Severity::kFatalError},
+          Status::Severity::kFatalError},
         {std::make_tuple(BackgroundErrorReason::kCompaction, false),
-         Status::Severity::kNoError},
+          Status::Severity::kNoError},
         // Errors during BG flush
         {std::make_tuple(BackgroundErrorReason::kFlush, true),
-         Status::Severity::kFatalError},
+          Status::Severity::kFatalError},
         {std::make_tuple(BackgroundErrorReason::kFlush, false),
-         Status::Severity::kNoError},
+          Status::Severity::kNoError},
         // Errors during Write
         {std::make_tuple(BackgroundErrorReason::kWriteCallback, true),
-         Status::Severity::kFatalError},
+          Status::Severity::kFatalError},
         {std::make_tuple(BackgroundErrorReason::kWriteCallback, false),
-         Status::Severity::kFatalError},
+          Status::Severity::kFatalError},
         // Errors during Memtable update
         {std::make_tuple(BackgroundErrorReason::kMemTable, true),
-         Status::Severity::kFatalError},
+          Status::Severity::kFatalError},
         {std::make_tuple(BackgroundErrorReason::kMemTable, false),
-         Status::Severity::kFatalError},
+          Status::Severity::kFatalError},
 };
 
 void ErrorHandler::CancelErrorRecovery() {
@@ -263,10 +247,6 @@ Status ErrorHandler::SetBGError(const IOStatus& bg_io_err,
   if (recovery_in_prog_ && recovery_error_.ok()) {
     recovery_error_ = bg_io_err;
   }
-  if (BackgroundErrorReason::kManifestWrite == reason) {
-    // Always returns ok
-    db_->DisableFileDeletionsWithLock();
-  }
   Status new_bg_io_err = bg_io_err;
   Status s;
   if (bg_io_err.GetDataLoss()) {

db/external_sst_file_basic_test.cc

@@ -6,7 +6,6 @@
 #include <functional>
 
 #include "db/db_test_util.h"
-#include "db/version_edit.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
 #include "rocksdb/sst_file_writer.h"
@@ -42,29 +41,6 @@ class ExternalSSTFileBasicTest
     return db_->IngestExternalFile(files, opts);
   }
 
-  Status AddFileWithFileChecksum(
-      const std::vector<std::string>& files,
-      const std::vector<std::string>& files_checksums,
-      const std::vector<std::string>& files_checksum_func_names,
-      bool verify_file_checksum = true, bool move_files = false,
-      bool skip_snapshot_check = false, bool write_global_seqno = true) {
-    IngestExternalFileOptions opts;
-    opts.move_files = move_files;
-    opts.snapshot_consistency = !skip_snapshot_check;
-    opts.allow_global_seqno = false;
-    opts.allow_blocking_flush = false;
-    opts.write_global_seqno = write_global_seqno;
-    opts.verify_file_checksum = verify_file_checksum;
-
-    IngestExternalFileArg arg;
-    arg.column_family = db_->DefaultColumnFamily();
-    arg.external_files = files;
-    arg.options = opts;
-    arg.files_checksums = files_checksums;
-    arg.files_checksum_func_names = files_checksum_func_names;
-    return db_->IngestExternalFiles({arg});
-  }
-
   Status GenerateAndAddExternalFile(
       const Options options, std::vector<int> keys,
       const std::vector<ValueType>& value_types,
@@ -198,8 +174,6 @@ TEST_F(ExternalSSTFileBasicTest, Basic) {
   ASSERT_EQ(file1_info.num_range_del_entries, 0);
   ASSERT_EQ(file1_info.smallest_range_del_key, "");
   ASSERT_EQ(file1_info.largest_range_del_key, "");
-  ASSERT_EQ(file1_info.file_checksum, kUnknownFileChecksum);
-  ASSERT_EQ(file1_info.file_checksum_func_name, kUnknownFileChecksumFuncName);
   // sst_file_writer already finished, cannot add this value
   s = sst_file_writer.Put(Key(100), "bad_val");
   ASSERT_FALSE(s.ok()) << s.ToString();
@@ -218,369 +192,6 @@ TEST_F(ExternalSSTFileBasicTest, Basic) {
   DestroyAndRecreateExternalSSTFilesDir();
 }
 
-class ChecksumVerifyHelper {
- private:
-  Options options_;
-
- public:
-  ChecksumVerifyHelper(Options& options) : options_(options) {}
-  ~ChecksumVerifyHelper() {}
-
-  Status GetSingleFileChecksumAndFuncName(
-      const std::string& file_path, std::string* file_checksum,
-      std::string* file_checksum_func_name) {
-    Status s;
-    EnvOptions soptions;
-    std::unique_ptr<SequentialFile> file_reader;
-    s = options_.env->NewSequentialFile(file_path, &file_reader, soptions);
-    if (!s.ok()) {
-      return s;
-    }
-    std::unique_ptr<char[]> scratch(new char[2048]);
-    Slice result;
-    FileChecksumGenFactory* file_checksum_gen_factory =
-        options_.file_checksum_gen_factory.get();
-    if (file_checksum_gen_factory == nullptr) {
-      *file_checksum = kUnknownFileChecksum;
-      *file_checksum_func_name = kUnknownFileChecksumFuncName;
-      return Status::OK();
-    } else {
-      FileChecksumGenContext gen_context;
-      std::unique_ptr<FileChecksumGenerator> file_checksum_gen =
-          file_checksum_gen_factory->CreateFileChecksumGenerator(gen_context);
-      *file_checksum_func_name = file_checksum_gen->Name();
-      s = file_reader->Read(2048, &result, scratch.get());
-      if (!s.ok()) {
-        return s;
-      }
-      while (result.size() != 0) {
-        file_checksum_gen->Update(scratch.get(), result.size());
-        s = file_reader->Read(2048, &result, scratch.get());
-        if (!s.ok()) {
-          return s;
-        }
-      }
-      file_checksum_gen->Finalize();
-      *file_checksum = file_checksum_gen->GetChecksum();
-    }
-    return Status::OK();
-  }
-};
-
-TEST_F(ExternalSSTFileBasicTest, BasicWithFileChecksumCrc32c) {
-  Options options = CurrentOptions();
-  options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
-  ChecksumVerifyHelper checksum_helper(options);
-
-  SstFileWriter sst_file_writer(EnvOptions(), options);
-
-  // Current file size should be 0 after sst_file_writer init and before open a
-  // file.
-  ASSERT_EQ(sst_file_writer.FileSize(), 0);
-
-  // file1.sst (0 => 99)
-  std::string file1 = sst_files_dir_ + "file1.sst";
-  ASSERT_OK(sst_file_writer.Open(file1));
-  for (int k = 0; k < 100; k++) {
-    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
-  }
-  ExternalSstFileInfo file1_info;
-  Status s = sst_file_writer.Finish(&file1_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  std::string file_checksum, file_checksum_func_name;
-  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file1, &file_checksum, &file_checksum_func_name));
-
-  // Current file size should be non-zero after success write.
-  ASSERT_GT(sst_file_writer.FileSize(), 0);
-
-  ASSERT_EQ(file1_info.file_path, file1);
-  ASSERT_EQ(file1_info.num_entries, 100);
-  ASSERT_EQ(file1_info.smallest_key, Key(0));
-  ASSERT_EQ(file1_info.largest_key, Key(99));
-  ASSERT_EQ(file1_info.num_range_del_entries, 0);
-  ASSERT_EQ(file1_info.smallest_range_del_key, "");
-  ASSERT_EQ(file1_info.largest_range_del_key, "");
-  ASSERT_EQ(file1_info.file_checksum, file_checksum);
-  ASSERT_EQ(file1_info.file_checksum_func_name, file_checksum_func_name);
-  // sst_file_writer already finished, cannot add this value
-  s = sst_file_writer.Put(Key(100), "bad_val");
-  ASSERT_FALSE(s.ok()) << s.ToString();
-  s = sst_file_writer.DeleteRange(Key(100), Key(200));
-  ASSERT_FALSE(s.ok()) << s.ToString();
-
-  DestroyAndReopen(options);
-  // Add file using file path
-  s = DeprecatedAddFile({file1});
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
-  for (int k = 0; k < 100; k++) {
-    ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
-  }
-
-  DestroyAndRecreateExternalSSTFilesDir();
-}
-
-TEST_F(ExternalSSTFileBasicTest, IngestFileWithFileChecksum) {
-  Options old_options = CurrentOptions();
-  Options options = CurrentOptions();
-  options.file_checksum_gen_factory = GetFileChecksumGenCrc32cFactory();
-  const ImmutableCFOptions ioptions(options);
-  ChecksumVerifyHelper checksum_helper(options);
-
-  SstFileWriter sst_file_writer(EnvOptions(), options);
-
-  // file01.sst (1000 => 1099)
-  std::string file1 = sst_files_dir_ + "file01.sst";
-  ASSERT_OK(sst_file_writer.Open(file1));
-  for (int k = 1000; k < 1100; k++) {
-    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
-  }
-  ExternalSstFileInfo file1_info;
-  Status s = sst_file_writer.Finish(&file1_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file1_info.file_path, file1);
-  ASSERT_EQ(file1_info.num_entries, 100);
-  ASSERT_EQ(file1_info.smallest_key, Key(1000));
-  ASSERT_EQ(file1_info.largest_key, Key(1099));
-  std::string file_checksum1, file_checksum_func_name1;
-  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file1, &file_checksum1, &file_checksum_func_name1));
-  ASSERT_EQ(file1_info.file_checksum, file_checksum1);
-  ASSERT_EQ(file1_info.file_checksum_func_name, file_checksum_func_name1);
-
-  // file02.sst (1100 => 1299)
-  std::string file2 = sst_files_dir_ + "file02.sst";
-  ASSERT_OK(sst_file_writer.Open(file2));
-  for (int k = 1100; k < 1300; k++) {
-    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val"));
-  }
-  ExternalSstFileInfo file2_info;
-  s = sst_file_writer.Finish(&file2_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file2_info.file_path, file2);
-  ASSERT_EQ(file2_info.num_entries, 200);
-  ASSERT_EQ(file2_info.smallest_key, Key(1100));
-  ASSERT_EQ(file2_info.largest_key, Key(1299));
-  std::string file_checksum2, file_checksum_func_name2;
-  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file2, &file_checksum2, &file_checksum_func_name2));
-  ASSERT_EQ(file2_info.file_checksum, file_checksum2);
-  ASSERT_EQ(file2_info.file_checksum_func_name, file_checksum_func_name2);
-
-  // file03.sst (1300 => 1499)
-  std::string file3 = sst_files_dir_ + "file03.sst";
-  ASSERT_OK(sst_file_writer.Open(file3));
-  for (int k = 1300; k < 1500; k++) {
-    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
-  }
-  ExternalSstFileInfo file3_info;
-  s = sst_file_writer.Finish(&file3_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file3_info.file_path, file3);
-  ASSERT_EQ(file3_info.num_entries, 200);
-  ASSERT_EQ(file3_info.smallest_key, Key(1300));
-  ASSERT_EQ(file3_info.largest_key, Key(1499));
-  std::string file_checksum3, file_checksum_func_name3;
-  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file3, &file_checksum3, &file_checksum_func_name3));
-  ASSERT_EQ(file3_info.file_checksum, file_checksum3);
-  ASSERT_EQ(file3_info.file_checksum_func_name, file_checksum_func_name3);
-
-  // file04.sst (1500 => 1799)
-  std::string file4 = sst_files_dir_ + "file04.sst";
-  ASSERT_OK(sst_file_writer.Open(file4));
-  for (int k = 1500; k < 1800; k++) {
-    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
-  }
-  ExternalSstFileInfo file4_info;
-  s = sst_file_writer.Finish(&file4_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file4_info.file_path, file4);
-  ASSERT_EQ(file4_info.num_entries, 300);
-  ASSERT_EQ(file4_info.smallest_key, Key(1500));
-  ASSERT_EQ(file4_info.largest_key, Key(1799));
-  std::string file_checksum4, file_checksum_func_name4;
-  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file4, &file_checksum4, &file_checksum_func_name4));
-  ASSERT_EQ(file4_info.file_checksum, file_checksum4);
-  ASSERT_EQ(file4_info.file_checksum_func_name, file_checksum_func_name4);
-
-  // file05.sst (1800 => 1899)
-  std::string file5 = sst_files_dir_ + "file05.sst";
-  ASSERT_OK(sst_file_writer.Open(file5));
-  for (int k = 1800; k < 2000; k++) {
-    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
-  }
-  ExternalSstFileInfo file5_info;
-  s = sst_file_writer.Finish(&file5_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file5_info.file_path, file5);
-  ASSERT_EQ(file5_info.num_entries, 200);
-  ASSERT_EQ(file5_info.smallest_key, Key(1800));
-  ASSERT_EQ(file5_info.largest_key, Key(1999));
-  std::string file_checksum5, file_checksum_func_name5;
-  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file5, &file_checksum5, &file_checksum_func_name5));
-  ASSERT_EQ(file5_info.file_checksum, file_checksum5);
-  ASSERT_EQ(file5_info.file_checksum_func_name, file_checksum_func_name5);
-
-  // file06.sst (2000 => 2199)
-  std::string file6 = sst_files_dir_ + "file06.sst";
-  ASSERT_OK(sst_file_writer.Open(file6));
-  for (int k = 2000; k < 2200; k++) {
-    ASSERT_OK(sst_file_writer.Put(Key(k), Key(k) + "_val_overlap"));
-  }
-  ExternalSstFileInfo file6_info;
-  s = sst_file_writer.Finish(&file6_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file6_info.file_path, file6);
-  ASSERT_EQ(file6_info.num_entries, 200);
-  ASSERT_EQ(file6_info.smallest_key, Key(2000));
-  ASSERT_EQ(file6_info.largest_key, Key(2199));
-  std::string file_checksum6, file_checksum_func_name6;
-  ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-      file6, &file_checksum6, &file_checksum_func_name6));
-  ASSERT_EQ(file6_info.file_checksum, file_checksum6);
-  ASSERT_EQ(file6_info.file_checksum_func_name, file_checksum_func_name6);
-
-  s = AddFileWithFileChecksum({file1}, {file_checksum1, "xyz"},
-                              {file_checksum1}, true, false, false, false);
-  // does not care the checksum input since db does not enable file checksum
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_OK(env_->FileExists(file1));
-  std::vector<LiveFileMetaData> live_files;
-  dbfull()->GetLiveFilesMetaData(&live_files);
-  std::set<std::string> set1;
-  for (auto f : live_files) {
-    set1.insert(f.name);
-    ASSERT_EQ(f.file_checksum, kUnknownFileChecksum);
-    ASSERT_EQ(f.file_checksum_func_name, kUnknownFileChecksumFuncName);
-  }
-
-  // Reopen Db with checksum enabled
-  Reopen(options);
-  // Enable verify_file_checksum option
-  // The checksum vector does not match, fail the ingestion
-  s = AddFileWithFileChecksum({file2}, {file_checksum2, "xyz"},
-                              {file_checksum_func_name2}, true, false, false,
-                              false);
-  ASSERT_FALSE(s.ok()) << s.ToString();
-
-  // Enable verify_file_checksum option
-  // The checksum name does not match, fail the ingestion
-  s = AddFileWithFileChecksum({file2}, {file_checksum2}, {"xyz"}, true, false,
-                              false, false);
-  ASSERT_FALSE(s.ok()) << s.ToString();
-
-  // Enable verify_file_checksum option
-  // The checksum itself does not match, fail the ingestion
-  s = AddFileWithFileChecksum({file2}, {"xyz"}, {file_checksum_func_name2},
-                              true, false, false, false);
-  ASSERT_FALSE(s.ok()) << s.ToString();
-
-  // Enable verify_file_checksum option
-  // All matches, ingestion is successful
-  s = AddFileWithFileChecksum({file2}, {file_checksum2},
-                              {file_checksum_func_name2}, true, false, false,
-                              false);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  std::vector<LiveFileMetaData> live_files1;
-  dbfull()->GetLiveFilesMetaData(&live_files1);
-  for (auto f : live_files1) {
-    if (set1.find(f.name) == set1.end()) {
-      ASSERT_EQ(f.file_checksum, file_checksum2);
-      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name2);
-      set1.insert(f.name);
-    }
-  }
-  ASSERT_OK(env_->FileExists(file2));
-
-  // Enable verify_file_checksum option
-  // No checksum information is provided, generate it when ingesting
-  std::vector<std::string> checksum, checksum_func;
-  s = AddFileWithFileChecksum({file3}, checksum, checksum_func, true, false,
-                              false, false);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  std::vector<LiveFileMetaData> live_files2;
-  dbfull()->GetLiveFilesMetaData(&live_files2);
-  for (auto f : live_files2) {
-    if (set1.find(f.name) == set1.end()) {
-      ASSERT_EQ(f.file_checksum, file_checksum3);
-      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name3);
-      set1.insert(f.name);
-    }
-  }
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_OK(env_->FileExists(file3));
-
-  // Does not enable verify_file_checksum options
-  // The checksum name does not match, fail the ingestion
-  s = AddFileWithFileChecksum({file4}, {file_checksum4}, {"xyz"}, false, false,
-                              false, false);
-  ASSERT_FALSE(s.ok()) << s.ToString();
-
-  // Does not enable verify_file_checksum options
-  // Checksum function name matches, store the checksum being ingested.
-  s = AddFileWithFileChecksum({file4}, {"asd"}, {file_checksum_func_name4},
-                              false, false, false, false);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  std::vector<LiveFileMetaData> live_files3;
-  dbfull()->GetLiveFilesMetaData(&live_files3);
-  for (auto f : live_files3) {
-    if (set1.find(f.name) == set1.end()) {
-      ASSERT_FALSE(f.file_checksum == file_checksum4);
-      ASSERT_EQ(f.file_checksum, "asd");
-      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name4);
-      set1.insert(f.name);
-    }
-  }
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_OK(env_->FileExists(file4));
-
-  // enable verify_file_checksum options, DB enable checksum, and enable
-  // write_global_seq. So the checksum stored is different from the one
-  // ingested due to the sequence number changes.
-  s = AddFileWithFileChecksum({file5}, {file_checksum5},
-                              {file_checksum_func_name5}, true, false, false,
-                              true);
-  ASSERT_OK(s);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  std::vector<LiveFileMetaData> live_files4;
-  dbfull()->GetLiveFilesMetaData(&live_files4);
-  for (auto f : live_files4) {
-    if (set1.find(f.name) == set1.end()) {
-      std::string cur_checksum5, cur_checksum_func_name5;
-      ASSERT_OK(checksum_helper.GetSingleFileChecksumAndFuncName(
-          dbname_ + f.name, &cur_checksum5, &cur_checksum_func_name5));
-      ASSERT_EQ(f.file_checksum, cur_checksum5);
-      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name5);
-      set1.insert(f.name);
-    }
-  }
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_OK(env_->FileExists(file5));
-
-  // Does not enable verify_file_checksum options and also the ingested file
-  // checksum information is empty. DB will generate and store the checksum
-  // in Manifest.
-  std::vector<std::string> files_c6, files_name6;
-  s = AddFileWithFileChecksum({file6}, files_c6, files_name6, false, false,
-                              false, false);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  std::vector<LiveFileMetaData> live_files6;
-  dbfull()->GetLiveFilesMetaData(&live_files6);
-  for (auto f : live_files6) {
-    if (set1.find(f.name) == set1.end()) {
-      ASSERT_EQ(f.file_checksum, file_checksum6);
-      ASSERT_EQ(f.file_checksum_func_name, file_checksum_func_name6);
-      set1.insert(f.name);
-    }
-  }
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_OK(env_->FileExists(file6));
-}
-
 TEST_F(ExternalSSTFileBasicTest, NoCopy) {
   Options options = CurrentOptions();
   const ImmutableCFOptions ioptions(options);

db/external_sst_file_ingestion_job.cc

@@ -28,8 +28,6 @@ namespace ROCKSDB_NAMESPACE {
 
 Status ExternalSstFileIngestionJob::Prepare(
     const std::vector<std::string>& external_files_paths,
-    const std::vector<std::string>& files_checksums,
-    const std::vector<std::string>& files_checksum_func_names,
     uint64_t next_file_number, SuperVersion* sv) {
   Status status;
 
@@ -70,7 +68,7 @@ Status ExternalSstFileIngestionJob::Prepare(
                                    info2->smallest_internal_key) < 0;
         });
 
-    for (size_t i = 0; i + 1 < num_files; i++) {
+    for (size_t i = 0; i < num_files - 1; i++) {
       if (sstableKeyCompare(ucmp, sorted_files[i]->largest_internal_key,
                             sorted_files[i + 1]->smallest_internal_key) >= 0) {
         files_overlap_ = true;
@@ -144,9 +142,6 @@ Status ExternalSstFileIngestionJob::Prepare(
       break;
     }
     f.internal_file_path = path_inside_db;
-    // Initialize the checksum information of ingested files.
-    f.file_checksum = kUnknownFileChecksum;
-    f.file_checksum_func_name = kUnknownFileChecksumFuncName;
     ingestion_path_ids.insert(f.fd.GetPathId());
   }
 
@@ -165,128 +160,6 @@ Status ExternalSstFileIngestionJob::Prepare(
   }
   TEST_SYNC_POINT("ExternalSstFileIngestionJob::AfterSyncDir");
 
-  // Generate and check the sst file checksum. Note that, if
-  // IngestExternalFileOptions::write_global_seqno is true, we will not update
-  // the checksum information in the files_to_ingests_ here, since the file is
-  // upadted with the new global_seqno. After global_seqno is updated, DB will
-  // generate the new checksum and store it in the Manifest. In all other cases
-  // if ingestion_options_.write_global_seqno == true and
-  // verify_file_checksum is false, we only check the checksum function name.
-  if (status.ok() && db_options_.file_checksum_gen_factory != nullptr) {
-    if (ingestion_options_.verify_file_checksum == false &&
-        files_checksums.size() == files_to_ingest_.size() &&
-        files_checksum_func_names.size() == files_to_ingest_.size()) {
-      // Only when verify_file_checksum == false and the checksum for ingested
-      // files are provided, DB will use the provided checksum and does not
-      // generate the checksum for ingested files.
-      need_generate_file_checksum_ = false;
-    } else {
-      need_generate_file_checksum_ = true;
-    }
-    FileChecksumGenContext gen_context;
-    std::unique_ptr<FileChecksumGenerator> file_checksum_gen =
-        db_options_.file_checksum_gen_factory->CreateFileChecksumGenerator(
-            gen_context);
-    std::vector<std::string> generated_checksums;
-    std::vector<std::string> generated_checksum_func_names;
-    // Step 1: generate the checksum for ingested sst file.
-    if (need_generate_file_checksum_) {
-      for (size_t i = 0; i < files_to_ingest_.size(); i++) {
-        std::string generated_checksum, generated_checksum_func_name;
-        IOStatus io_s = GenerateOneFileChecksum(
-            fs_, files_to_ingest_[i].internal_file_path,
-            db_options_.file_checksum_gen_factory.get(), &generated_checksum,
-            &generated_checksum_func_name,
-            ingestion_options_.verify_checksums_readahead_size,
-            db_options_.allow_mmap_reads);
-        if (!io_s.ok()) {
-          status = io_s;
-          ROCKS_LOG_WARN(db_options_.info_log,
-                         "Sst file checksum generation of file: %s failed: %s",
-                         files_to_ingest_[i].internal_file_path.c_str(),
-                         status.ToString().c_str());
-          break;
-        }
-        if (ingestion_options_.write_global_seqno == false) {
-          files_to_ingest_[i].file_checksum = generated_checksum;
-          files_to_ingest_[i].file_checksum_func_name =
-              generated_checksum_func_name;
-        }
-        generated_checksums.push_back(generated_checksum);
-        generated_checksum_func_names.push_back(generated_checksum_func_name);
-      }
-    }
-
-    // Step 2: based on the verify_file_checksum and ingested checksum
-    // information, do the verification.
-    if (status.ok()) {
-      if (files_checksums.size() == files_to_ingest_.size() &&
-          files_checksum_func_names.size() == files_to_ingest_.size()) {
-        // Verify the checksum and checksum function name.
-        if (ingestion_options_.verify_file_checksum) {
-          for (size_t i = 0; i < files_to_ingest_.size(); i++) {
-            if (files_checksum_func_names[i] !=
-                generated_checksum_func_names[i]) {
-              status = Status::InvalidArgument(
-                  "Checksum function name does not match with the checksum "
-                  "function name of this DB");
-              ROCKS_LOG_WARN(
-                  db_options_.info_log,
-                  "Sst file checksum verification of file: %s failed: %s",
-                  external_files_paths[i].c_str(), status.ToString().c_str());
-              break;
-            }
-            if (files_checksums[i] != generated_checksums[i]) {
-              status = Status::Corruption(
-                  "Ingested checksum does not match with the generated "
-                  "checksum");
-              ROCKS_LOG_WARN(
-                  db_options_.info_log,
-                  "Sst file checksum verification of file: %s failed: %s",
-                  files_to_ingest_[i].internal_file_path.c_str(),
-                  status.ToString().c_str());
-              break;
-            }
-          }
-        } else {
-          // If verify_file_checksum is not enabled, we only verify the
-          // checksum function name. If it does not match, fail the ingestion.
-          // If matches, we trust the ingested checksum information and store
-          // in the Manifest.
-          for (size_t i = 0; i < files_to_ingest_.size(); i++) {
-            if (files_checksum_func_names[i] != file_checksum_gen->Name()) {
-              status = Status::InvalidArgument(
-                  "Checksum function name does not match with the checksum "
-                  "function name of this DB");
-              ROCKS_LOG_WARN(
-                  db_options_.info_log,
-                  "Sst file checksum verification of file: %s failed: %s",
-                  external_files_paths[i].c_str(), status.ToString().c_str());
-              break;
-            }
-            files_to_ingest_[i].file_checksum = files_checksums[i];
-            files_to_ingest_[i].file_checksum_func_name =
-                files_checksum_func_names[i];
-          }
-        }
-      } else if (files_checksums.size() != files_checksum_func_names.size() ||
-                 (files_checksums.size() == files_checksum_func_names.size() &&
-                  files_checksums.size() != 0)) {
-        // The checksum or checksum function name vector are not both empty
-        // and they are incomplete.
-        status = Status::InvalidArgument(
-            "The checksum information of ingested sst files are nonempty and "
-            "the size of checksums or the size of the checksum function "
-            "names "
-            "does not match with the number of ingested sst files");
-        ROCKS_LOG_WARN(
-            db_options_.info_log,
-            "The ingested sst files checksum information is incomplete: %s",
-            status.ToString().c_str());
-      }
-    }
-  }
-
   // TODO: The following is duplicated with Cleanup().
   if (!status.ok()) {
     // We failed, remove all files that we copied into the db
@@ -372,11 +245,6 @@ Status ExternalSstFileIngestionJob::Run() {
       return status;
     }
 
-    status = GenerateChecksumForIngestedFile(&f);
-    if (!status.ok()) {
-      return status;
-    }
-
     // We use the import time as the ancester time. This is the time the data
     // is written to the database.
     int64_t temp_current_time = 0;
@@ -387,11 +255,11 @@ Status ExternalSstFileIngestionJob::Run() {
           static_cast<uint64_t>(temp_current_time);
     }
 
-    edit_.AddFile(f.picked_level, f.fd.GetNumber(), f.fd.GetPathId(),
-                  f.fd.GetFileSize(), f.smallest_internal_key,
-                  f.largest_internal_key, f.assigned_seqno, f.assigned_seqno,
-                  false, kInvalidBlobFileNumber, oldest_ancester_time,
-                  current_time, f.file_checksum, f.file_checksum_func_name);
+    edit_.AddFile(
+        f.picked_level, f.fd.GetNumber(), f.fd.GetPathId(), f.fd.GetFileSize(),
+        f.smallest_internal_key, f.largest_internal_key, f.assigned_seqno,
+        f.assigned_seqno, false, kInvalidBlobFileNumber, oldest_ancester_time,
+        current_time, kUnknownFileChecksum, kUnknownFileChecksumFuncName);
   }
   return status;
 }
@@ -819,31 +687,6 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
   return Status::OK();
 }
 
-IOStatus ExternalSstFileIngestionJob::GenerateChecksumForIngestedFile(
-    IngestedFileInfo* file_to_ingest) {
-  if (db_options_.file_checksum_gen_factory == nullptr ||
-      need_generate_file_checksum_ == false ||
-      ingestion_options_.write_global_seqno == false) {
-    // If file_checksum_gen_factory is not set, we are not able to generate
-    // the checksum. if write_global_seqno is false, it means we will use
-    // file checksum generated during Prepare(). This step will be skipped.
-    return IOStatus::OK();
-  }
-  std::string file_checksum, file_checksum_func_name;
-  IOStatus io_s = GenerateOneFileChecksum(
-      fs_, file_to_ingest->internal_file_path,
-      db_options_.file_checksum_gen_factory.get(), &file_checksum,
-      &file_checksum_func_name,
-      ingestion_options_.verify_checksums_readahead_size,
-      db_options_.allow_mmap_reads);
-  if (!io_s.ok()) {
-    return io_s;
-  }
-  file_to_ingest->file_checksum = file_checksum;
-  file_to_ingest->file_checksum_func_name = file_checksum_func_name;
-  return IOStatus::OK();
-}
-
 bool ExternalSstFileIngestionJob::IngestedFileFitInLevel(
     const IngestedFileInfo* file_to_ingest, int level) {
   if (level == 0) {

db/external_sst_file_ingestion_job.h

@@ -63,10 +63,6 @@ struct IngestedFileInfo {
   // ingestion_options.move_files is false by default, thus copy_file is true
   // by default.
   bool copy_file = true;
-  // The checksum of ingested file
-  std::string file_checksum;
-  // The name of checksum function that generate the checksum
-  std::string file_checksum_func_name;
 };
 
 class ExternalSstFileIngestionJob {
@@ -94,8 +90,6 @@ class ExternalSstFileIngestionJob {
 
   // Prepare the job by copying external files into the DB.
   Status Prepare(const std::vector<std::string>& external_files_paths,
-                 const std::vector<std::string>& files_checksums,
-                 const std::vector<std::string>& files_checksum_func_names,
                  uint64_t next_file_number, SuperVersion* sv);
 
   // Check if we need to flush the memtable before running the ingestion job
@@ -154,8 +148,6 @@ class ExternalSstFileIngestionJob {
   // Set the file global sequence number to `seqno`
   Status AssignGlobalSeqnoForIngestedFile(IngestedFileInfo* file_to_ingest,
                                           SequenceNumber seqno);
-  // Generate the file checksum and store in the IngestedFileInfo
-  IOStatus GenerateChecksumForIngestedFile(IngestedFileInfo* file_to_ingest);
 
   // Check if `file_to_ingest` can fit in level `level`
   // REQUIRES: Mutex held
@@ -183,9 +175,6 @@ class ExternalSstFileIngestionJob {
   // Set in ExternalSstFileIngestionJob::Prepare(), if true all files are
   // ingested in L0
   bool files_overlap_{false};
-  // Set in ExternalSstFileIngestionJob::Prepare(), if true and DB
-  // file_checksum_gen_factory is set, DB will generate checksum each file.
-  bool need_generate_file_checksum_{true};
 };
 
 }  // namespace ROCKSDB_NAMESPACE

db/external_sst_file_test.cc

@@ -2801,7 +2801,7 @@ TEST_P(ExternalSSTFileTest, IngestFilesTriggerFlushingWithTwoWriteQueue) {
   GenerateAndAddExternalFile(options, data);
 }
 
-TEST_P(ExternalSSTFileTest, DeltaEncodingWhileGlobalSeqnoPresent) {
+TEST_P(ExternalSSTFileTest, DeltaEncodingWhileGlobalSeqnoPresents) {
   Options options = CurrentOptions();
   DestroyAndReopen(options);
   constexpr size_t kValueSize = 8;
@@ -2842,61 +2842,6 @@ TEST_P(ExternalSSTFileTest, DeltaEncodingWhileGlobalSeqnoPresent) {
   ASSERT_EQ(value, Get(key2));
 }
 
-TEST_P(ExternalSSTFileTest,
-       DeltaEncodingWhileGlobalSeqnoPresentIteratorSwitch) {
-  // Regression test for bug where global seqno corrupted the shared bytes
-  // buffer when switching from reverse iteration to forward iteration.
-  constexpr size_t kValueSize = 8;
-  Options options = CurrentOptions();
-
-  Random rnd(301);
-  std::string value(RandomString(&rnd, kValueSize));
-
-  std::string key0 = "aa";
-  std::string key1 = "ab";
-  // Make the prefix of key2 is same with key1 add zero seqno. The tail of every
-  // key is composed as (seqno << 8 | value_type), and here `1` represents
-  // ValueType::kTypeValue
-  std::string key2 = "ab";
-  PutFixed64(&key2, PackSequenceAndType(0, kTypeValue));
-  key2 += "cdefghijkl";
-  std::string key3 = key2 + "_";
-
-  // Write some key to make global seqno larger than zero
-  ASSERT_OK(Put(key0, value));
-
-  std::string fname = sst_files_dir_ + "test_file";
-  rocksdb::SstFileWriter writer(EnvOptions(), options);
-  ASSERT_OK(writer.Open(fname));
-
-  // key0 is a dummy to ensure the turnaround point (key1) comes from Prev
-  // cache rather than block (restart keys are pinned in block).
-  ASSERT_OK(writer.Put(key0, value));
-  ASSERT_OK(writer.Put(key1, value));
-  ASSERT_OK(writer.Put(key2, value));
-  ASSERT_OK(writer.Put(key3, value));
-
-  ExternalSstFileInfo info;
-  ASSERT_OK(writer.Finish(&info));
-
-  ASSERT_OK(dbfull()->IngestExternalFile({info.file_path},
-                                         IngestExternalFileOptions()));
-  ReadOptions read_opts;
-  // Prevents Seek() when switching directions, which circumvents the bug.
-  read_opts.total_order_seek = true;
-  Iterator* iter = db_->NewIterator(read_opts);
-  // Scan backwards to key2. File iterator will then be positioned at key1.
-  iter->Seek(key3);
-  ASSERT_EQ(key3, iter->key());
-  iter->Prev();
-  ASSERT_EQ(key2, iter->key());
-  // Scan forwards and make sure key3 is present. Previously key3 would be
-  // corrupted by the global seqno from key1.
-  iter->Next();
-  ASSERT_EQ(key3, iter->key());
-  delete iter;
-}
-
 INSTANTIATE_TEST_CASE_P(ExternalSSTFileTest, ExternalSSTFileTest,
                         testing::Values(std::make_tuple(false, false),
                                         std::make_tuple(false, true),

db/forward_iterator.cc

@@ -84,9 +84,9 @@ class ForwardLevelIterator : public InternalIterator {
         prefix_extractor_, /*table_reader_ptr=*/nullptr,
         /*file_read_hist=*/nullptr, TableReaderCaller::kUserIterator,
         /*arena=*/nullptr, /*skip_filters=*/false, /*level=*/-1,
-        /*max_file_size_for_l0_meta_pin=*/0,
         /*smallest_compaction_key=*/nullptr,
-        /*largest_compaction_key=*/nullptr, allow_unprepared_value_);
+        /*largest_compaction_key=*/nullptr,
+        allow_unprepared_value_);
     file_iter_->SetPinnedItersMgr(pinned_iters_mgr_);
     valid_ = false;
     if (!range_del_agg.IsEmpty()) {
@@ -686,9 +686,9 @@ void ForwardIterator::RebuildIterators(bool refresh_sv) {
         /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr,
         TableReaderCaller::kUserIterator, /*arena=*/nullptr,
         /*skip_filters=*/false, /*level=*/-1,
-        MaxFileSizeForL0MetaPin(sv_->mutable_cf_options),
         /*smallest_compaction_key=*/nullptr,
-        /*largest_compaction_key=*/nullptr, allow_unprepared_value_));
+        /*largest_compaction_key=*/nullptr,
+        allow_unprepared_value_));
   }
   BuildLevelIterators(vstorage);
   current_ = nullptr;
@@ -764,9 +764,9 @@ void ForwardIterator::RenewIterators() {
         /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr,
         TableReaderCaller::kUserIterator, /*arena=*/nullptr,
         /*skip_filters=*/false, /*level=*/-1,
-        MaxFileSizeForL0MetaPin(svnew->mutable_cf_options),
         /*smallest_compaction_key=*/nullptr,
-        /*largest_compaction_key=*/nullptr, allow_unprepared_value_));
+        /*largest_compaction_key=*/nullptr,
+        allow_unprepared_value_));
   }
 
   for (auto* f : l0_iters_) {
@@ -830,9 +830,9 @@ void ForwardIterator::ResetIncompleteIterators() {
         /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr,
         TableReaderCaller::kUserIterator, /*arena=*/nullptr,
         /*skip_filters=*/false, /*level=*/-1,
-        MaxFileSizeForL0MetaPin(sv_->mutable_cf_options),
         /*smallest_compaction_key=*/nullptr,
-        /*largest_compaction_key=*/nullptr, allow_unprepared_value_);
+        /*largest_compaction_key=*/nullptr,
+        allow_unprepared_value_);
     l0_iters_[i]->SetPinnedItersMgr(pinned_iters_mgr_);
   }
 

db/import_column_family_test.cc

@@ -14,7 +14,7 @@ class ImportColumnFamilyTest : public DBTestBase {
   ImportColumnFamilyTest() : DBTestBase("/import_column_family_test") {
     sst_files_dir_ = dbname_ + "/sst_files/";
     DestroyAndRecreateExternalSSTFilesDir();
-    export_files_dir_ = test::PerThreadDBPath(env_, "export");
+    export_files_dir_ = test::TmpDir(env_) + "/export";
     import_cfh_ = nullptr;
     import_cfh2_ = nullptr;
     metadata_ptr_ = nullptr;

db/internal_stats.cc

@@ -798,7 +798,7 @@ bool InternalStats::HandleCurrentSuperVersionNumber(uint64_t* value,
 
 bool InternalStats::HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db,
                                                  Version* /*version*/) {
-  *value = db->IsFileDeletionsEnabled() ? 1 : 0;
+  *value = db->IsFileDeletionsEnabled();
   return true;
 }
 

db/job_context.h

@@ -141,14 +141,11 @@ struct JobContext {
   std::vector<CandidateFileInfo> full_scan_candidate_files;
 
   // the list of all live sst files that cannot be deleted
-  std::vector<uint64_t> sst_live;
+  std::vector<FileDescriptor> sst_live;
 
-  // the list of sst files that we need to delete
+  // a list of sst files that we need to delete
   std::vector<ObsoleteFileInfo> sst_delete_files;
 
-  // the list of all live blob files that cannot be deleted
-  std::vector<uint64_t> blob_live;
-
   // the list of blob files that we need to delete
   std::vector<ObsoleteBlobFileInfo> blob_delete_files;
 

db/log_reader.cc

@@ -281,7 +281,6 @@ bool Reader::ReadMore(size_t* drop_size, int *error) {
     // Last read was a full read, so this is a trailer to skip
     buffer_.clear();
     Status status = file_->Read(kBlockSize, &buffer_, backing_store_);
-    TEST_SYNC_POINT_CALLBACK("LogReader::ReadMore:AfterReadFile", &status);
     end_of_buffer_offset_ += buffer_.size();
     if (!status.ok()) {
       buffer_.clear();

db/memtable.cc

@@ -724,7 +724,6 @@ static bool SaveValue(void* arg, const char* entry) {
         return false;
       }
       case kTypeDeletion:
-      case kTypeDeletionWithTimestamp:
       case kTypeSingleDeletion:
       case kTypeRangeDeletion: {
         if (*(s->merge_in_progress)) {
@@ -766,13 +765,7 @@ static bool SaveValue(void* arg, const char* entry) {
         }
         return true;
       }
-      default: {
-        std::string msg("Unrecognized value type: " +
-                        std::to_string(static_cast<int>(type)) + ". ");
-        msg.append("User key: " + user_key_slice.ToString(/*hex=*/true) + ". ");
-        msg.append("seq: " + std::to_string(seq) + ".");
-        *(s->status) = Status::Corruption(msg.c_str());
-      }
+      default:
         assert(false);
         return true;
     }
@@ -936,18 +929,8 @@ void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
 
     if (found_final_value) {
       iter->value->PinSelf();
-      range->AddValueSize(iter->value->size());
       range->MarkKeyDone(iter);
       RecordTick(moptions_.statistics, MEMTABLE_HIT);
-      if (range->GetValueSize() > read_options.value_size_soft_limit) {
-        // Set all remaining keys in range to Abort
-        for (auto range_iter = range->begin(); range_iter != range->end();
-             ++range_iter) {
-          range->MarkKeyDone(range_iter);
-          *(range_iter->s) = Status::Aborted();
-        }
-        break;
-      }
     }
   }
   PERF_COUNTER_ADD(get_from_memtable_count, 1);

db/memtable_list.cc

@@ -43,20 +43,22 @@ void MemTableListVersion::UnrefMemTable(autovector<MemTable*>* to_delete,
 }
 
 MemTableListVersion::MemTableListVersion(
-    size_t* parent_memtable_list_memory_usage, const MemTableListVersion& old)
+    size_t* parent_memtable_list_memory_usage, MemTableListVersion* old)
     : max_write_buffer_number_to_maintain_(
-          old.max_write_buffer_number_to_maintain_),
+          old->max_write_buffer_number_to_maintain_),
       max_write_buffer_size_to_maintain_(
-          old.max_write_buffer_size_to_maintain_),
+          old->max_write_buffer_size_to_maintain_),
       parent_memtable_list_memory_usage_(parent_memtable_list_memory_usage) {
-  memlist_ = old.memlist_;
-  for (auto& m : memlist_) {
-    m->Ref();
-  }
+  if (old != nullptr) {
+    memlist_ = old->memlist_;
+    for (auto& m : memlist_) {
+      m->Ref();
+    }
 
-  memlist_history_ = old.memlist_history_;
-  for (auto& m : memlist_history_) {
-    m->Ref();
+    memlist_history_ = old->memlist_history_;
+    for (auto& m : memlist_history_) {
+      m->Ref();
+    }
   }
 }
 
@@ -470,6 +472,7 @@ Status MemTableList::TryInstallMemtableFlushResults(
       }
 
       // this can release and reacquire the mutex.
+      vset->SetIOStatusOK();
       s = vset->LogAndApply(cfd, mutable_cf_options, edit_list, mu,
                             db_directory);
       *io_s = vset->io_status();
@@ -601,7 +604,7 @@ void MemTableList::InstallNewVersion() {
   } else {
     // somebody else holds the current version, we need to create new one
     MemTableListVersion* version = current_;
-    current_ = new MemTableListVersion(&current_memory_usage_, *version);
+    current_ = new MemTableListVersion(&current_memory_usage_, current_);
     current_->Ref();
     version->Unref();
   }

db/memtable_list.h

@@ -44,7 +44,7 @@ struct FlushJobInfo;
 class MemTableListVersion {
  public:
   explicit MemTableListVersion(size_t* parent_memtable_list_memory_usage,
-                               const MemTableListVersion& old);
+                               MemTableListVersion* old = nullptr);
   explicit MemTableListVersion(size_t* parent_memtable_list_memory_usage,
                                int max_write_buffer_number_to_maintain,
                                int64_t max_write_buffer_size_to_maintain);

db/obsolete_files_test.cc

@@ -10,7 +10,6 @@
 #ifndef ROCKSDB_LITE
 
 #include <stdlib.h>
-#include <algorithm>
 #include <map>
 #include <string>
 #include <vector>
@@ -194,124 +193,6 @@ TEST_F(ObsoleteFilesTest, DeleteObsoleteOptionsFile) {
   ASSERT_EQ(2, opts_file_count);
 }
 
-TEST_F(ObsoleteFilesTest, BlobFiles) {
-  VersionSet* const versions = dbfull()->TEST_GetVersionSet();
-  assert(versions);
-  assert(versions->GetColumnFamilySet());
-
-  ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault();
-  assert(cfd);
-
-  // Add a blob file that consists of nothing but garbage (and is thus obsolete)
-  // and another one that is live.
-  VersionEdit edit;
-
-  constexpr uint64_t first_blob_file_number = 234;
-  constexpr uint64_t first_total_blob_count = 555;
-  constexpr uint64_t first_total_blob_bytes = 66666;
-  constexpr char first_checksum_method[] = "CRC32";
-  constexpr char first_checksum_value[] = "3d87ff57";
-
-  edit.AddBlobFile(first_blob_file_number, first_total_blob_count,
-                   first_total_blob_bytes, first_checksum_method,
-                   first_checksum_value);
-  edit.AddBlobFileGarbage(first_blob_file_number, first_total_blob_count,
-                          first_total_blob_bytes);
-
-  constexpr uint64_t second_blob_file_number = 456;
-  constexpr uint64_t second_total_blob_count = 100;
-  constexpr uint64_t second_total_blob_bytes = 2000000;
-  constexpr char second_checksum_method[] = "CRC32B";
-  constexpr char second_checksum_value[] = "6dbdf23a";
-  edit.AddBlobFile(second_blob_file_number, second_total_blob_count,
-                   second_total_blob_bytes, second_checksum_method,
-                   second_checksum_value);
-
-  dbfull()->TEST_LockMutex();
-  Status s = versions->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
-                                   &edit, dbfull()->mutex());
-  dbfull()->TEST_UnlockMutex();
-
-  ASSERT_OK(s);
-
-  // Check for obsolete files and make sure the first blob file is picked up
-  // and grabbed for purge. The second blob file should be on the live list.
-  constexpr int job_id = 0;
-  JobContext job_context{job_id};
-
-  dbfull()->TEST_LockMutex();
-  constexpr bool force_full_scan = false;
-  dbfull()->FindObsoleteFiles(&job_context, force_full_scan);
-  dbfull()->TEST_UnlockMutex();
-
-  ASSERT_TRUE(job_context.HaveSomethingToDelete());
-  ASSERT_EQ(job_context.blob_delete_files.size(), 1);
-  ASSERT_EQ(job_context.blob_delete_files[0].GetBlobFileNumber(),
-            first_blob_file_number);
-
-  const auto& files_grabbed_for_purge =
-      dbfull()->TEST_GetFilesGrabbedForPurge();
-  ASSERT_NE(files_grabbed_for_purge.find(first_blob_file_number),
-            files_grabbed_for_purge.end());
-
-  ASSERT_EQ(job_context.blob_live.size(), 1);
-  ASSERT_EQ(job_context.blob_live[0], second_blob_file_number);
-
-  // Hack the job context a bit by adding a few files to the full scan
-  // list and adjusting the pending file number. We add the two files
-  // above as well as two additional ones, where one is old
-  // and should be cleaned up, and the other is still pending.
-  assert(cfd->ioptions());
-  assert(!cfd->ioptions()->cf_paths.empty());
-  const std::string& path = cfd->ioptions()->cf_paths.front().path;
-
-  constexpr uint64_t old_blob_file_number = 123;
-  constexpr uint64_t pending_blob_file_number = 567;
-
-  job_context.full_scan_candidate_files.emplace_back(
-      BlobFileName(old_blob_file_number), path);
-  job_context.full_scan_candidate_files.emplace_back(
-      BlobFileName(first_blob_file_number), path);
-  job_context.full_scan_candidate_files.emplace_back(
-      BlobFileName(second_blob_file_number), path);
-  job_context.full_scan_candidate_files.emplace_back(
-      BlobFileName(pending_blob_file_number), path);
-
-  job_context.min_pending_output = pending_blob_file_number;
-
-  // Purge obsolete files and make sure we purge the old file and the first file
-  // (and keep the second file and the pending file).
-  std::vector<std::string> deleted_files;
-  SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::DeleteObsoleteFileImpl::BeforeDeletion", [&](void* arg) {
-        const std::string* file = static_cast<std::string*>(arg);
-        assert(file);
-
-        constexpr char blob_extension[] = ".blob";
-
-        if (file->find(blob_extension) != std::string::npos) {
-          deleted_files.emplace_back(*file);
-        }
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  dbfull()->PurgeObsoleteFiles(job_context);
-  job_context.Clean();
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  ASSERT_EQ(files_grabbed_for_purge.find(first_blob_file_number),
-            files_grabbed_for_purge.end());
-
-  std::sort(deleted_files.begin(), deleted_files.end());
-  const std::vector<std::string> expected_deleted_files{
-      BlobFileName(path, old_blob_file_number),
-      BlobFileName(path, first_blob_file_number)};
-
-  ASSERT_EQ(deleted_files, expected_deleted_files);
-}
-
 }  // namespace ROCKSDB_NAMESPACE
 
 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS

db/repair.cc

@@ -528,8 +528,7 @@ class Repairer {
           cfd->GetLatestMutableCFOptions()->prefix_extractor.get(),
           /*table_reader_ptr=*/nullptr, /*file_read_hist=*/nullptr,
           TableReaderCaller::kRepair, /*arena=*/nullptr, /*skip_filters=*/false,
-          /*level=*/-1, /*max_file_size_for_l0_meta_pin=*/0,
-          /*smallest_compaction_key=*/nullptr,
+          /*level=*/-1, /*smallest_compaction_key=*/nullptr,
           /*largest_compaction_key=*/nullptr,
           /*allow_unprepared_value=*/false);
       ParsedInternalKey parsed;

db/table_cache.cc

@@ -97,8 +97,7 @@ Status TableCache::GetTableReader(
     bool sequential_mode, bool record_read_stats, HistogramImpl* file_read_hist,
     std::unique_ptr<TableReader>* table_reader,
     const SliceTransform* prefix_extractor, bool skip_filters, int level,
-    bool prefetch_index_and_filter_in_cache,
-    size_t max_file_size_for_l0_meta_pin) {
+    bool prefetch_index_and_filter_in_cache) {
   std::string fname =
       TableFileName(ioptions_.cf_paths, fd.GetNumber(), fd.GetPathId());
   std::unique_ptr<FSRandomAccessFile> file;
@@ -124,9 +123,7 @@ Status TableCache::GetTableReader(
     s = ioptions_.table_factory->NewTableReader(
         TableReaderOptions(ioptions_, prefix_extractor, file_options,
                            internal_comparator, skip_filters, immortal_tables_,
-                           false /* force_direct_prefetch */, level,
-                           fd.largest_seqno, block_cache_tracer_,
-                           max_file_size_for_l0_meta_pin),
+                           level, fd.largest_seqno, block_cache_tracer_),
         std::move(file_reader), fd.GetFileSize(), table_reader,
         prefetch_index_and_filter_in_cache);
     TEST_SYNC_POINT("TableCache::GetTableReader:0");
@@ -147,8 +144,8 @@ Status TableCache::FindTable(const FileOptions& file_options,
                              const SliceTransform* prefix_extractor,
                              const bool no_io, bool record_read_stats,
                              HistogramImpl* file_read_hist, bool skip_filters,
-                             int level, bool prefetch_index_and_filter_in_cache,
-                             size_t max_file_size_for_l0_meta_pin) {
+                             int level,
+                             bool prefetch_index_and_filter_in_cache) {
   PERF_TIMER_GUARD_WITH_ENV(find_table_nanos, ioptions_.env);
   Status s;
   uint64_t number = fd.GetNumber();
@@ -172,8 +169,7 @@ Status TableCache::FindTable(const FileOptions& file_options,
     s = GetTableReader(file_options, internal_comparator, fd,
                        false /* sequential mode */, record_read_stats,
                        file_read_hist, &table_reader, prefix_extractor,
-                       skip_filters, level, prefetch_index_and_filter_in_cache,
-                       max_file_size_for_l0_meta_pin);
+                       skip_filters, level, prefetch_index_and_filter_in_cache);
     if (!s.ok()) {
       assert(table_reader == nullptr);
       RecordTick(ioptions_.statistics, NO_FILE_ERRORS);
@@ -197,7 +193,6 @@ InternalIterator* TableCache::NewIterator(
     RangeDelAggregator* range_del_agg, const SliceTransform* prefix_extractor,
     TableReader** table_reader_ptr, HistogramImpl* file_read_hist,
     TableReaderCaller caller, Arena* arena, bool skip_filters, int level,
-    size_t max_file_size_for_l0_meta_pin,
     const InternalKey* smallest_compaction_key,
     const InternalKey* largest_compaction_key, bool allow_unprepared_value) {
   PERF_TIMER_GUARD(new_table_iterator_nanos);
@@ -215,9 +210,7 @@ InternalIterator* TableCache::NewIterator(
     s = FindTable(file_options, icomparator, fd, &handle, prefix_extractor,
                   options.read_tier == kBlockCacheTier /* no_io */,
                   !for_compaction /* record_read_stats */, file_read_hist,
-                  skip_filters, level,
-                  true /* prefetch_index_and_filter_in_cache */,
-                  max_file_size_for_l0_meta_pin);
+                  skip_filters, level);
     if (s.ok()) {
       table_reader = GetTableReaderFromHandle(handle);
     }
@@ -378,7 +371,7 @@ Status TableCache::Get(const ReadOptions& options,
                        GetContext* get_context,
                        const SliceTransform* prefix_extractor,
                        HistogramImpl* file_read_hist, bool skip_filters,
-                       int level, size_t max_file_size_for_l0_meta_pin) {
+                       int level) {
   auto& fd = file_meta.fd;
   std::string* row_cache_entry = nullptr;
   bool done = false;
@@ -403,12 +396,10 @@ Status TableCache::Get(const ReadOptions& options,
   Cache::Handle* handle = nullptr;
   if (!done && s.ok()) {
     if (t == nullptr) {
-      s = FindTable(file_options_, internal_comparator, fd, &handle,
-                    prefix_extractor,
-                    options.read_tier == kBlockCacheTier /* no_io */,
-                    true /* record_read_stats */, file_read_hist, skip_filters,
-                    level, true /* prefetch_index_and_filter_in_cache */,
-                    max_file_size_for_l0_meta_pin);
+      s = FindTable(
+          file_options_, internal_comparator, fd, &handle, prefix_extractor,
+          options.read_tier == kBlockCacheTier /* no_io */,
+          true /* record_read_stats */, file_read_hist, skip_filters, level);
       if (s.ok()) {
         t = GetTableReaderFromHandle(handle);
       }

db/table_cache.h

@@ -72,8 +72,7 @@ class TableCache {
       const FileMetaData& file_meta, RangeDelAggregator* range_del_agg,
       const SliceTransform* prefix_extractor, TableReader** table_reader_ptr,
       HistogramImpl* file_read_hist, TableReaderCaller caller, Arena* arena,
-      bool skip_filters, int level, size_t max_file_size_for_l0_meta_pin,
-      const InternalKey* smallest_compaction_key,
+      bool skip_filters, int level, const InternalKey* smallest_compaction_key,
       const InternalKey* largest_compaction_key, bool allow_unprepared_value);
 
   // If a seek to internal key "k" in specified file finds an entry,
@@ -92,7 +91,7 @@ class TableCache {
              GetContext* get_context,
              const SliceTransform* prefix_extractor = nullptr,
              HistogramImpl* file_read_hist = nullptr, bool skip_filters = false,
-             int level = -1, size_t max_file_size_for_l0_meta_pin = 0);
+             int level = -1);
 
   // Return the range delete tombstone iterator of the file specified by
   // `file_meta`.
@@ -136,8 +135,7 @@ class TableCache {
                    const bool no_io = false, bool record_read_stats = true,
                    HistogramImpl* file_read_hist = nullptr,
                    bool skip_filters = false, int level = -1,
-                   bool prefetch_index_and_filter_in_cache = true,
-                   size_t max_file_size_for_l0_meta_pin = 0);
+                   bool prefetch_index_and_filter_in_cache = true);
 
   // Get TableReader from a cache handle.
   TableReader* GetTableReaderFromHandle(Cache::Handle* handle);
@@ -202,8 +200,7 @@ class TableCache {
                         std::unique_ptr<TableReader>* table_reader,
                         const SliceTransform* prefix_extractor = nullptr,
                         bool skip_filters = false, int level = -1,
-                        bool prefetch_index_and_filter_in_cache = true,
-                        size_t max_file_size_for_l0_meta_pin = 0);
+                        bool prefetch_index_and_filter_in_cache = true);
 
   // Create a key prefix for looking up the row cache. The prefix is of the
   // format row_cache_id + fd_number + seq_no. Later, the user key can be

db/version_builder.cc

@@ -86,86 +86,6 @@ class VersionBuilder::Rep {
     std::unordered_map<uint64_t, FileMetaData*> added_files;
   };
 
-  class BlobFileMetaDataDelta {
-   public:
-    bool IsEmpty() const {
-      return !shared_meta_ && !additional_garbage_count_ &&
-             !additional_garbage_bytes_ && newly_linked_ssts_.empty() &&
-             newly_unlinked_ssts_.empty();
-    }
-
-    std::shared_ptr<SharedBlobFileMetaData> GetSharedMeta() const {
-      return shared_meta_;
-    }
-
-    uint64_t GetAdditionalGarbageCount() const {
-      return additional_garbage_count_;
-    }
-
-    uint64_t GetAdditionalGarbageBytes() const {
-      return additional_garbage_bytes_;
-    }
-
-    const std::unordered_set<uint64_t>& GetNewlyLinkedSsts() const {
-      return newly_linked_ssts_;
-    }
-
-    const std::unordered_set<uint64_t>& GetNewlyUnlinkedSsts() const {
-      return newly_unlinked_ssts_;
-    }
-
-    void SetSharedMeta(std::shared_ptr<SharedBlobFileMetaData> shared_meta) {
-      assert(!shared_meta_);
-      assert(shared_meta);
-
-      shared_meta_ = std::move(shared_meta);
-    }
-
-    void AddGarbage(uint64_t count, uint64_t bytes) {
-      additional_garbage_count_ += count;
-      additional_garbage_bytes_ += bytes;
-    }
-
-    void LinkSst(uint64_t sst_file_number) {
-      assert(newly_linked_ssts_.find(sst_file_number) ==
-             newly_linked_ssts_.end());
-
-      // Reconcile with newly unlinked SSTs on the fly. (Note: an SST can be
-      // linked to and unlinked from the same blob file in the case of a trivial
-      // move.)
-      auto it = newly_unlinked_ssts_.find(sst_file_number);
-
-      if (it != newly_unlinked_ssts_.end()) {
-        newly_unlinked_ssts_.erase(it);
-      } else {
-        newly_linked_ssts_.emplace(sst_file_number);
-      }
-    }
-
-    void UnlinkSst(uint64_t sst_file_number) {
-      assert(newly_unlinked_ssts_.find(sst_file_number) ==
-             newly_unlinked_ssts_.end());
-
-      // Reconcile with newly linked SSTs on the fly. (Note: an SST can be
-      // linked to and unlinked from the same blob file in the case of a trivial
-      // move.)
-      auto it = newly_linked_ssts_.find(sst_file_number);
-
-      if (it != newly_linked_ssts_.end()) {
-        newly_linked_ssts_.erase(it);
-      } else {
-        newly_unlinked_ssts_.emplace(sst_file_number);
-      }
-    }
-
-   private:
-    std::shared_ptr<SharedBlobFileMetaData> shared_meta_;
-    uint64_t additional_garbage_count_ = 0;
-    uint64_t additional_garbage_bytes_ = 0;
-    std::unordered_set<uint64_t> newly_linked_ssts_;
-    std::unordered_set<uint64_t> newly_unlinked_ssts_;
-  };
-
   const FileOptions& file_options_;
   const ImmutableCFOptions* const ioptions_;
   TableCache* table_cache_;
@@ -173,21 +93,19 @@ class VersionBuilder::Rep {
   VersionSet* version_set_;
   int num_levels_;
   LevelState* levels_;
-  // Store sizes of levels larger than num_levels_. We do this instead of
+  // Store states of levels larger than num_levels_. We do this instead of
   // storing them in levels_ to avoid regression in case there are no files
   // on invalid levels. The version is not consistent if in the end the files
   // on invalid levels don't cancel out.
-  std::unordered_map<int, size_t> invalid_level_sizes_;
+  std::map<int, std::unordered_set<uint64_t>> invalid_levels_;
   // Whether there are invalid new files or invalid deletion on levels larger
   // than num_levels_.
   bool has_invalid_levels_;
-  // Current levels of table files affected by additions/deletions.
-  std::unordered_map<uint64_t, int> table_file_levels_;
   FileComparator level_zero_cmp_;
   FileComparator level_nonzero_cmp_;
 
-  // Metadata delta for all blob files affected by the series of version edits.
-  std::map<uint64_t, BlobFileMetaDataDelta> blob_file_meta_deltas_;
+  // Metadata for all blob files affected by the series of version edits.
+  std::map<uint64_t, std::shared_ptr<BlobFileMetaData>> changed_blob_files_;
 
  public:
   Rep(const FileOptions& file_options, const ImmutableCFOptions* ioptions,
@@ -232,12 +150,14 @@ class VersionBuilder::Rep {
     }
   }
 
-  bool IsBlobFileInVersion(uint64_t blob_file_number) const {
-    auto delta_it = blob_file_meta_deltas_.find(blob_file_number);
-    if (delta_it != blob_file_meta_deltas_.end()) {
-      if (delta_it->second.GetSharedMeta()) {
-        return true;
-      }
+  std::shared_ptr<BlobFileMetaData> GetBlobFileMetaData(
+      uint64_t blob_file_number) const {
+    auto changed_it = changed_blob_files_.find(blob_file_number);
+    if (changed_it != changed_blob_files_.end()) {
+      const auto& meta = changed_it->second;
+      assert(meta);
+
+      return meta;
     }
 
     assert(base_vstorage_);
@@ -246,28 +166,39 @@ class VersionBuilder::Rep {
 
     auto base_it = base_blob_files.find(blob_file_number);
     if (base_it != base_blob_files.end()) {
-      assert(base_it->second);
-      assert(base_it->second->GetSharedMeta());
+      const auto& meta = base_it->second;
+      assert(meta);
 
-      return true;
+      return meta;
     }
 
-    return false;
+    return std::shared_ptr<BlobFileMetaData>();
   }
 
-  using ExpectedLinkedSsts =
-      std::unordered_map<uint64_t, BlobFileMetaData::LinkedSsts>;
+  Status CheckConsistencyOfOldestBlobFileReference(
+      const VersionStorageInfo* vstorage, uint64_t blob_file_number) const {
+    assert(vstorage);
 
-  static void UpdateExpectedLinkedSsts(
-      uint64_t table_file_number, uint64_t blob_file_number,
-      ExpectedLinkedSsts* expected_linked_ssts) {
-    assert(expected_linked_ssts);
-
-    if (blob_file_number == kInvalidBlobFileNumber) {
-      return;
+    // TODO: remove this check once we actually start recoding metadata for
+    // blob files in the MANIFEST.
+    if (vstorage->GetBlobFiles().empty()) {
+      return Status::OK();
     }
 
-    (*expected_linked_ssts)[blob_file_number].emplace(table_file_number);
+    if (blob_file_number == kInvalidBlobFileNumber) {
+      return Status::OK();
+    }
+
+    const auto meta = GetBlobFileMetaData(blob_file_number);
+    if (!meta) {
+      std::ostringstream oss;
+      oss << "Blob file #" << blob_file_number
+          << " is not part of this version";
+
+      return Status::Corruption("VersionBuilder", oss.str());
+    }
+
+    return Status::OK();
   }
 
   Status CheckConsistency(VersionStorageInfo* vstorage) {
@@ -278,13 +209,9 @@ class VersionBuilder::Rep {
       return Status::OK();
     }
 #endif
-    // Make sure the files are sorted correctly and that the links between
-    // table files and blob files are consistent. The latter is checked using
-    // the following mapping, which is built using the forward links
-    // (table file -> blob file), and is subsequently compared with the inverse
-    // mapping stored in the BlobFileMetaData objects.
-    ExpectedLinkedSsts expected_linked_ssts;
-
+    // Make sure the files are sorted correctly and that the oldest blob file
+    // reference for each table file points to a valid blob file in this
+    // version.
     for (int level = 0; level < num_levels_; level++) {
       auto& level_files = vstorage->LevelFiles(level);
 
@@ -293,23 +220,29 @@ class VersionBuilder::Rep {
       }
 
       assert(level_files[0]);
-      UpdateExpectedLinkedSsts(level_files[0]->fd.GetNumber(),
-                               level_files[0]->oldest_blob_file_number,
-                               &expected_linked_ssts);
+      Status s = CheckConsistencyOfOldestBlobFileReference(
+          vstorage, level_files[0]->oldest_blob_file_number);
+      if (!s.ok()) {
+        return s;
+      }
+
       for (size_t i = 1; i < level_files.size(); i++) {
         assert(level_files[i]);
-        UpdateExpectedLinkedSsts(level_files[i]->fd.GetNumber(),
-                                 level_files[i]->oldest_blob_file_number,
-                                 &expected_linked_ssts);
+        s = CheckConsistencyOfOldestBlobFileReference(
+            vstorage, level_files[i]->oldest_blob_file_number);
+        if (!s.ok()) {
+          return s;
+        }
 
         auto f1 = level_files[i - 1];
         auto f2 = level_files[i];
-        if (level == 0) {
 #ifndef NDEBUG
-          auto pair = std::make_pair(&f1, &f2);
-          TEST_SYNC_POINT_CALLBACK("VersionBuilder::CheckConsistency0", &pair);
+        auto pair = std::make_pair(&f1, &f2);
+        TEST_SYNC_POINT_CALLBACK("VersionBuilder::CheckConsistency", &pair);
 #endif
+        if (level == 0) {
           if (!level_zero_cmp_(f1, f2)) {
+            fprintf(stderr, "L0 files are not sorted properly");
             return Status::Corruption("L0 files are not sorted properly");
           }
 
@@ -318,6 +251,11 @@ class VersionBuilder::Rep {
             SequenceNumber external_file_seqno = f2->fd.smallest_seqno;
             if (!(external_file_seqno < f1->fd.largest_seqno ||
                   external_file_seqno == 0)) {
+              fprintf(stderr,
+                      "L0 file with seqno %" PRIu64 " %" PRIu64
+                      " vs. file with global_seqno %" PRIu64 "\n",
+                      f1->fd.smallest_seqno, f1->fd.largest_seqno,
+                      external_file_seqno);
               return Status::Corruption(
                   "L0 file with seqno " +
                   NumberToString(f1->fd.smallest_seqno) + " " +
@@ -327,6 +265,11 @@ class VersionBuilder::Rep {
                   NumberToString(f1->fd.GetNumber()));
             }
           } else if (f1->fd.smallest_seqno <= f2->fd.smallest_seqno) {
+            fprintf(stderr,
+                    "L0 files seqno %" PRIu64 " %" PRIu64 " vs. %" PRIu64
+                    " %" PRIu64 "\n",
+                    f1->fd.smallest_seqno, f1->fd.largest_seqno,
+                    f2->fd.smallest_seqno, f2->fd.largest_seqno);
             return Status::Corruption(
                 "L0 files seqno " + NumberToString(f1->fd.smallest_seqno) +
                 " " + NumberToString(f1->fd.largest_seqno) + " " +
@@ -336,11 +279,8 @@ class VersionBuilder::Rep {
                 NumberToString(f2->fd.GetNumber()));
           }
         } else {
-#ifndef NDEBUG
-          auto pair = std::make_pair(&f1, &f2);
-          TEST_SYNC_POINT_CALLBACK("VersionBuilder::CheckConsistency1", &pair);
-#endif
           if (!level_nonzero_cmp_(f1, f2)) {
+            fprintf(stderr, "L%d files are not sorted properly", level);
             return Status::Corruption("L" + NumberToString(level) +
                                       " files are not sorted properly");
           }
@@ -348,6 +288,9 @@ class VersionBuilder::Rep {
           // Make sure there is no overlap in levels > 0
           if (vstorage->InternalComparator()->Compare(f1->largest,
                                                       f2->smallest) >= 0) {
+            fprintf(stderr, "L%d have overlapping ranges %s vs. %s\n", level,
+                    (f1->largest).DebugString(true).c_str(),
+                    (f2->smallest).DebugString(true).c_str());
             return Status::Corruption(
                 "L" + NumberToString(level) + " have overlapping ranges " +
                 (f1->largest).DebugString(true) + " vs. " +
@@ -360,27 +303,17 @@ class VersionBuilder::Rep {
     // Make sure that all blob files in the version have non-garbage data.
     const auto& blob_files = vstorage->GetBlobFiles();
     for (const auto& pair : blob_files) {
-      const uint64_t blob_file_number = pair.first;
       const auto& blob_file_meta = pair.second;
       assert(blob_file_meta);
 
       if (blob_file_meta->GetGarbageBlobCount() >=
           blob_file_meta->GetTotalBlobCount()) {
         std::ostringstream oss;
-        oss << "Blob file #" << blob_file_number
+        oss << "Blob file #" << blob_file_meta->GetBlobFileNumber()
             << " consists entirely of garbage";
 
         return Status::Corruption("VersionBuilder", oss.str());
       }
-
-      if (blob_file_meta->GetLinkedSsts() !=
-          expected_linked_ssts[blob_file_number]) {
-        std::ostringstream oss;
-        oss << "Links are inconsistent between table files and blob file #"
-            << blob_file_number;
-
-        return Status::Corruption("VersionBuilder", oss.str());
-      }
     }
 
     Status ret_s;
@@ -389,26 +322,73 @@ class VersionBuilder::Rep {
     return ret_s;
   }
 
-  bool CheckConsistencyForNumLevels() const {
+  Status CheckConsistencyForDeletes(VersionEdit* /*edit*/, uint64_t number,
+                                    int level) {
+#ifdef NDEBUG
+    if (!base_vstorage_->force_consistency_checks()) {
+      // Dont run consistency checks in release mode except if
+      // explicitly asked to
+      return Status::OK();
+    }
+#endif
+    // a file to be deleted better exist in the previous version
+    bool found = false;
+    for (int l = 0; !found && l < num_levels_; l++) {
+      const std::vector<FileMetaData*>& base_files =
+          base_vstorage_->LevelFiles(l);
+      for (size_t i = 0; i < base_files.size(); i++) {
+        FileMetaData* f = base_files[i];
+        if (f->fd.GetNumber() == number) {
+          found = true;
+          break;
+        }
+      }
+    }
+    // if the file did not exist in the previous version, then it
+    // is possibly moved from lower level to higher level in current
+    // version
+    for (int l = level + 1; !found && l < num_levels_; l++) {
+      auto& level_added = levels_[l].added_files;
+      auto got = level_added.find(number);
+      if (got != level_added.end()) {
+        found = true;
+        break;
+      }
+    }
+
+    // maybe this file was added in a previous edit that was Applied
+    if (!found) {
+      auto& level_added = levels_[level].added_files;
+      auto got = level_added.find(number);
+      if (got != level_added.end()) {
+        found = true;
+      }
+    }
+    if (!found) {
+      fprintf(stderr, "not found %" PRIu64 "\n", number);
+      return Status::Corruption("not found " + NumberToString(number));
+    }
+    return Status::OK();
+  }
+
+  bool CheckConsistencyForNumLevels() {
     // Make sure there are no files on or beyond num_levels().
     if (has_invalid_levels_) {
       return false;
     }
-
-    for (const auto& pair : invalid_level_sizes_) {
-      const size_t level_size = pair.second;
-      if (level_size != 0) {
+    for (auto& level : invalid_levels_) {
+      if (level.second.size() > 0) {
         return false;
       }
     }
-
     return true;
   }
 
   Status ApplyBlobFileAddition(const BlobFileAddition& blob_file_addition) {
     const uint64_t blob_file_number = blob_file_addition.GetBlobFileNumber();
 
-    if (IsBlobFileInVersion(blob_file_number)) {
+    auto meta = GetBlobFileMetaData(blob_file_number);
+    if (meta) {
       std::ostringstream oss;
       oss << "Blob file #" << blob_file_number << " already added";
 
@@ -439,8 +419,12 @@ class VersionBuilder::Rep {
         blob_file_addition.GetChecksumMethod(),
         blob_file_addition.GetChecksumValue(), deleter);
 
-    blob_file_meta_deltas_[blob_file_number].SetSharedMeta(
-        std::move(shared_meta));
+    constexpr uint64_t garbage_blob_count = 0;
+    constexpr uint64_t garbage_blob_bytes = 0;
+    auto new_meta = BlobFileMetaData::Create(
+        std::move(shared_meta), garbage_blob_count, garbage_blob_bytes);
+
+    changed_blob_files_.emplace(blob_file_number, std::move(new_meta));
 
     return Status::OK();
   }
@@ -448,181 +432,84 @@ class VersionBuilder::Rep {
   Status ApplyBlobFileGarbage(const BlobFileGarbage& blob_file_garbage) {
     const uint64_t blob_file_number = blob_file_garbage.GetBlobFileNumber();
 
-    if (!IsBlobFileInVersion(blob_file_number)) {
+    auto meta = GetBlobFileMetaData(blob_file_number);
+    if (!meta) {
       std::ostringstream oss;
       oss << "Blob file #" << blob_file_number << " not found";
 
       return Status::Corruption("VersionBuilder", oss.str());
     }
 
-    blob_file_meta_deltas_[blob_file_number].AddGarbage(
-        blob_file_garbage.GetGarbageBlobCount(),
-        blob_file_garbage.GetGarbageBlobBytes());
+    assert(meta->GetBlobFileNumber() == blob_file_number);
 
-    return Status::OK();
-  }
+    auto new_meta = BlobFileMetaData::Create(
+        meta->GetSharedMeta(),
+        meta->GetGarbageBlobCount() + blob_file_garbage.GetGarbageBlobCount(),
+        meta->GetGarbageBlobBytes() + blob_file_garbage.GetGarbageBlobBytes());
 
-  int GetCurrentLevelForTableFile(uint64_t file_number) const {
-    auto it = table_file_levels_.find(file_number);
-    if (it != table_file_levels_.end()) {
-      return it->second;
-    }
-
-    assert(base_vstorage_);
-    return base_vstorage_->GetFileLocation(file_number).GetLevel();
-  }
-
-  uint64_t GetOldestBlobFileNumberForTableFile(int level,
-                                               uint64_t file_number) const {
-    assert(level < num_levels_);
-
-    const auto& added_files = levels_[level].added_files;
-
-    auto it = added_files.find(file_number);
-    if (it != added_files.end()) {
-      const FileMetaData* const meta = it->second;
-      assert(meta);
-
-      return meta->oldest_blob_file_number;
-    }
-
-    assert(base_vstorage_);
-    const FileMetaData* const meta =
-        base_vstorage_->GetFileMetaDataByNumber(file_number);
-    assert(meta);
-
-    return meta->oldest_blob_file_number;
-  }
-
-  Status ApplyFileDeletion(int level, uint64_t file_number) {
-    assert(level != VersionStorageInfo::FileLocation::Invalid().GetLevel());
-
-    const int current_level = GetCurrentLevelForTableFile(file_number);
-
-    if (level != current_level) {
-      if (level >= num_levels_) {
-        has_invalid_levels_ = true;
-      }
-
-      std::ostringstream oss;
-      oss << "Cannot delete table file #" << file_number << " from level "
-          << level << " since it is ";
-      if (current_level ==
-          VersionStorageInfo::FileLocation::Invalid().GetLevel()) {
-        oss << "not in the LSM tree";
-      } else {
-        oss << "on level " << current_level;
-      }
-
-      return Status::Corruption("VersionBuilder", oss.str());
-    }
-
-    if (level >= num_levels_) {
-      assert(invalid_level_sizes_[level] > 0);
-      --invalid_level_sizes_[level];
-
-      table_file_levels_[file_number] =
-          VersionStorageInfo::FileLocation::Invalid().GetLevel();
-
-      return Status::OK();
-    }
-
-    const uint64_t blob_file_number =
-        GetOldestBlobFileNumberForTableFile(level, file_number);
-
-    if (blob_file_number != kInvalidBlobFileNumber &&
-        IsBlobFileInVersion(blob_file_number)) {
-      blob_file_meta_deltas_[blob_file_number].UnlinkSst(file_number);
-    }
-
-    auto& level_state = levels_[level];
-
-    auto& add_files = level_state.added_files;
-    auto add_it = add_files.find(file_number);
-    if (add_it != add_files.end()) {
-      UnrefFile(add_it->second);
-      add_files.erase(add_it);
-    }
-
-    auto& del_files = level_state.deleted_files;
-    assert(del_files.find(file_number) == del_files.end());
-    del_files.emplace(file_number);
-
-    table_file_levels_[file_number] =
-        VersionStorageInfo::FileLocation::Invalid().GetLevel();
-
-    return Status::OK();
-  }
-
-  Status ApplyFileAddition(int level, const FileMetaData& meta) {
-    assert(level != VersionStorageInfo::FileLocation::Invalid().GetLevel());
-
-    const uint64_t file_number = meta.fd.GetNumber();
-
-    const int current_level = GetCurrentLevelForTableFile(file_number);
-
-    if (current_level !=
-        VersionStorageInfo::FileLocation::Invalid().GetLevel()) {
-      if (level >= num_levels_) {
-        has_invalid_levels_ = true;
-      }
-
-      std::ostringstream oss;
-      oss << "Cannot add table file #" << file_number << " to level " << level
-          << " since it is already in the LSM tree on level " << current_level;
-      return Status::Corruption("VersionBuilder", oss.str());
-    }
-
-    if (level >= num_levels_) {
-      ++invalid_level_sizes_[level];
-      table_file_levels_[file_number] = level;
-
-      return Status::OK();
-    }
-
-    auto& level_state = levels_[level];
-
-    auto& del_files = level_state.deleted_files;
-    auto del_it = del_files.find(file_number);
-    if (del_it != del_files.end()) {
-      del_files.erase(del_it);
-    }
-
-    FileMetaData* const f = new FileMetaData(meta);
-    f->refs = 1;
-
-    auto& add_files = level_state.added_files;
-    assert(add_files.find(file_number) == add_files.end());
-    add_files.emplace(file_number, f);
-
-    const uint64_t blob_file_number = f->oldest_blob_file_number;
-
-    if (blob_file_number != kInvalidBlobFileNumber &&
-        IsBlobFileInVersion(blob_file_number)) {
-      blob_file_meta_deltas_[blob_file_number].LinkSst(file_number);
-    }
-
-    table_file_levels_[file_number] = level;
+    changed_blob_files_[blob_file_number] = std::move(new_meta);
 
     return Status::OK();
   }
 
   // Apply all of the edits in *edit to the current state.
   Status Apply(VersionEdit* edit) {
-    {
-      const Status s = CheckConsistency(base_vstorage_);
-      if (!s.ok()) {
-        return s;
+    Status s = CheckConsistency(base_vstorage_);
+    if (!s.ok()) {
+      return s;
+    }
+
+    // Delete files
+    const auto& del = edit->GetDeletedFiles();
+    for (const auto& del_file : del) {
+      const auto level = del_file.first;
+      const auto number = del_file.second;
+      if (level < num_levels_) {
+        levels_[level].deleted_files.insert(number);
+        s = CheckConsistencyForDeletes(edit, number, level);
+        if (!s.ok()) {
+          return s;
+        }
+
+        auto exising = levels_[level].added_files.find(number);
+        if (exising != levels_[level].added_files.end()) {
+          UnrefFile(exising->second);
+          levels_[level].added_files.erase(exising);
+        }
+      } else {
+        if (invalid_levels_[level].erase(number) == 0) {
+          // Deleting an non-existing file on invalid level.
+          has_invalid_levels_ = true;
+        }
       }
     }
 
-    // Note: we process the blob file related changes first because the
-    // table file addition/deletion logic depends on the blob files
-    // already being there.
+    // Add new files
+    for (const auto& new_file : edit->GetNewFiles()) {
+      const int level = new_file.first;
+      if (level < num_levels_) {
+        FileMetaData* f = new FileMetaData(new_file.second);
+        f->refs = 1;
+
+        assert(levels_[level].added_files.find(f->fd.GetNumber()) ==
+               levels_[level].added_files.end());
+        levels_[level].deleted_files.erase(f->fd.GetNumber());
+        levels_[level].added_files[f->fd.GetNumber()] = f;
+      } else {
+        uint64_t number = new_file.second.fd.GetNumber();
+        auto& lvls = invalid_levels_[level];
+        if (lvls.count(number) == 0) {
+          lvls.insert(number);
+        } else {
+          // Creating an already existing file on invalid level.
+          has_invalid_levels_ = true;
+        }
+      }
+    }
 
     // Add new blob files
     for (const auto& blob_file_addition : edit->GetBlobFileAdditions()) {
-      const Status s = ApplyBlobFileAddition(blob_file_addition);
+      s = ApplyBlobFileAddition(blob_file_addition);
       if (!s.ok()) {
         return s;
       }
@@ -630,96 +517,13 @@ class VersionBuilder::Rep {
 
     // Increase the amount of garbage for blob files affected by GC
     for (const auto& blob_file_garbage : edit->GetBlobFileGarbages()) {
-      const Status s = ApplyBlobFileGarbage(blob_file_garbage);
+      s = ApplyBlobFileGarbage(blob_file_garbage);
       if (!s.ok()) {
         return s;
       }
     }
 
-    // Delete table files
-    for (const auto& deleted_file : edit->GetDeletedFiles()) {
-      const int level = deleted_file.first;
-      const uint64_t file_number = deleted_file.second;
-
-      const Status s = ApplyFileDeletion(level, file_number);
-      if (!s.ok()) {
-        return s;
-      }
-    }
-
-    // Add new table files
-    for (const auto& new_file : edit->GetNewFiles()) {
-      const int level = new_file.first;
-      const FileMetaData& meta = new_file.second;
-
-      const Status s = ApplyFileAddition(level, meta);
-      if (!s.ok()) {
-        return s;
-      }
-    }
-
-    return Status::OK();
-  }
-
-  static BlobFileMetaData::LinkedSsts ApplyLinkedSstChanges(
-      const BlobFileMetaData::LinkedSsts& base,
-      const std::unordered_set<uint64_t>& newly_linked,
-      const std::unordered_set<uint64_t>& newly_unlinked) {
-    BlobFileMetaData::LinkedSsts result(base);
-
-    for (uint64_t sst_file_number : newly_unlinked) {
-      assert(result.find(sst_file_number) != result.end());
-
-      result.erase(sst_file_number);
-    }
-
-    for (uint64_t sst_file_number : newly_linked) {
-      assert(result.find(sst_file_number) == result.end());
-
-      result.emplace(sst_file_number);
-    }
-
-    return result;
-  }
-
-  static std::shared_ptr<BlobFileMetaData> CreateMetaDataForNewBlobFile(
-      const BlobFileMetaDataDelta& delta) {
-    auto shared_meta = delta.GetSharedMeta();
-    assert(shared_meta);
-
-    assert(delta.GetNewlyUnlinkedSsts().empty());
-
-    auto meta = BlobFileMetaData::Create(
-        std::move(shared_meta), delta.GetNewlyLinkedSsts(),
-        delta.GetAdditionalGarbageCount(), delta.GetAdditionalGarbageBytes());
-
-    return meta;
-  }
-
-  static std::shared_ptr<BlobFileMetaData>
-  GetOrCreateMetaDataForExistingBlobFile(
-      const std::shared_ptr<BlobFileMetaData>& base_meta,
-      const BlobFileMetaDataDelta& delta) {
-    assert(base_meta);
-    assert(!delta.GetSharedMeta());
-
-    if (delta.IsEmpty()) {
-      return base_meta;
-    }
-
-    auto shared_meta = base_meta->GetSharedMeta();
-    assert(shared_meta);
-
-    auto linked_ssts = ApplyLinkedSstChanges(base_meta->GetLinkedSsts(),
-                                             delta.GetNewlyLinkedSsts(),
-                                             delta.GetNewlyUnlinkedSsts());
-
-    auto meta = BlobFileMetaData::Create(
-        std::move(shared_meta), std::move(linked_ssts),
-        base_meta->GetGarbageBlobCount() + delta.GetAdditionalGarbageCount(),
-        base_meta->GetGarbageBlobBytes() + delta.GetAdditionalGarbageBytes());
-
-    return meta;
+    return s;
   }
 
   void AddBlobFileIfNeeded(
@@ -728,8 +532,7 @@ class VersionBuilder::Rep {
     assert(vstorage);
     assert(meta);
 
-    if (meta->GetGarbageBlobCount() < meta->GetTotalBlobCount() ||
-        !meta->GetLinkedSsts().empty()) {
+    if (meta->GetGarbageBlobCount() < meta->GetTotalBlobCount()) {
       vstorage->AddBlobFile(meta);
     }
   }
@@ -744,41 +547,42 @@ class VersionBuilder::Rep {
     auto base_it = base_blob_files.begin();
     const auto base_it_end = base_blob_files.end();
 
-    auto delta_it = blob_file_meta_deltas_.begin();
-    const auto delta_it_end = blob_file_meta_deltas_.end();
+    auto changed_it = changed_blob_files_.begin();
+    const auto changed_it_end = changed_blob_files_.end();
 
-    while (base_it != base_it_end && delta_it != delta_it_end) {
+    while (base_it != base_it_end && changed_it != changed_it_end) {
       const uint64_t base_blob_file_number = base_it->first;
-      const uint64_t delta_blob_file_number = delta_it->first;
+      const uint64_t changed_blob_file_number = changed_it->first;
 
-      if (base_blob_file_number < delta_blob_file_number) {
-        const auto& base_meta = base_it->second;
-        assert(base_meta);
+      const auto& base_meta = base_it->second;
+      const auto& changed_meta = changed_it->second;
+
+      assert(base_meta);
+      assert(changed_meta);
+
+      if (base_blob_file_number < changed_blob_file_number) {
         assert(base_meta->GetGarbageBlobCount() <
                base_meta->GetTotalBlobCount());
 
         vstorage->AddBlobFile(base_meta);
 
         ++base_it;
-      } else if (delta_blob_file_number < base_blob_file_number) {
-        // Note: blob file numbers are strictly increasing over time and
-        // once blob files get marked obsolete, they never reappear. Thus,
-        // this case is not possible.
-        assert(false);
+      } else if (changed_blob_file_number < base_blob_file_number) {
+        AddBlobFileIfNeeded(vstorage, changed_meta);
 
-        ++delta_it;
+        ++changed_it;
       } else {
-        assert(base_blob_file_number == delta_blob_file_number);
+        assert(base_blob_file_number == changed_blob_file_number);
+        assert(base_meta->GetSharedMeta() == changed_meta->GetSharedMeta());
+        assert(base_meta->GetGarbageBlobCount() <=
+               changed_meta->GetGarbageBlobCount());
+        assert(base_meta->GetGarbageBlobBytes() <=
+               changed_meta->GetGarbageBlobBytes());
 
-        const auto& base_meta = base_it->second;
-        const auto& delta = delta_it->second;
-
-        auto meta = GetOrCreateMetaDataForExistingBlobFile(base_meta, delta);
-
-        AddBlobFileIfNeeded(vstorage, meta);
+        AddBlobFileIfNeeded(vstorage, changed_meta);
 
         ++base_it;
-        ++delta_it;
+        ++changed_it;
       }
     }
 
@@ -791,14 +595,12 @@ class VersionBuilder::Rep {
       ++base_it;
     }
 
-    while (delta_it != delta_it_end) {
-      const auto& delta = delta_it->second;
+    while (changed_it != changed_it_end) {
+      const auto& changed_meta = changed_it->second;
+      assert(changed_meta);
 
-      auto meta = CreateMetaDataForNewBlobFile(delta);
-
-      AddBlobFileIfNeeded(vstorage, meta);
-
-      ++delta_it;
+      AddBlobFileIfNeeded(vstorage, changed_meta);
+      ++changed_it;
     }
   }
 
@@ -865,8 +667,7 @@ class VersionBuilder::Rep {
   Status LoadTableHandlers(InternalStats* internal_stats, int max_threads,
                            bool prefetch_index_and_filter_in_cache,
                            bool is_initial_load,
-                           const SliceTransform* prefix_extractor,
-                           size_t max_file_size_for_l0_meta_pin) {
+                           const SliceTransform* prefix_extractor) {
     assert(table_cache_ != nullptr);
 
     size_t table_cache_capacity = table_cache_->get_cache()->GetCapacity();
@@ -935,7 +736,7 @@ class VersionBuilder::Rep {
             file_meta->fd, &file_meta->table_reader_handle, prefix_extractor,
             false /*no_io */, true /* record_read_stats */,
             internal_stats->GetFileReadHist(level), false, level,
-            prefetch_index_and_filter_in_cache, max_file_size_for_l0_meta_pin);
+            prefetch_index_and_filter_in_cache);
         if (file_meta->table_reader_handle != nullptr) {
           // Load table_reader
           file_meta->fd.table_reader = table_cache_->GetTableReaderFromHandle(
@@ -961,28 +762,12 @@ class VersionBuilder::Rep {
   }
 
   void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f) {
-    const uint64_t file_number = f->fd.GetNumber();
-
-    const auto& level_state = levels_[level];
-
-    const auto& del_files = level_state.deleted_files;
-    const auto del_it = del_files.find(file_number);
-
-    if (del_it != del_files.end()) {
+    if (levels_[level].deleted_files.count(f->fd.GetNumber()) > 0) {
       // f is to-be-deleted table file
       vstorage->RemoveCurrentStats(f);
     } else {
-      const auto& add_files = level_state.added_files;
-      const auto add_it = add_files.find(file_number);
-
-      // Note: if the file appears both in the base version and in the added
-      // list, the added FileMetaData supersedes the one in the base version.
-      if (add_it != add_files.end() && add_it->second != f) {
-        vstorage->RemoveCurrentStats(f);
-      } else {
-        assert(ioptions_);
-        vstorage->AddFile(level, f, ioptions_->info_log);
-      }
+      assert(ioptions_);
+      vstorage->AddFile(level, f, ioptions_->info_log);
     }
   }
 };
@@ -1010,11 +795,10 @@ Status VersionBuilder::SaveTo(VersionStorageInfo* vstorage) {
 Status VersionBuilder::LoadTableHandlers(
     InternalStats* internal_stats, int max_threads,
     bool prefetch_index_and_filter_in_cache, bool is_initial_load,
-    const SliceTransform* prefix_extractor,
-    size_t max_file_size_for_l0_meta_pin) {
-  return rep_->LoadTableHandlers(
-      internal_stats, max_threads, prefetch_index_and_filter_in_cache,
-      is_initial_load, prefix_extractor, max_file_size_for_l0_meta_pin);
+    const SliceTransform* prefix_extractor) {
+  return rep_->LoadTableHandlers(internal_stats, max_threads,
+                                 prefetch_index_and_filter_in_cache,
+                                 is_initial_load, prefix_extractor);
 }
 
 BaseReferencedVersionBuilder::BaseReferencedVersionBuilder(

db/version_builder.h

@@ -42,8 +42,7 @@ class VersionBuilder {
   Status LoadTableHandlers(InternalStats* internal_stats, int max_threads,
                            bool prefetch_index_and_filter_in_cache,
                            bool is_initial_load,
-                           const SliceTransform* prefix_extractor,
-                           size_t max_file_size_for_l0_meta_pin);
+                           const SliceTransform* prefix_extractor);
 
  private:
   class Rep;

db/version_builder_test.cc

@@ -4,11 +4,8 @@
 //  (found in the LICENSE.Apache file in the root directory).
 
 #include <cstring>
-#include <iomanip>
 #include <memory>
-#include <sstream>
 #include <string>
-
 #include "db/version_edit.h"
 #include "db/version_set.h"
 #include "logging/logging.h"
@@ -57,7 +54,7 @@ class VersionBuilderTest : public testing::Test {
     return InternalKey(ukey, smallest_seq, kTypeValue);
   }
 
-  void Add(int level, uint64_t file_number, const char* smallest,
+  void Add(int level, uint32_t file_number, const char* smallest,
            const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
            SequenceNumber smallest_seq = 100, SequenceNumber largest_seq = 100,
            uint64_t num_entries = 0, uint64_t num_deletions = 0,
@@ -83,15 +80,13 @@ class VersionBuilderTest : public testing::Test {
 
   void AddBlob(uint64_t blob_file_number, uint64_t total_blob_count,
                uint64_t total_blob_bytes, std::string checksum_method,
-               std::string checksum_value,
-               BlobFileMetaData::LinkedSsts linked_ssts,
-               uint64_t garbage_blob_count, uint64_t garbage_blob_bytes) {
+               std::string checksum_value, uint64_t garbage_blob_count,
+               uint64_t garbage_blob_bytes) {
     auto shared_meta = SharedBlobFileMetaData::Create(
         blob_file_number, total_blob_count, total_blob_bytes,
         std::move(checksum_method), std::move(checksum_value));
-    auto meta =
-        BlobFileMetaData::Create(std::move(shared_meta), std::move(linked_ssts),
-                                 garbage_blob_count, garbage_blob_bytes);
+    auto meta = BlobFileMetaData::Create(
+        std::move(shared_meta), garbage_blob_count, garbage_blob_bytes);
 
     vstorage_.AddBlobFile(std::move(meta));
   }
@@ -372,242 +367,6 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
   UnrefFilesInVersion(&new_vstorage);
 }
 
-TEST_F(VersionBuilderTest, ApplyFileDeletionIncorrectLevel) {
-  constexpr int level = 1;
-  constexpr uint64_t file_number = 2345;
-  constexpr char smallest[] = "bar";
-  constexpr char largest[] = "foo";
-
-  Add(level, file_number, smallest, largest);
-
-  EnvOptions env_options;
-  constexpr TableCache* table_cache = nullptr;
-  constexpr VersionSet* version_set = nullptr;
-
-  VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_,
-                         version_set);
-
-  VersionEdit edit;
-
-  constexpr int incorrect_level = 3;
-
-  edit.DeleteFile(incorrect_level, file_number);
-
-  const Status s = builder.Apply(&edit);
-  ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(std::strstr(s.getState(),
-                          "Cannot delete table file #2345 from level 3 since "
-                          "it is on level 1"));
-}
-
-TEST_F(VersionBuilderTest, ApplyFileDeletionNotInLSMTree) {
-  EnvOptions env_options;
-  constexpr TableCache* table_cache = nullptr;
-  constexpr VersionSet* version_set = nullptr;
-
-  VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_,
-                         version_set);
-
-  VersionEdit edit;
-
-  constexpr int level = 3;
-  constexpr uint64_t file_number = 1234;
-
-  edit.DeleteFile(level, file_number);
-
-  const Status s = builder.Apply(&edit);
-  ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(std::strstr(s.getState(),
-                          "Cannot delete table file #1234 from level 3 since "
-                          "it is not in the LSM tree"));
-}
-
-TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) {
-  constexpr int level = 1;
-  constexpr uint64_t file_number = 2345;
-  constexpr char smallest[] = "bar";
-  constexpr char largest[] = "foo";
-  constexpr uint64_t file_size = 10000;
-  constexpr uint32_t path_id = 0;
-  constexpr SequenceNumber smallest_seq = 100;
-  constexpr SequenceNumber largest_seq = 500;
-  constexpr uint64_t num_entries = 0;
-  constexpr uint64_t num_deletions = 0;
-  constexpr bool sampled = false;
-  constexpr SequenceNumber smallest_seqno = 1;
-  constexpr SequenceNumber largest_seqno = 1000;
-
-  Add(level, file_number, smallest, largest, file_size, path_id, smallest_seq,
-      largest_seq, num_entries, num_deletions, sampled, smallest_seqno,
-      largest_seqno);
-
-  EnvOptions env_options;
-  constexpr TableCache* table_cache = nullptr;
-  constexpr VersionSet* version_set = nullptr;
-
-  VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_,
-                         version_set);
-
-  VersionEdit deletion;
-
-  deletion.DeleteFile(level, file_number);
-
-  ASSERT_OK(builder.Apply(&deletion));
-
-  VersionEdit addition;
-
-  constexpr bool marked_for_compaction = false;
-
-  addition.AddFile(level, file_number, path_id, file_size,
-                   GetInternalKey(smallest, smallest_seq),
-                   GetInternalKey(largest, largest_seq), smallest_seqno,
-                   largest_seqno, marked_for_compaction, kInvalidBlobFileNumber,
-                   kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-                   kUnknownFileChecksum, kUnknownFileChecksumFuncName);
-
-  ASSERT_OK(builder.Apply(&addition));
-
-  constexpr bool force_consistency_checks = false;
-  VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
-                                  kCompactionStyleLevel, &vstorage_,
-                                  force_consistency_checks);
-
-  ASSERT_OK(builder.SaveTo(&new_vstorage));
-  ASSERT_EQ(new_vstorage.GetFileLocation(file_number).GetLevel(), level);
-
-  UnrefFilesInVersion(&new_vstorage);
-}
-
-TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyInBase) {
-  constexpr int level = 1;
-  constexpr uint64_t file_number = 2345;
-  constexpr char smallest[] = "bar";
-  constexpr char largest[] = "foo";
-
-  Add(level, file_number, smallest, largest);
-
-  EnvOptions env_options;
-  constexpr TableCache* table_cache = nullptr;
-  constexpr VersionSet* version_set = nullptr;
-
-  VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_,
-                         version_set);
-
-  VersionEdit edit;
-
-  constexpr int new_level = 2;
-  constexpr uint32_t path_id = 0;
-  constexpr uint64_t file_size = 10000;
-  constexpr SequenceNumber smallest_seqno = 100;
-  constexpr SequenceNumber largest_seqno = 1000;
-  constexpr bool marked_for_compaction = false;
-
-  edit.AddFile(new_level, file_number, path_id, file_size,
-               GetInternalKey(smallest), GetInternalKey(largest),
-               smallest_seqno, largest_seqno, marked_for_compaction,
-               kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
-               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
-
-  const Status s = builder.Apply(&edit);
-  ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(std::strstr(s.getState(),
-                          "Cannot add table file #2345 to level 2 since it is "
-                          "already in the LSM tree on level 1"));
-}
-
-TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) {
-  EnvOptions env_options;
-  constexpr TableCache* table_cache = nullptr;
-  constexpr VersionSet* version_set = nullptr;
-
-  VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_,
-                         version_set);
-
-  VersionEdit edit;
-
-  constexpr int level = 3;
-  constexpr uint64_t file_number = 2345;
-  constexpr uint32_t path_id = 0;
-  constexpr uint64_t file_size = 10000;
-  constexpr char smallest[] = "bar";
-  constexpr char largest[] = "foo";
-  constexpr SequenceNumber smallest_seqno = 100;
-  constexpr SequenceNumber largest_seqno = 1000;
-  constexpr bool marked_for_compaction = false;
-
-  edit.AddFile(level, file_number, path_id, file_size, GetInternalKey(smallest),
-               GetInternalKey(largest), smallest_seqno, largest_seqno,
-               marked_for_compaction, kInvalidBlobFileNumber,
-               kUnknownOldestAncesterTime, kUnknownFileCreationTime,
-               kUnknownFileChecksum, kUnknownFileChecksumFuncName);
-
-  ASSERT_OK(builder.Apply(&edit));
-
-  VersionEdit other_edit;
-
-  constexpr int new_level = 2;
-
-  other_edit.AddFile(new_level, file_number, path_id, file_size,
-                     GetInternalKey(smallest), GetInternalKey(largest),
-                     smallest_seqno, largest_seqno, marked_for_compaction,
-                     kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
-                     kUnknownFileCreationTime, kUnknownFileChecksum,
-                     kUnknownFileChecksumFuncName);
-
-  const Status s = builder.Apply(&other_edit);
-  ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(std::strstr(s.getState(),
-                          "Cannot add table file #2345 to level 2 since it is "
-                          "already in the LSM tree on level 3"));
-}
-
-TEST_F(VersionBuilderTest, ApplyFileAdditionAndDeletion) {
-  constexpr int level = 1;
-  constexpr uint64_t file_number = 2345;
-  constexpr uint32_t path_id = 0;
-  constexpr uint64_t file_size = 10000;
-  constexpr char smallest[] = "bar";
-  constexpr char largest[] = "foo";
-  constexpr SequenceNumber smallest_seqno = 100;
-  constexpr SequenceNumber largest_seqno = 1000;
-  constexpr bool marked_for_compaction = false;
-
-  EnvOptions env_options;
-  constexpr TableCache* table_cache = nullptr;
-  constexpr VersionSet* version_set = nullptr;
-
-  VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_,
-                         version_set);
-
-  VersionEdit addition;
-
-  addition.AddFile(level, file_number, path_id, file_size,
-                   GetInternalKey(smallest), GetInternalKey(largest),
-                   smallest_seqno, largest_seqno, marked_for_compaction,
-                   kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
-                   kUnknownFileCreationTime, kUnknownFileChecksum,
-                   kUnknownFileChecksumFuncName);
-
-  ASSERT_OK(builder.Apply(&addition));
-
-  VersionEdit deletion;
-
-  deletion.DeleteFile(level, file_number);
-
-  ASSERT_OK(builder.Apply(&deletion));
-
-  constexpr bool force_consistency_checks = false;
-  VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
-                                  kCompactionStyleLevel, &vstorage_,
-                                  force_consistency_checks);
-
-  ASSERT_OK(builder.SaveTo(&new_vstorage));
-  ASSERT_FALSE(new_vstorage.GetFileLocation(file_number).IsValid());
-
-  UnrefFilesInVersion(&new_vstorage);
-}
-
 TEST_F(VersionBuilderTest, ApplyBlobFileAddition) {
   EnvOptions env_options;
   constexpr TableCache* table_cache = nullptr;
@@ -647,7 +406,6 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAddition) {
   ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes);
   ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method);
   ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value);
-  ASSERT_TRUE(new_meta->GetLinkedSsts().empty());
   ASSERT_EQ(new_meta->GetGarbageBlobCount(), 0);
   ASSERT_EQ(new_meta->GetGarbageBlobBytes(), 0);
 }
@@ -664,8 +422,7 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyInBase) {
   constexpr uint64_t garbage_blob_bytes = 456789;
 
   AddBlob(blob_file_number, total_blob_count, total_blob_bytes, checksum_method,
-          checksum_value, BlobFileMetaData::LinkedSsts(), garbage_blob_count,
-          garbage_blob_bytes);
+          checksum_value, garbage_blob_count, garbage_blob_bytes);
 
   EnvOptions env_options;
   constexpr TableCache* table_cache = nullptr;
@@ -724,8 +481,7 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) {
   constexpr uint64_t garbage_blob_bytes = 456789;
 
   AddBlob(blob_file_number, total_blob_count, total_blob_bytes, checksum_method,
-          checksum_value, BlobFileMetaData::LinkedSsts(), garbage_blob_count,
-          garbage_blob_bytes);
+          checksum_value, garbage_blob_count, garbage_blob_bytes);
 
   const auto meta =
       GetBlobFileMetaData(vstorage_.GetBlobFiles(), blob_file_number);
@@ -767,7 +523,6 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) {
   ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes);
   ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method);
   ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value);
-  ASSERT_TRUE(new_meta->GetLinkedSsts().empty());
   ASSERT_EQ(new_meta->GetGarbageBlobCount(),
             garbage_blob_count + new_garbage_blob_count);
   ASSERT_EQ(new_meta->GetGarbageBlobBytes(),
@@ -825,7 +580,6 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileAdditionApplied) {
   ASSERT_EQ(new_meta->GetTotalBlobBytes(), total_blob_bytes);
   ASSERT_EQ(new_meta->GetChecksumMethod(), checksum_method);
   ASSERT_EQ(new_meta->GetChecksumValue(), checksum_value);
-  ASSERT_TRUE(new_meta->GetLinkedSsts().empty());
   ASSERT_EQ(new_meta->GetGarbageBlobCount(), garbage_blob_count);
   ASSERT_EQ(new_meta->GetGarbageBlobBytes(), garbage_blob_bytes);
 }
@@ -866,8 +620,8 @@ TEST_F(VersionBuilderTest, SaveBlobFilesTo) {
 
     AddBlob(blob_file_number, total_blob_count, total_blob_bytes,
             /* checksum_method */ std::string(),
-            /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts(),
-            garbage_blob_count, garbage_blob_bytes);
+            /* checksum_value */ std::string(), garbage_blob_count,
+            garbage_blob_bytes);
   }
 
   EnvOptions env_options;
@@ -956,7 +710,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
   AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000,
           /* total_blob_bytes */ 1000000,
           /* checksum_method */ std::string(),
-          /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1},
+          /* checksum_value */ std::string(),
           /* garbage_blob_count */ 500, /* garbage_blob_bytes */ 300000);
 
   UpdateVersionStorageInfo();
@@ -1005,9 +759,9 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
   UnrefFilesInVersion(&new_vstorage);
 }
 
-TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesInconsistentLinks) {
-  // Initialize base version. Links between the table file and the blob file
-  // are inconsistent.
+TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesNotInVersion) {
+  // Initialize base version. The table file points to a blob file that is
+  // not in this version.
 
   Add(/* level */ 1, /* file_number */ 1, /* smallest */ "150",
       /* largest */ "200", /* file_size */ 100,
@@ -1019,7 +773,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesInconsistentLinks) {
   AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000,
           /* total_blob_bytes */ 1000000,
           /* checksum_method */ std::string(),
-          /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1},
+          /* checksum_value */ std::string(),
           /* garbage_blob_count */ 500, /* garbage_blob_bytes */ 300000);
 
   UpdateVersionStorageInfo();
@@ -1039,9 +793,8 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesInconsistentLinks) {
 
   const Status s = builder.SaveTo(&new_vstorage);
   ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(std::strstr(
-      s.getState(),
-      "Links are inconsistent between table files and blob file #16"));
+  ASSERT_TRUE(
+      std::strstr(s.getState(), "Blob file #256 is not part of this version"));
 
   UnrefFilesInVersion(&new_vstorage);
 }
@@ -1060,7 +813,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbage) {
   AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000,
           /* total_blob_bytes */ 1000000,
           /* checksum_method */ std::string(),
-          /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1},
+          /* checksum_value */ std::string(),
           /* garbage_blob_count */ 1000, /* garbage_blob_bytes */ 1000000);
 
   UpdateVersionStorageInfo();
@@ -1086,224 +839,6 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbage) {
   UnrefFilesInVersion(&new_vstorage);
 }
 
-TEST_F(VersionBuilderTest, CheckConsistencyForBlobFilesAllGarbageLinkedSsts) {
-  // Initialize base version, with a table file pointing to a blob file
-  // that has no garbage at this point.
-
-  Add(/* level */ 1, /* file_number */ 1, /* smallest */ "150",
-      /* largest */ "200", /* file_size */ 100,
-      /* path_id */ 0, /* smallest_seq */ 100, /* largest_seq */ 100,
-      /* num_entries */ 0, /* num_deletions */ 0,
-      /* sampled */ false, /* smallest_seqno */ 100, /* largest_seqno */ 100,
-      /* oldest_blob_file_number */ 16);
-
-  AddBlob(/* blob_file_number */ 16, /* total_blob_count */ 1000,
-          /* total_blob_bytes */ 1000000,
-          /* checksum_method */ std::string(),
-          /* checksum_value */ std::string(), BlobFileMetaData::LinkedSsts{1},
-          /* garbage_blob_count */ 0, /* garbage_blob_bytes */ 0);
-
-  UpdateVersionStorageInfo();
-
-  // Mark the entire blob file garbage but do not remove the linked SST.
-  EnvOptions env_options;
-  constexpr TableCache* table_cache = nullptr;
-  constexpr VersionSet* version_set = nullptr;
-
-  VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_,
-                         version_set);
-
-  VersionEdit edit;
-
-  edit.AddBlobFileGarbage(/* blob_file_number */ 16,
-                          /* garbage_blob_count */ 1000,
-                          /* garbage_blob_bytes */ 1000000);
-
-  ASSERT_OK(builder.Apply(&edit));
-
-  // Save to a new version in order to trigger consistency checks.
-  constexpr bool force_consistency_checks = true;
-  VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
-                                  kCompactionStyleLevel, &vstorage_,
-                                  force_consistency_checks);
-
-  const Status s = builder.SaveTo(&new_vstorage);
-  ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(
-      std::strstr(s.getState(), "Blob file #16 consists entirely of garbage"));
-
-  UnrefFilesInVersion(&new_vstorage);
-}
-
-TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
-  // Initialize base version. Table files 1..10 are linked to blob files 1..5,
-  // while table files 11..20 are not linked to any blob files.
-
-  for (uint64_t i = 1; i <= 10; ++i) {
-    std::ostringstream oss;
-    oss << std::setw(2) << std::setfill('0') << i;
-
-    const std::string key = oss.str();
-
-    Add(/* level */ 1, /* file_number */ i, /* smallest */ key.c_str(),
-        /* largest */ key.c_str(), /* file_size */ 100,
-        /* path_id */ 0, /* smallest_seq */ i * 100, /* largest_seq */ i * 100,
-        /* num_entries */ 0, /* num_deletions */ 0,
-        /* sampled */ false, /* smallest_seqno */ i * 100,
-        /* largest_seqno */ i * 100,
-        /* oldest_blob_file_number */ ((i - 1) % 5) + 1);
-  }
-
-  for (uint64_t i = 1; i <= 5; ++i) {
-    AddBlob(/* blob_file_number */ i, /* total_blob_count */ 2000,
-            /* total_blob_bytes */ 2000000,
-            /* checksum_method */ std::string(),
-            /* checksum_value */ std::string(),
-            BlobFileMetaData::LinkedSsts{i, i + 5},
-            /* garbage_blob_count */ 1000, /* garbage_blob_bytes */ 1000000);
-  }
-
-  for (uint64_t i = 11; i <= 20; ++i) {
-    std::ostringstream oss;
-    oss << std::setw(2) << std::setfill('0') << i;
-
-    const std::string key = oss.str();
-
-    Add(/* level */ 1, /* file_number */ i, /* smallest */ key.c_str(),
-        /* largest */ key.c_str(), /* file_size */ 100,
-        /* path_id */ 0, /* smallest_seq */ i * 100, /* largest_seq */ i * 100,
-        /* num_entries */ 0, /* num_deletions */ 0,
-        /* sampled */ false, /* smallest_seqno */ i * 100,
-        /* largest_seqno */ i * 100, kInvalidBlobFileNumber);
-  }
-
-  UpdateVersionStorageInfo();
-
-  {
-    const auto& blob_files = vstorage_.GetBlobFiles();
-    ASSERT_EQ(blob_files.size(), 5);
-
-    const std::vector<BlobFileMetaData::LinkedSsts> expected_linked_ssts{
-        {1, 6}, {2, 7}, {3, 8}, {4, 9}, {5, 10}};
-
-    for (size_t i = 0; i < 5; ++i) {
-      const auto meta =
-          GetBlobFileMetaData(blob_files, /* blob_file_number */ i + 1);
-      ASSERT_NE(meta, nullptr);
-      ASSERT_EQ(meta->GetLinkedSsts(), expected_linked_ssts[i]);
-    }
-  }
-
-  VersionEdit edit;
-
-  // Add an SST that references a blob file.
-  edit.AddFile(
-      /* level */ 1, /* file_number */ 21, /* path_id */ 0,
-      /* file_size */ 100, /* smallest */ GetInternalKey("21", 2100),
-      /* largest */ GetInternalKey("21", 2100), /* smallest_seqno */ 2100,
-      /* largest_seqno */ 2100, /* marked_for_compaction */ false,
-      /* oldest_blob_file_number */ 1, kUnknownOldestAncesterTime,
-      kUnknownFileCreationTime, kUnknownFileChecksum,
-      kUnknownFileChecksumFuncName);
-
-  // Add an SST that does not reference any blob files.
-  edit.AddFile(
-      /* level */ 1, /* file_number */ 22, /* path_id */ 0,
-      /* file_size */ 100, /* smallest */ GetInternalKey("22", 2200),
-      /* largest */ GetInternalKey("22", 2200), /* smallest_seqno */ 2200,
-      /* largest_seqno */ 2200, /* marked_for_compaction */ false,
-      kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
-      kUnknownFileCreationTime, kUnknownFileChecksum,
-      kUnknownFileChecksumFuncName);
-
-  // Delete a file that references a blob file.
-  edit.DeleteFile(/* level */ 1, /* file_number */ 6);
-
-  // Delete a file that does not reference any blob files.
-  edit.DeleteFile(/* level */ 1, /* file_number */ 16);
-
-  // Trivially move a file that references a blob file. Note that we save
-  // the original BlobFileMetaData object so we can check that no new object
-  // gets created.
-  auto meta3 =
-      GetBlobFileMetaData(vstorage_.GetBlobFiles(), /* blob_file_number */ 3);
-
-  edit.DeleteFile(/* level */ 1, /* file_number */ 3);
-  edit.AddFile(/* level */ 2, /* file_number */ 3, /* path_id */ 0,
-               /* file_size */ 100, /* smallest */ GetInternalKey("03", 300),
-               /* largest */ GetInternalKey("03", 300),
-               /* smallest_seqno */ 300,
-               /* largest_seqno */ 300, /* marked_for_compaction */ false,
-               /* oldest_blob_file_number */ 3, kUnknownOldestAncesterTime,
-               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
-
-  // Trivially move a file that does not reference any blob files.
-  edit.DeleteFile(/* level */ 1, /* file_number */ 13);
-  edit.AddFile(/* level */ 2, /* file_number */ 13, /* path_id */ 0,
-               /* file_size */ 100, /* smallest */ GetInternalKey("13", 1300),
-               /* largest */ GetInternalKey("13", 1300),
-               /* smallest_seqno */ 1300,
-               /* largest_seqno */ 1300, /* marked_for_compaction */ false,
-               kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
-               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
-
-  // Add one more SST file that references a blob file, then promptly
-  // delete it in a second version edit before the new version gets saved.
-  // This file should not show up as linked to the blob file in the new version.
-  edit.AddFile(/* level */ 1, /* file_number */ 23, /* path_id */ 0,
-               /* file_size */ 100, /* smallest */ GetInternalKey("23", 2300),
-               /* largest */ GetInternalKey("23", 2300),
-               /* smallest_seqno */ 2300,
-               /* largest_seqno */ 2300, /* marked_for_compaction */ false,
-               /* oldest_blob_file_number */ 5, kUnknownOldestAncesterTime,
-               kUnknownFileCreationTime, kUnknownFileChecksum,
-               kUnknownFileChecksumFuncName);
-
-  VersionEdit edit2;
-
-  edit2.DeleteFile(/* level */ 1, /* file_number */ 23);
-
-  EnvOptions env_options;
-  constexpr TableCache* table_cache = nullptr;
-  constexpr VersionSet* version_set = nullptr;
-
-  VersionBuilder builder(env_options, &ioptions_, table_cache, &vstorage_,
-                         version_set);
-
-  ASSERT_OK(builder.Apply(&edit));
-  ASSERT_OK(builder.Apply(&edit2));
-
-  constexpr bool force_consistency_checks = true;
-  VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
-                                  kCompactionStyleLevel, &vstorage_,
-                                  force_consistency_checks);
-
-  ASSERT_OK(builder.SaveTo(&new_vstorage));
-
-  {
-    const auto& blob_files = new_vstorage.GetBlobFiles();
-    ASSERT_EQ(blob_files.size(), 5);
-
-    const std::vector<BlobFileMetaData::LinkedSsts> expected_linked_ssts{
-        {1, 21}, {2, 7}, {3, 8}, {4, 9}, {5, 10}};
-
-    for (size_t i = 0; i < 5; ++i) {
-      const auto meta =
-          GetBlobFileMetaData(blob_files, /* blob_file_number */ i + 1);
-      ASSERT_NE(meta, nullptr);
-      ASSERT_EQ(meta->GetLinkedSsts(), expected_linked_ssts[i]);
-    }
-
-    // Make sure that no new BlobFileMetaData got created for the blob file
-    // affected by the trivial move.
-    ASSERT_EQ(GetBlobFileMetaData(blob_files, /* blob_file_number */ 3), meta3);
-  }
-
-  UnrefFilesInVersion(&new_vstorage);
-}
-
 TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) {
   Add(0, 1U, "150", "200", 100U);
   UpdateVersionStorageInfo();
@@ -1324,10 +859,10 @@ TEST_F(VersionBuilderTest, CheckConsistencyForFileDeletedTwice) {
   ASSERT_OK(version_builder.SaveTo(&new_vstorage));
 
   VersionBuilder version_builder2(env_options, &ioptions_, table_cache,
-                                 &new_vstorage, version_set);
+                                  &new_vstorage, version_set);
   VersionStorageInfo new_vstorage2(&icmp_, ucmp_, options_.num_levels,
-                                  kCompactionStyleLevel, nullptr,
-                                  true /* force_consistency_checks */);
+                                   kCompactionStyleLevel, nullptr,
+                                   true /* force_consistency_checks */);
   ASSERT_NOK(version_builder2.Apply(&version_edit));
 
   UnrefFilesInVersion(&new_vstorage);

db/version_edit.cc

@@ -18,6 +18,10 @@
 #include "util/string_util.h"
 
 namespace ROCKSDB_NAMESPACE {
+// The unknown file checksum.
+const std::string kUnknownFileChecksum("");
+// The unknown sst file checksum function name.
+const std::string kUnknownFileChecksumFuncName("Unknown");
 
 namespace {
 

db/version_edit.h

@@ -29,6 +29,9 @@ constexpr uint64_t kFileNumberMask = 0x3FFFFFFFFFFFFFFF;
 constexpr uint64_t kUnknownOldestAncesterTime = 0;
 constexpr uint64_t kUnknownFileCreationTime = 0;
 
+extern const std::string kUnknownFileChecksum;
+extern const std::string kUnknownFileChecksumFuncName;
+
 extern uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id);
 
 // A copyable structure contains information needed to read data from an SST

db/version_edit_handler.cc

@@ -420,8 +420,7 @@ Status VersionEditHandler::LoadTables(ColumnFamilyData* cfd,
       cfd->internal_stats(),
       version_set_->db_options_->max_file_opening_threads,
       prefetch_index_and_filter_in_cache, is_initial_load,
-      cfd->GetLatestMutableCFOptions()->prefix_extractor.get(),
-      MaxFileSizeForL0MetaPin(*cfd->GetLatestMutableCFOptions()));
+      cfd->GetLatestMutableCFOptions()->prefix_extractor.get());
   if ((s.IsPathNotFound() || s.IsCorruption()) &&
       no_error_if_table_files_missing_) {
     s = Status::OK();

db/version_set.cc

@@ -368,7 +368,6 @@ class FilePickerMultiGet {
         range_(range),
         batch_iter_(range->begin()),
         batch_iter_prev_(range->begin()),
-        upper_key_(range->begin()),
         maybe_repeat_key_(false),
         current_level_range_(*range, range->begin(), range->end()),
         current_file_range_(*range, range->begin(), range->end()),
@@ -485,19 +484,9 @@ class FilePickerMultiGet {
       }
       if (cmp_largest == 0) {
         // cmp_largest is 0, which means the next key will not be in this
-        // file, so stop looking further. However, its possible there are
-        // duplicates in the batch, so find the upper bound for the batch
-        // in this file (upper_key_) by skipping past the duplicates. We
-        // leave batch_iter_ as is since we may have to pick up from there
-        // for the next file, if this file has a merge value rather than
-        // final value
-        upper_key_ = batch_iter_;
-        ++upper_key_;
-        while (upper_key_ != current_level_range_.end() &&
-               user_comparator_->Compare(batch_iter_->ukey, upper_key_->ukey) ==
-                   0) {
-          ++upper_key_;
-        }
+        // file, so stop looking further. Also don't increment megt_iter_
+        // as we may have to look for this key in the next file if we don't
+        // find it in this one
         break;
       } else {
         if (curr_level_ == 0) {
@@ -517,12 +506,6 @@ class FilePickerMultiGet {
     *fd = f;
     *file_index = curr_file_index;
     *is_last_key_in_file = cmp_largest == 0;
-    if (!*is_last_key_in_file) {
-      // If the largest key in the batch overlapping the file is not the
-      // largest key in the file, upper_ley_ would not have been updated so
-      // update it here
-      upper_key_ = batch_iter_;
-    }
     return file_hit;
   }
 
@@ -544,7 +527,7 @@ class FilePickerMultiGet {
           // file regardless for all keys not found yet
           if (current_level_range_.CheckKeyDone(batch_iter_) ||
               curr_level_ == 0) {
-            batch_iter_ = upper_key_;
+            ++batch_iter_;
           }
         }
         // batch_iter_prev_ will become the start key for the next file
@@ -564,20 +547,18 @@ class FilePickerMultiGet {
                                       &is_last_key_in_file)) {
         search_ended_ = !PrepareNextLevel();
       } else {
+        MultiGetRange::Iterator upper_key = batch_iter_;
         if (is_last_key_in_file) {
           // Since cmp_largest is 0, batch_iter_ still points to the last key
           // that falls in this file, instead of the next one. Increment
-          // the file index for all keys between batch_iter_ and upper_key_
-          auto tmp_iter = batch_iter_;
-          while (tmp_iter != upper_key_) {
-            ++(fp_ctx_array_[tmp_iter.index()].curr_index_in_curr_level);
-            ++tmp_iter;
-          }
+          // upper_key so we can set the range properly for SST MultiGet
+          ++upper_key;
+          ++(fp_ctx_array_[batch_iter_.index()].curr_index_in_curr_level);
           maybe_repeat_key_ = true;
         }
         // Set the range for this file
         current_file_range_ =
-            MultiGetRange(next_file_range, batch_iter_prev_, upper_key_);
+            MultiGetRange(next_file_range, batch_iter_prev_, upper_key);
         returned_file_level_ = curr_level_;
         hit_file_level_ = curr_level_;
         is_hit_file_last_in_level_ =
@@ -629,7 +610,6 @@ class FilePickerMultiGet {
   // key found in the previous SST file, in order to serve as the start of
   // the batch key range for the next SST file
   MultiGetRange::Iterator batch_iter_prev_;
-  MultiGetRange::Iterator upper_key_;
   bool maybe_repeat_key_;
   MultiGetRange current_level_range_;
   MultiGetRange current_file_range_;
@@ -649,7 +629,7 @@ class FilePickerMultiGet {
       if (fp_ctx_array_[mget_iter.index()].curr_index_in_curr_level <
           curr_file_level_->num_files) {
         batch_iter_prev_ = current_level_range_.begin();
-        upper_key_ = batch_iter_ = current_level_range_.begin();
+        batch_iter_ = current_level_range_.begin();
         return true;
       }
     }
@@ -744,7 +724,7 @@ class FilePickerMultiGet {
       }
       if (level_contains_keys) {
         batch_iter_prev_ = current_level_range_.begin();
-        upper_key_ = batch_iter_ = current_level_range_.begin();
+        batch_iter_ = current_level_range_.begin();
         return true;
       }
       curr_level_++;
@@ -1007,8 +987,7 @@ class LevelIterator final : public InternalIterator {
         read_options_, file_options_, icomparator_, *file_meta.file_metadata,
         range_del_agg_, prefix_extractor_,
         nullptr /* don't need reference to table */, file_read_hist_, caller_,
-        /*arena=*/nullptr, skip_filters_, level_,
-        /*max_file_size_for_l0_meta_pin=*/0, smallest_compaction_key,
+        /*arena=*/nullptr, skip_filters_, level_, smallest_compaction_key,
         largest_compaction_key, allow_unprepared_value_);
   }
 
@@ -1603,9 +1582,10 @@ void Version::AddIteratorsForLevel(const ReadOptions& read_options,
           mutable_cf_options_.prefix_extractor.get(), nullptr,
           cfd_->internal_stats()->GetFileReadHist(0),
           TableReaderCaller::kUserIterator, arena,
-          /*skip_filters=*/false, /*level=*/0, max_file_size_for_l0_meta_pin_,
+          /*skip_filters=*/false, /*level=*/0,
           /*smallest_compaction_key=*/nullptr,
-          /*largest_compaction_key=*/nullptr, allow_unprepared_value));
+          /*largest_compaction_key=*/nullptr,
+          allow_unprepared_value));
     }
     if (should_sample) {
       // Count ones for every L0 files. This is done per iterator creation
@@ -1627,8 +1607,8 @@ void Version::AddIteratorsForLevel(const ReadOptions& read_options,
         mutable_cf_options_.prefix_extractor.get(), should_sample_file_read(),
         cfd_->internal_stats()->GetFileReadHist(level),
         TableReaderCaller::kUserIterator, IsFilterSkipped(level), level,
-        range_del_agg,
-        /*compaction_boundaries=*/nullptr, allow_unprepared_value));
+        range_del_agg, /*largest_compaction_key=*/nullptr,
+        allow_unprepared_value));
   }
 }
 
@@ -1662,7 +1642,7 @@ Status Version::OverlapWithLevelIterator(const ReadOptions& read_options,
           mutable_cf_options_.prefix_extractor.get(), nullptr,
           cfd_->internal_stats()->GetFileReadHist(0),
           TableReaderCaller::kUserIterator, &arena,
-          /*skip_filters=*/false, /*level=*/0, max_file_size_for_l0_meta_pin_,
+          /*skip_filters=*/false, /*level=*/0,
           /*smallest_compaction_key=*/nullptr,
           /*largest_compaction_key=*/nullptr,
           /*allow_unprepared_value=*/false));
@@ -1766,8 +1746,6 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset,
       refs_(0),
       file_options_(file_opt),
       mutable_cf_options_(mutable_cf_options),
-      max_file_size_for_l0_meta_pin_(
-          MaxFileSizeForL0MetaPin(mutable_cf_options_)),
       version_number_(version_number) {}
 
 void Version::Get(const ReadOptions& read_options, const LookupKey& k,
@@ -1831,11 +1809,11 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
         cfd_->internal_stats()->GetFileReadHist(fp.GetHitFileLevel()),
         IsFilterSkipped(static_cast<int>(fp.GetHitFileLevel()),
                         fp.IsHitFileLastInLevel()),
-        fp.GetHitFileLevel(), max_file_size_for_l0_meta_pin_);
+        fp.GetCurrentLevel());
     // TODO: examine the behavior for corrupted key
     if (timer_enabled) {
       PERF_COUNTER_BY_LEVEL_ADD(get_from_table_nanos, timer.ElapsedNanos(),
-                                fp.GetHitFileLevel());
+                                fp.GetCurrentLevel());
     }
     if (!status->ok()) {
       return;
@@ -1957,7 +1935,6 @@ void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
       &storage_info_.level_files_brief_, storage_info_.num_non_empty_levels_,
       &storage_info_.file_indexer_, user_comparator(), internal_comparator());
   FdWithKeyRange* f = fp.GetNextFile();
-  Status s;
 
   while (f != nullptr) {
     MultiGetRange file_range = fp.CurrentFileRange();
@@ -1965,17 +1942,17 @@ void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
         GetPerfLevel() >= PerfLevel::kEnableTimeExceptForMutex &&
         get_perf_context()->per_level_perf_context_enabled;
     StopWatchNano timer(env_, timer_enabled /* auto_start */);
-    s = table_cache_->MultiGet(
+    Status s = table_cache_->MultiGet(
         read_options, *internal_comparator(), *f->file_metadata, &file_range,
         mutable_cf_options_.prefix_extractor.get(),
         cfd_->internal_stats()->GetFileReadHist(fp.GetHitFileLevel()),
         IsFilterSkipped(static_cast<int>(fp.GetHitFileLevel()),
                         fp.IsHitFileLastInLevel()),
-        fp.GetHitFileLevel());
+        fp.GetCurrentLevel());
     // TODO: examine the behavior for corrupted key
     if (timer_enabled) {
       PERF_COUNTER_BY_LEVEL_ADD(get_from_table_nanos, timer.ElapsedNanos(),
-                                fp.GetHitFileLevel());
+                                fp.GetCurrentLevel());
     }
     if (!s.ok()) {
       // TODO: Set status for individual keys appropriately
@@ -1986,8 +1963,7 @@ void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
       return;
     }
     uint64_t batch_size = 0;
-    for (auto iter = file_range.begin(); s.ok() && iter != file_range.end();
-         ++iter) {
+    for (auto iter = file_range.begin(); iter != file_range.end(); ++iter) {
       GetContext& get_context = *iter->get_context;
       Status* status = iter->s;
       // The Status in the KeyContext takes precedence over GetContext state
@@ -2033,12 +2009,7 @@ void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
           }
           PERF_COUNTER_BY_LEVEL_ADD(user_key_return_count, 1,
                                     fp.GetHitFileLevel());
-          file_range.AddValueSize(iter->value->size());
           file_range.MarkKeyDone(iter);
-          if (file_range.GetValueSize() > read_options.value_size_soft_limit) {
-            s = Status::Aborted();
-            break;
-          }
           continue;
         case GetContext::kDeleted:
           // Use empty error message for speed
@@ -2060,14 +2031,14 @@ void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
       }
     }
     RecordInHistogram(db_statistics_, SST_BATCH_SIZE, batch_size);
-    if (!s.ok() || file_picker_range.empty()) {
+    if (file_picker_range.empty()) {
       break;
     }
     f = fp.GetNextFile();
   }
 
   // Process any left over keys
-  for (auto iter = range->begin(); s.ok() && iter != range->end(); ++iter) {
+  for (auto iter = range->begin(); iter != range->end(); ++iter) {
     GetContext& get_context = *iter->get_context;
     Status* status = iter->s;
     Slice user_key = iter->lkey->user_key();
@@ -2092,23 +2063,12 @@ void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
           nullptr /* result_operand */, true);
       if (LIKELY(iter->value != nullptr)) {
         iter->value->PinSelf();
-        range->AddValueSize(iter->value->size());
-        range->MarkKeyDone(iter);
-        if (range->GetValueSize() > read_options.value_size_soft_limit) {
-          s = Status::Aborted();
-          break;
-        }
       }
     } else {
       range->MarkKeyDone(iter);
       *status = Status::NotFound();  // Use an empty error message for speed
     }
   }
-
-  for (auto iter = range->begin(); iter != range->end(); ++iter) {
-    range->MarkKeyDone(iter);
-    *(iter->s) = s;
-  }
 }
 
 bool Version::IsFilterSkipped(int level, bool is_file_last_in_level) {
@@ -2661,12 +2621,6 @@ void VersionStorageInfo::AddFile(int level, FileMetaData* f, Logger* info_log) {
 #endif
   f->refs++;
   level_files->push_back(f);
-
-  const uint64_t file_number = f->fd.GetNumber();
-
-  assert(file_locations_.find(file_number) == file_locations_.end());
-  file_locations_.emplace(file_number,
-                          FileLocation(level, level_files->size() - 1));
 }
 
 void VersionStorageInfo::AddBlobFile(
@@ -3471,27 +3425,13 @@ bool VersionStorageInfo::RangeMightExistAfterSortedRun(
   return false;
 }
 
-void Version::AddLiveFiles(std::vector<uint64_t>* live_table_files,
-                           std::vector<uint64_t>* live_blob_files) const {
-  assert(live_table_files);
-  assert(live_blob_files);
-
-  for (int level = 0; level < storage_info_.num_levels(); ++level) {
-    const auto& level_files = storage_info_.LevelFiles(level);
-    for (const auto& meta : level_files) {
-      assert(meta);
-
-      live_table_files->emplace_back(meta->fd.GetNumber());
+void Version::AddLiveFiles(std::vector<FileDescriptor>* live) {
+  for (int level = 0; level < storage_info_.num_levels(); level++) {
+    const std::vector<FileMetaData*>& files = storage_info_.files_[level];
+    for (const auto& file : files) {
+      live->push_back(file->fd);
     }
   }
-
-  const auto& blob_files = storage_info_.GetBlobFiles();
-  for (const auto& pair : blob_files) {
-    const auto& meta = pair.second;
-    assert(meta);
-
-    live_blob_files->emplace_back(meta->GetBlobFileNumber());
-  }
 }
 
 std::string Version::DebugString(bool hex, bool print_stats) const {
@@ -3878,6 +3818,10 @@ Status VersionSet::ProcessManifestWrites(
   }
 #endif  // NDEBUG
 
+  uint64_t new_manifest_file_size = 0;
+  Status s;
+  IOStatus io_s;
+
   assert(pending_manifest_file_number_ == 0);
   if (!descriptor_log_ ||
       manifest_file_size_ > db_options_->max_manifest_file_size) {
@@ -3907,9 +3851,6 @@ Status VersionSet::ProcessManifestWrites(
     }
   }
 
-  uint64_t new_manifest_file_size = 0;
-  Status s;
-  IOStatus io_s;
   {
     FileOptions opt_file_opts = fs_->OptimizeForManifestWrite(file_options_);
     mu->Unlock();
@@ -3926,8 +3867,7 @@ Status VersionSet::ProcessManifestWrites(
             cfd->internal_stats(), 1 /* max_threads */,
             true /* prefetch_index_and_filter_in_cache */,
             false /* is_initial_load */,
-            mutable_cf_options_ptrs[i]->prefix_extractor.get(),
-            MaxFileSizeForL0MetaPin(*mutable_cf_options_ptrs[i]));
+            mutable_cf_options_ptrs[i]->prefix_extractor.get());
         if (!s.ok()) {
           if (db_options_->paranoid_checks) {
             break;
@@ -3946,9 +3886,9 @@ Status VersionSet::ProcessManifestWrites(
       std::string descriptor_fname =
           DescriptorFileName(dbname_, pending_manifest_file_number_);
       std::unique_ptr<FSWritableFile> descriptor_file;
-      io_s = NewWritableFile(fs_, descriptor_fname, &descriptor_file,
-                             opt_file_opts);
-      if (io_s.ok()) {
+      s = NewWritableFile(fs_, descriptor_fname, &descriptor_file,
+                          opt_file_opts);
+      if (s.ok()) {
         descriptor_file->SetPreallocationBlockSize(
             db_options_->manifest_preallocation_size);
 
@@ -3957,10 +3897,7 @@ Status VersionSet::ProcessManifestWrites(
             nullptr, db_options_->listeners));
         descriptor_log_.reset(
             new log::Writer(std::move(file_writer), 0, false));
-        s = WriteCurrentStateToManifest(curr_state, descriptor_log_.get(),
-                                        io_s);
-      } else {
-        s = io_s;
+        s = WriteCurrentStateToManifest(curr_state, descriptor_log_.get());
       }
     }
 
@@ -3996,16 +3933,16 @@ Status VersionSet::ProcessManifestWrites(
 #endif /* !NDEBUG */
         io_s = descriptor_log_->AddRecord(record);
         if (!io_s.ok()) {
+          io_status_ = io_s;
           s = io_s;
           break;
         }
       }
       if (s.ok()) {
         io_s = SyncManifest(env_, db_options_, descriptor_log_->file());
-        TEST_SYNC_POINT_CALLBACK(
-            "VersionSet::ProcessManifestWrites:AfterSyncManifest", &io_s);
       }
       if (!io_s.ok()) {
+        io_status_ = io_s;
         s = io_s;
         ROCKS_LOG_ERROR(db_options_->info_log, "MANIFEST write %s\n",
                         s.ToString().c_str());
@@ -4018,6 +3955,7 @@ Status VersionSet::ProcessManifestWrites(
       io_s = SetCurrentFile(fs_, dbname_, pending_manifest_file_number_,
                             db_directory);
       if (!io_s.ok()) {
+        io_status_ = io_s;
         s = io_s;
       }
       TEST_SYNC_POINT("VersionSet::ProcessManifestWrites:AfterNewManifest");
@@ -4039,14 +3977,6 @@ Status VersionSet::ProcessManifestWrites(
     mu->Lock();
   }
 
-  if (!io_s.ok()) {
-    if (io_status_.ok()) {
-      io_status_ = io_s;
-    }
-  } else if (!io_status_.ok()) {
-    io_status_ = io_s;
-  }
-
   // Append the old manifest file to the obsolete_manifest_ list to be deleted
   // by PurgeObsoleteFiles later.
   if (s.ok() && new_descriptor_log) {
@@ -4669,8 +4599,7 @@ Status VersionSet::Recover(
           cfd->internal_stats(), db_options_->max_file_opening_threads,
           false /* prefetch_index_and_filter_in_cache */,
           true /* is_initial_load */,
-          cfd->GetLatestMutableCFOptions()->prefix_extractor.get(),
-          MaxFileSizeForL0MetaPin(*cfd->GetLatestMutableCFOptions()));
+          cfd->GetLatestMutableCFOptions()->prefix_extractor.get());
       if (!s.ok()) {
         if (db_options_->paranoid_checks) {
           return s;
@@ -5017,19 +4946,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
   }
 
   if (first_nonempty_level > 0) {
-    auto& new_last_level = new_files_list[new_levels - 1];
-
-    new_last_level = vstorage->LevelFiles(first_nonempty_level);
-
-    for (size_t i = 0; i < new_last_level.size(); ++i) {
-      const FileMetaData* const meta = new_last_level[i];
-      assert(meta);
-
-      const uint64_t file_number = meta->fd.GetNumber();
-
-      vstorage->file_locations_[file_number] =
-          VersionStorageInfo::FileLocation(new_levels - 1, i);
-    }
+    new_files_list[new_levels - 1] = vstorage->LevelFiles(first_nonempty_level);
   }
 
   delete[] vstorage -> files_;
@@ -5300,7 +5217,7 @@ void VersionSet::MarkMinLogNumberToKeep2PC(uint64_t number) {
 
 Status VersionSet::WriteCurrentStateToManifest(
     const std::unordered_map<uint32_t, MutableCFState>& curr_state,
-    log::Writer* log, IOStatus& io_s) {
+    log::Writer* log) {
   // TODO: Break up into multiple records to reduce memory usage on recovery?
 
   // WARNING: This method doesn't hold a mutex!!
@@ -5309,7 +5226,6 @@ Status VersionSet::WriteCurrentStateToManifest(
   // LogAndApply. Column family manipulations can only happen within LogAndApply
   // (the same single thread), so we're safe to iterate.
 
-  assert(io_s.ok());
   if (db_options_->write_dbid_to_manifest) {
     VersionEdit edit_for_db_id;
     assert(!db_id_.empty());
@@ -5319,9 +5235,10 @@ Status VersionSet::WriteCurrentStateToManifest(
       return Status::Corruption("Unable to Encode VersionEdit:" +
                                 edit_for_db_id.DebugString(true));
     }
-    io_s = log->AddRecord(db_id_record);
+    IOStatus io_s = log->AddRecord(db_id_record);
     if (!io_s.ok()) {
-      return io_s;
+      io_status_ = io_s;
+      return std::move(io_s);
     }
   }
 
@@ -5348,9 +5265,10 @@ Status VersionSet::WriteCurrentStateToManifest(
         return Status::Corruption(
             "Unable to Encode VersionEdit:" + edit.DebugString(true));
       }
-      io_s = log->AddRecord(record);
+      IOStatus io_s = log->AddRecord(record);
       if (!io_s.ok()) {
-        return io_s;
+        io_status_ = io_s;
+        return std::move(io_s);
       }
     }
 
@@ -5400,9 +5318,10 @@ Status VersionSet::WriteCurrentStateToManifest(
         return Status::Corruption(
             "Unable to Encode VersionEdit:" + edit.DebugString(true));
       }
-      io_s = log->AddRecord(record);
+      IOStatus io_s = log->AddRecord(record);
       if (!io_s.ok()) {
-        return io_s;
+        io_status_ = io_s;
+        return std::move(io_s);
       }
     }
   }
@@ -5527,8 +5446,7 @@ uint64_t VersionSet::ApproximateSize(const SizeApproximationOptions& options,
                         static_cast<uint64_t>(total_full_size * margin)) {
     total_full_size += total_intersecting_size / 2;
   } else {
-    // Estimate for all the first files (might also be last files), at each
-    // level
+    // Estimate for all the first files, at each level
     for (const auto file_ptr : first_files) {
       total_full_size += ApproximateSize(v, *file_ptr, start, end, caller);
     }
@@ -5608,70 +5526,44 @@ uint64_t VersionSet::ApproximateSize(Version* v, const FdWithKeyRange& f,
       v->GetMutableCFOptions().prefix_extractor.get());
 }
 
-void VersionSet::AddLiveFiles(std::vector<uint64_t>* live_table_files,
-                              std::vector<uint64_t>* live_blob_files) const {
-  assert(live_table_files);
-  assert(live_blob_files);
-
+void VersionSet::AddLiveFiles(std::vector<FileDescriptor>* live_list) {
   // pre-calculate space requirement
-  size_t total_table_files = 0;
-  size_t total_blob_files = 0;
-
-  assert(column_family_set_);
+  int64_t total_files = 0;
   for (auto cfd : *column_family_set_) {
-    assert(cfd);
-
     if (!cfd->initialized()) {
       continue;
     }
-
-    Version* const dummy_versions = cfd->dummy_versions();
-    assert(dummy_versions);
-
+    Version* dummy_versions = cfd->dummy_versions();
     for (Version* v = dummy_versions->next_; v != dummy_versions;
          v = v->next_) {
-      assert(v);
-
       const auto* vstorage = v->storage_info();
-      assert(vstorage);
-
-      for (int level = 0; level < vstorage->num_levels(); ++level) {
-        total_table_files += vstorage->LevelFiles(level).size();
+      for (int level = 0; level < vstorage->num_levels(); level++) {
+        total_files += vstorage->LevelFiles(level).size();
       }
-
-      total_blob_files += vstorage->GetBlobFiles().size();
     }
   }
 
   // just one time extension to the right size
-  live_table_files->reserve(live_table_files->size() + total_table_files);
-  live_blob_files->reserve(live_blob_files->size() + total_blob_files);
+  live_list->reserve(live_list->size() + static_cast<size_t>(total_files));
 
-  assert(column_family_set_);
   for (auto cfd : *column_family_set_) {
-    assert(cfd);
     if (!cfd->initialized()) {
       continue;
     }
-
     auto* current = cfd->current();
     bool found_current = false;
-
-    Version* const dummy_versions = cfd->dummy_versions();
-    assert(dummy_versions);
-
+    Version* dummy_versions = cfd->dummy_versions();
     for (Version* v = dummy_versions->next_; v != dummy_versions;
          v = v->next_) {
-      v->AddLiveFiles(live_table_files, live_blob_files);
+      v->AddLiveFiles(live_list);
       if (v == current) {
         found_current = true;
       }
     }
-
     if (!found_current && current != nullptr) {
       // Should never happen unless it is a bug.
       assert(false);
-      current->AddLiveFiles(live_table_files, live_blob_files);
+      current->AddLiveFiles(live_list);
     }
   }
 }
@@ -5704,14 +5596,13 @@ InternalIterator* VersionSet::MakeInputIterator(
         for (size_t i = 0; i < flevel->num_files; i++) {
           list[num++] = cfd->table_cache()->NewIterator(
               read_options, file_options_compactions,
-              cfd->internal_comparator(), *flevel->files[i].file_metadata,
-              range_del_agg, c->mutable_cf_options()->prefix_extractor.get(),
+              cfd->internal_comparator(),
+              *flevel->files[i].file_metadata, range_del_agg,
+              c->mutable_cf_options()->prefix_extractor.get(),
               /*table_reader_ptr=*/nullptr,
               /*file_read_hist=*/nullptr, TableReaderCaller::kCompaction,
               /*arena=*/nullptr,
-              /*skip_filters=*/false,
-              /*level=*/static_cast<int>(c->level(which)),
-              MaxFileSizeForL0MetaPin(*c->mutable_cf_options()),
+              /*skip_filters=*/false, /*level=*/static_cast<int>(which),
               /*smallest_compaction_key=*/nullptr,
               /*largest_compaction_key=*/nullptr,
               /*allow_unprepared_value=*/false);
@@ -5725,7 +5616,7 @@ InternalIterator* VersionSet::MakeInputIterator(
             /*should_sample=*/false,
             /*no per level latency histogram=*/nullptr,
             TableReaderCaller::kCompaction, /*skip_filters=*/false,
-            /*level=*/static_cast<int>(c->level(which)), range_del_agg,
+            /*level=*/static_cast<int>(which), range_del_agg,
             c->boundaries(which));
       }
     }
@@ -6040,8 +5931,7 @@ Status ReactiveVersionSet::Recover(
                 cfd->internal_stats(), db_options_->max_file_opening_threads,
                 false /* prefetch_index_and_filter_in_cache */,
                 true /* is_initial_load */,
-                cfd->GetLatestMutableCFOptions()->prefix_extractor.get(),
-                MaxFileSizeForL0MetaPin(*cfd->GetLatestMutableCFOptions()));
+                cfd->GetLatestMutableCFOptions()->prefix_extractor.get());
             if (!s.ok()) {
               enough = false;
               if (s.IsPathNotFound()) {
@@ -6317,8 +6207,7 @@ Status ReactiveVersionSet::ApplyOneVersionEditToBuilder(
         cfd->internal_stats(), db_options_->max_file_opening_threads,
         false /* prefetch_index_and_filter_in_cache */,
         false /* is_initial_load */,
-        cfd->GetLatestMutableCFOptions()->prefix_extractor.get(),
-        MaxFileSizeForL0MetaPin(*cfd->GetLatestMutableCFOptions()));
+        cfd->GetLatestMutableCFOptions()->prefix_extractor.get());
     TEST_SYNC_POINT_CALLBACK(
         "ReactiveVersionSet::ApplyOneVersionEditToBuilder:"
         "AfterLoadTableHandlers",

db/version_set.h

@@ -283,58 +283,6 @@ class VersionStorageInfo {
     return files_[level];
   }
 
-  class FileLocation {
-   public:
-    FileLocation() = default;
-    FileLocation(int level, size_t position)
-        : level_(level), position_(position) {}
-
-    int GetLevel() const { return level_; }
-    size_t GetPosition() const { return position_; }
-
-    bool IsValid() const { return level_ >= 0; }
-
-    bool operator==(const FileLocation& rhs) const {
-      return level_ == rhs.level_ && position_ == rhs.position_;
-    }
-
-    bool operator!=(const FileLocation& rhs) const { return !(*this == rhs); }
-
-    static FileLocation Invalid() { return FileLocation(); }
-
-   private:
-    int level_ = -1;
-    size_t position_ = 0;
-  };
-
-  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  FileLocation GetFileLocation(uint64_t file_number) const {
-    const auto it = file_locations_.find(file_number);
-
-    if (it == file_locations_.end()) {
-      return FileLocation::Invalid();
-    }
-
-    assert(it->second.GetLevel() < num_levels_);
-    assert(it->second.GetPosition() < files_[it->second.GetLevel()].size());
-    assert(files_[it->second.GetLevel()][it->second.GetPosition()]);
-    assert(files_[it->second.GetLevel()][it->second.GetPosition()]
-               ->fd.GetNumber() == file_number);
-
-    return it->second;
-  }
-
-  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  FileMetaData* GetFileMetaDataByNumber(uint64_t file_number) const {
-    auto location = GetFileLocation(file_number);
-
-    if (!location.IsValid()) {
-      return nullptr;
-    }
-
-    return files_[location.GetLevel()][location.GetPosition()];
-  }
-
   // REQUIRES: This version has been saved (see VersionSet::SaveTo)
   using BlobFiles = std::map<uint64_t, std::shared_ptr<BlobFileMetaData>>;
   const BlobFiles& GetBlobFiles() const { return blob_files_; }
@@ -513,11 +461,6 @@ class VersionStorageInfo {
   // in increasing order of keys
   std::vector<FileMetaData*>* files_;
 
-  // Map of all table files in version. Maps file number to (level, position on
-  // level).
-  using FileLocations = std::unordered_map<uint64_t, FileLocation>;
-  FileLocations file_locations_;
-
   // Map of blob files in version by number.
   BlobFiles blob_files_;
 
@@ -686,10 +629,8 @@ class Version {
   // and return true. Otherwise, return false.
   bool Unref();
 
-  // Add all files listed in the current version to *live_table_files and
-  // *live_blob_files.
-  void AddLiveFiles(std::vector<uint64_t>* live_table_files,
-                    std::vector<uint64_t>* live_blob_files) const;
+  // Add all files listed in the current version to *live.
+  void AddLiveFiles(std::vector<FileDescriptor>* live);
 
   // Return a human readable string that describes this version's contents.
   std::string DebugString(bool hex = false, bool print_stats = false) const;
@@ -803,8 +744,6 @@ class Version {
   int refs_;                    // Number of live refs to this version
   const FileOptions file_options_;
   const MutableCFOptions mutable_cf_options_;
-  // Cached value to avoid recomputing it on every read.
-  const size_t max_file_size_for_l0_meta_pin_;
 
   // A version number that uniquely represents this version. This is
   // used for debugging and logging purposes only.
@@ -1108,10 +1047,8 @@ class VersionSet {
       const Compaction* c, RangeDelAggregator* range_del_agg,
       const FileOptions& file_options_compactions);
 
-  // Add all files listed in any live version to *live_table_files and
-  // *live_blob_files. Note that these lists may contain duplicates.
-  void AddLiveFiles(std::vector<uint64_t>* live_table_files,
-                    std::vector<uint64_t>* live_blob_files) const;
+  // Add all files listed in any live version to *live.
+  void AddLiveFiles(std::vector<FileDescriptor>* live_list);
 
   // Return the approximate size of data to be scanned for range [start, end)
   // in levels [start_level, end_level). If end_level == -1 it will search
@@ -1159,7 +1096,10 @@ class VersionSet {
   static uint64_t GetTotalSstFilesSize(Version* dummy_versions);
 
   // Get the IO Status returned by written Manifest.
-  const IOStatus& io_status() const { return io_status_; }
+  IOStatus io_status() const { return io_status_; }
+
+  // Set the IO Status to OK. Called before Manifest write if needed.
+  void SetIOStatusOK() { io_status_ = IOStatus::OK(); }
 
  protected:
   using VersionBuilderMap =
@@ -1202,7 +1142,7 @@ class VersionSet {
   // Save current contents to *log
   Status WriteCurrentStateToManifest(
       const std::unordered_map<uint32_t, MutableCFState>& curr_state,
-      log::Writer* log, IOStatus& io_s);
+      log::Writer* log);
 
   void AppendVersion(ColumnFamilyData* column_family_data, Version* v);
 

db/version_set_test.cc

@@ -372,19 +372,19 @@ TEST_F(VersionStorageInfoTest, EstimateLiveDataSize) {
   Add(2, 3U, "6", "8", 1U);  // Partial overlap with last level
   Add(3, 4U, "1", "9", 1U);  // Contains range of last level
   Add(4, 5U, "4", "5", 1U);  // Inside range of last level
-  Add(4, 6U, "6", "7", 1U);  // Inside range of last level
-  Add(5, 7U, "4", "7", 10U);
+  Add(4, 5U, "6", "7", 1U);  // Inside range of last level
+  Add(5, 6U, "4", "7", 10U);
   ASSERT_EQ(10U, vstorage_.EstimateLiveDataSize());
 }
 
 TEST_F(VersionStorageInfoTest, EstimateLiveDataSize2) {
   Add(0, 1U, "9", "9", 1U);  // Level 0 is not ordered
-  Add(0, 2U, "5", "6", 1U);  // Ignored because of [5,6] in l1
-  Add(1, 3U, "1", "2", 1U);  // Ignored because of [2,3] in l2
-  Add(1, 4U, "3", "4", 1U);  // Ignored because of [2,3] in l2
-  Add(1, 5U, "5", "6", 1U);
-  Add(2, 6U, "2", "3", 1U);
-  Add(3, 7U, "7", "8", 1U);
+  Add(0, 1U, "5", "6", 1U);  // Ignored because of [5,6] in l1
+  Add(1, 1U, "1", "2", 1U);  // Ignored because of [2,3] in l2
+  Add(1, 2U, "3", "4", 1U);  // Ignored because of [2,3] in l2
+  Add(1, 3U, "5", "6", 1U);
+  Add(2, 4U, "2", "3", 1U);
+  Add(3, 5U, "7", "8", 1U);
   ASSERT_EQ(4U, vstorage_.EstimateLiveDataSize());
 }
 
@@ -421,28 +421,6 @@ TEST_F(VersionStorageInfoTest, GetOverlappingInputs) {
       1, {"i", 0, kTypeValue}, {"j", 0, kTypeValue}));
 }
 
-TEST_F(VersionStorageInfoTest, FileLocationAndMetaDataByNumber) {
-  Add(0, 11U, "1", "2", 5000U);
-  Add(0, 12U, "1", "2", 5000U);
-
-  Add(2, 7U, "1", "2", 8000U);
-
-  ASSERT_EQ(vstorage_.GetFileLocation(11U),
-            VersionStorageInfo::FileLocation(0, 0));
-  ASSERT_NE(vstorage_.GetFileMetaDataByNumber(11U), nullptr);
-
-  ASSERT_EQ(vstorage_.GetFileLocation(12U),
-            VersionStorageInfo::FileLocation(0, 1));
-  ASSERT_NE(vstorage_.GetFileMetaDataByNumber(12U), nullptr);
-
-  ASSERT_EQ(vstorage_.GetFileLocation(7U),
-            VersionStorageInfo::FileLocation(2, 0));
-  ASSERT_NE(vstorage_.GetFileMetaDataByNumber(7U), nullptr);
-
-  ASSERT_FALSE(vstorage_.GetFileLocation(999U).IsValid());
-  ASSERT_EQ(vstorage_.GetFileMetaDataByNumber(999U), nullptr);
-}
-
 class VersionStorageInfoTimestampTest : public VersionStorageInfoTestBase {
  public:
   VersionStorageInfoTimestampTest()
@@ -982,84 +960,6 @@ TEST_F(VersionSetTest, PersistBlobFileStateInNewManifest) {
   SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
-TEST_F(VersionSetTest, AddLiveBlobFiles) {
-  // Initialize the database and add a blob file.
-  NewDB();
-
-  assert(versions_);
-  assert(versions_->GetColumnFamilySet());
-
-  ColumnFamilyData* const cfd = versions_->GetColumnFamilySet()->GetDefault();
-  assert(cfd);
-
-  VersionEdit first;
-
-  constexpr uint64_t first_blob_file_number = 234;
-  constexpr uint64_t first_total_blob_count = 555;
-  constexpr uint64_t first_total_blob_bytes = 66666;
-  constexpr char first_checksum_method[] = "CRC32";
-  constexpr char first_checksum_value[] = "3d87ff57";
-
-  first.AddBlobFile(first_blob_file_number, first_total_blob_count,
-                    first_total_blob_bytes, first_checksum_method,
-                    first_checksum_value);
-
-  mutex_.Lock();
-  Status s = versions_->LogAndApply(cfd, mutable_cf_options_, &first, &mutex_);
-  mutex_.Unlock();
-
-  ASSERT_OK(s);
-
-  // Reference the version so it stays alive even after the following version
-  // edit.
-  Version* const version = cfd->current();
-  assert(version);
-
-  version->Ref();
-
-  // Get live files directly from version.
-  std::vector<uint64_t> version_table_files;
-  std::vector<uint64_t> version_blob_files;
-
-  version->AddLiveFiles(&version_table_files, &version_blob_files);
-
-  ASSERT_EQ(version_blob_files.size(), 1);
-  ASSERT_EQ(version_blob_files[0], first_blob_file_number);
-
-  // Add another blob file.
-  VersionEdit second;
-
-  constexpr uint64_t second_blob_file_number = 456;
-  constexpr uint64_t second_total_blob_count = 100;
-  constexpr uint64_t second_total_blob_bytes = 2000000;
-  constexpr char second_checksum_method[] = "CRC32B";
-  constexpr char second_checksum_value[] = "6dbdf23a";
-  second.AddBlobFile(second_blob_file_number, second_total_blob_count,
-                     second_total_blob_bytes, second_checksum_method,
-                     second_checksum_value);
-
-  mutex_.Lock();
-  s = versions_->LogAndApply(cfd, mutable_cf_options_, &second, &mutex_);
-  mutex_.Unlock();
-
-  ASSERT_OK(s);
-
-  // Get all live files from version set. Note that the result contains
-  // duplicates.
-  std::vector<uint64_t> all_table_files;
-  std::vector<uint64_t> all_blob_files;
-
-  versions_->AddLiveFiles(&all_table_files, &all_blob_files);
-
-  ASSERT_EQ(all_blob_files.size(), 3);
-  ASSERT_EQ(all_blob_files[0], first_blob_file_number);
-  ASSERT_EQ(all_blob_files[1], first_blob_file_number);
-  ASSERT_EQ(all_blob_files[2], second_blob_file_number);
-
-  // Clean up previous version.
-  version->Unref();
-}
-
 TEST_F(VersionSetTest, ObsoleteBlobFile) {
   // Initialize the database and add a blob file (with no garbage just yet).
   NewDB();

db/write_batch.cc

@@ -908,14 +908,7 @@ Status WriteBatchInternal::Delete(WriteBatch* b, uint32_t column_family_id,
     b->rep_.push_back(static_cast<char>(kTypeColumnFamilyDeletion));
     PutVarint32(&b->rep_, column_family_id);
   }
-  if (0 == b->timestamp_size_) {
-    PutLengthPrefixedSlice(&b->rep_, key);
-  } else {
-    PutVarint32(&b->rep_,
-                static_cast<uint32_t>(key.size() + b->timestamp_size_));
-    b->rep_.append(key.data(), key.size());
-    b->rep_.append(b->timestamp_size_, '\0');
-  }
+  PutLengthPrefixedSlice(&b->rep_, key);
   b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
                               ContentFlags::HAS_DELETE,
                           std::memory_order_relaxed);
@@ -937,11 +930,7 @@ Status WriteBatchInternal::Delete(WriteBatch* b, uint32_t column_family_id,
     b->rep_.push_back(static_cast<char>(kTypeColumnFamilyDeletion));
     PutVarint32(&b->rep_, column_family_id);
   }
-  if (0 == b->timestamp_size_) {
-    PutLengthPrefixedSliceParts(&b->rep_, key);
-  } else {
-    PutLengthPrefixedSlicePartsWithPadding(&b->rep_, key, b->timestamp_size_);
-  }
+  PutLengthPrefixedSliceParts(&b->rep_, key);
   b->content_flags_.store(b->content_flags_.load(std::memory_order_relaxed) |
                               ContentFlags::HAS_DELETE,
                           std::memory_order_relaxed);
@@ -1557,14 +1546,7 @@ class MemTableInserter : public WriteBatch::Handler {
       return seek_status;
     }
 
-    ColumnFamilyData* cfd = cf_mems_->current();
-    assert(!cfd || cfd->user_comparator());
-    const size_t ts_sz = (cfd && cfd->user_comparator())
-                             ? cfd->user_comparator()->timestamp_size()
-                             : 0;
-    const ValueType delete_type =
-        (0 == ts_sz) ? kTypeDeletion : kTypeDeletionWithTimestamp;
-    auto ret_status = DeleteImpl(column_family_id, key, Slice(), delete_type);
+    auto ret_status = DeleteImpl(column_family_id, key, Slice(), kTypeDeletion);
     // optimize for non-recovery mode
     if (UNLIKELY(!ret_status.IsTryAgain() && rebuilding_trx_ != nullptr)) {
       assert(!write_after_commit_);
@@ -1648,15 +1630,6 @@ class MemTableInserter : public WriteBatch::Handler {
             cfd->ioptions()->table_factory->Name() + " in CF " +
             cfd->GetName());
       }
-      int cmp = cfd->user_comparator()->Compare(begin_key, end_key);
-      if (cmp > 0) {
-        // It's an empty range where endpoints appear mistaken. Don't bother
-        // applying it to the DB, and return an error to the user.
-        return Status::InvalidArgument("end key comes before start key");
-      } else if (cmp == 0) {
-        // It's an empty range. Don't bother applying it to the DB.
-        return Status::OK();
-      }
     }
 
     auto ret_status =

db_stress_tool/CMakeLists.txt

@@ -10,5 +10,5 @@ add_executable(db_stress${ARTIFACT_SUFFIX}
   db_stress_gflags.cc
   db_stress_tool.cc
   no_batched_ops_stress.cc)
-target_link_libraries(db_stress${ARTIFACT_SUFFIX} ${ROCKSDB_LIB} ${THIRDPARTY_LIBS})
+target_link_libraries(db_stress${ARTIFACT_SUFFIX} ${ROCKSDB_LIB})
 list(APPEND tool_deps db_stress)

db_stress_tool/db_stress_common.h

@@ -112,7 +112,6 @@ DECLARE_bool(memtable_whole_key_filtering);
 DECLARE_int32(open_files);
 DECLARE_int64(compressed_cache_size);
 DECLARE_int32(compaction_style);
-DECLARE_int32(num_levels);
 DECLARE_int32(level0_file_num_compaction_trigger);
 DECLARE_int32(level0_slowdown_writes_trigger);
 DECLARE_int32(level0_stop_writes_trigger);

db_stress_tool/db_stress_gflags.cc

@@ -186,9 +186,6 @@ DEFINE_int64(compressed_cache_size, -1,
 DEFINE_int32(compaction_style, ROCKSDB_NAMESPACE::Options().compaction_style,
              "");
 
-DEFINE_int32(num_levels, ROCKSDB_NAMESPACE::Options().num_levels,
-             "Number of levels in the DB");
-
 DEFINE_int32(level0_file_num_compaction_trigger,
              ROCKSDB_NAMESPACE::Options().level0_file_num_compaction_trigger,
              "Level0 compaction start trigger");

db_stress_tool/db_stress_tool.cc

@@ -245,7 +245,7 @@ int db_stress_tool(int argc, char** argv) {
     }
   } else {
     uint64_t keys_per_level = key_gen_ctx.window / levels;
-    for (unsigned int level = 0; level + 1 < levels; ++level) {
+    for (unsigned int level = 0; level < levels - 1; ++level) {
       key_gen_ctx.weights.emplace_back(keys_per_level);
     }
     key_gen_ctx.weights.emplace_back(key_gen_ctx.window -

db_stress_tool/no_batched_ops_stress.cc

@@ -36,8 +36,8 @@ class NonBatchedOpsStressTest : public StressTest {
       if (thread->shared->HasVerificationFailedYet()) {
         break;
       }
-      if (thread->rand.OneIn(3)) {
-        // 1/3 chance use iterator to verify this range
+      if (!thread->rand.OneIn(2)) {
+        // Use iterator to verify this range
         Slice prefix;
         std::string seek_key = Key(start);
         std::unique_ptr<Iterator> iter(
@@ -82,8 +82,8 @@ class NonBatchedOpsStressTest : public StressTest {
                           from_db.data(), from_db.length());
           }
         }
-      } else if (thread->rand.OneIn(2)) {
-        // 1/3 chance use Get to verify this range
+      } else {
+        // Use Get to verify this range
         for (auto i = start; i < end; i++) {
           if (thread->shared->HasVerificationFailedYet()) {
             break;
@@ -99,38 +99,6 @@ class NonBatchedOpsStressTest : public StressTest {
                           from_db.data(), from_db.length());
           }
         }
-      } else {
-        // 1/3 chance use MultiGet to verify this range
-        for (auto i = start; i < end;) {
-          if (thread->shared->HasVerificationFailedYet()) {
-            break;
-          }
-          // Keep the batch size to some reasonable value
-          size_t batch_size = thread->rand.Uniform(128) + 1;
-          batch_size = std::min<size_t>(batch_size, end - i);
-          std::vector<std::string> keystrs(batch_size);
-          std::vector<Slice> keys(batch_size);
-          std::vector<PinnableSlice> values(batch_size);
-          std::vector<Status> statuses(batch_size);
-          for (size_t j = 0; j < batch_size; ++j) {
-            keystrs[j] = Key(i + j);
-            keys[j] = Slice(keystrs[j].data(), keystrs[j].length());
-          }
-          db_->MultiGet(options, column_families_[cf], batch_size, keys.data(),
-                        values.data(), statuses.data());
-          for (size_t j = 0; j < batch_size; ++j) {
-            Status s = statuses[j];
-            std::string from_db = values[j].ToString();
-            VerifyValue(static_cast<int>(cf), i + j, options, shared, from_db,
-                        s, true);
-            if (from_db.length()) {
-              PrintKeyValue(static_cast<int>(cf), static_cast<uint32_t>(i + j),
-                            from_db.data(), from_db.length());
-            }
-          }
-
-          i += batch_size;
-        }
       }
     }
   }
@@ -241,14 +209,6 @@ class NonBatchedOpsStressTest : public StressTest {
     std::vector<Status> statuses(num_keys);
     ColumnFamilyHandle* cfh = column_families_[rand_column_families[0]];
     int error_count = 0;
-    // Do a consistency check between Get and MultiGet. Don't do it too
-    // often as it will slow db_stress down
-    bool do_consistency_check = thread->rand.OneIn(4);
-
-    ReadOptions readoptionscopy = read_opts;
-    if (do_consistency_check) {
-      readoptionscopy.snapshot = db_->GetSnapshot();
-    }
 
     // To appease clang analyzer
     const bool use_txn = FLAGS_use_txn;
@@ -315,7 +275,7 @@ class NonBatchedOpsStressTest : public StressTest {
         SharedState::ignore_read_error = false;
       }
 #endif // NDEBUG
-      db_->MultiGet(readoptionscopy, cfh, num_keys, keys.data(), values.data(),
+      db_->MultiGet(read_opts, cfh, num_keys, keys.data(), values.data(),
                     statuses.data());
 #ifndef NDEBUG
       if (fault_fs_guard) {
@@ -324,8 +284,9 @@ class NonBatchedOpsStressTest : public StressTest {
 #endif // NDEBUG
     } else {
 #ifndef ROCKSDB_LITE
-      txn->MultiGet(readoptionscopy, cfh, num_keys, keys.data(), values.data(),
+      txn->MultiGet(read_opts, cfh, num_keys, keys.data(), values.data(),
                     statuses.data());
+      RollbackTxn(txn);
 #endif
     }
 
@@ -348,57 +309,10 @@ class NonBatchedOpsStressTest : public StressTest {
         std::terminate();
       }
     }
-    if (fault_fs_guard) {
-      fault_fs_guard->DisableErrorInjection();
-    }
 #endif // NDEBUG
 
-    for (size_t i = 0; i < statuses.size(); ++i) {
-      Status s = statuses[i];
-      bool is_consistent = true;
-      // Only do the consistency check if no error was injected and MultiGet
-      // didn't return an unexpected error
-      if (do_consistency_check && !error_count && (s.ok() || s.IsNotFound())) {
-        Status tmp_s;
-        std::string value;
-
-        if (use_txn) {
-#ifndef ROCKSDB_LITE
-          tmp_s = txn->Get(readoptionscopy, cfh, keys[i], &value);
-#endif  // ROCKSDB_LITE
-        } else {
-          tmp_s = db_->Get(readoptionscopy, cfh, keys[i], &value);
-        }
-        if (!tmp_s.ok() && !tmp_s.IsNotFound()) {
-          fprintf(stderr, "Get error: %s\n", s.ToString().c_str());
-          is_consistent = false;
-        } else if (!s.ok() && tmp_s.ok()) {
-          fprintf(stderr, "MultiGet returned different results with key %s\n",
-                  keys[i].ToString(true).c_str());
-          fprintf(stderr, "Get returned ok, MultiGet returned not found\n");
-          is_consistent = false;
-        } else if (s.ok() && tmp_s.IsNotFound()) {
-          fprintf(stderr, "MultiGet returned different results with key %s\n",
-                  keys[i].ToString(true).c_str());
-          fprintf(stderr, "MultiGet returned ok, Get returned not found\n");
-          is_consistent = false;
-        } else if (s.ok() && value != values[i].ToString()) {
-          fprintf(stderr, "MultiGet returned different results with key %s\n",
-                  keys[i].ToString(true).c_str());
-          fprintf(stderr, "MultiGet returned value %s\n",
-                  values[i].ToString(true).c_str());
-          fprintf(stderr, "Get returned value %s\n", value.c_str());
-          is_consistent = false;
-        }
-      }
-
-      if (!is_consistent) {
-        fprintf(stderr, "TestMultiGet error: is_consistent is false\n");
-        thread->stats.AddErrors(1);
-        // Fail fast to preserve the DB state
-        thread->shared->SetVerificationFailure();
-        break;
-      } else if (s.ok()) {
+    for (const auto& s : statuses) {
+      if (s.ok()) {
         // found case
         thread->stats.AddGets(1, 1);
       } else if (s.IsNotFound()) {
@@ -417,15 +331,11 @@ class NonBatchedOpsStressTest : public StressTest {
         }
       }
     }
-
-    if (readoptionscopy.snapshot) {
-      db_->ReleaseSnapshot(readoptionscopy.snapshot);
-    }
-    if (use_txn) {
-#ifndef ROCKSDB_LITE
-      RollbackTxn(txn);
-#endif
+#ifndef NDEBUG
+    if (fault_fs_guard) {
+      fault_fs_guard->DisableErrorInjection();
     }
+#endif // NDEBUG
     return statuses;
   }
 

env/composite_env_wrapper.h

@@ -523,9 +523,7 @@ class CompositeEnvWrapper : public Env {
     return env_target_->GetThreadPoolQueueLen(pri);
   }
   Status GetTestDirectory(std::string* path) override {
-    IOOptions io_opts;
-    IODebugContext dbg;
-    return file_system_->GetTestDirectory(io_opts, path, &dbg);
+    return env_target_->GetTestDirectory(path);
   }
   uint64_t NowMicros() override { return env_target_->NowMicros(); }
   uint64_t NowNanos() override { return env_target_->NowNanos(); }

env/env_hdfs.cc

@@ -613,7 +613,7 @@ Status HdfsEnv::IsDirectory(const std::string& path, bool* is_dir) {
   hdfsFileInfo* pFileInfo = hdfsGetPathInfo(fileSys_, path.c_str());
   if (pFileInfo != nullptr) {
     if (is_dir != nullptr) {
-      *is_dir = (pFileInfo->mKind == kObjectKindDirectory);
+      *is_dir = (pFileInfo->mKind == tObjectKindDirectory);
     }
     hdfsFreeFileInfo(pFileInfo, 1);
     return Status::OK();

env/env_posix.cc

@@ -224,6 +224,20 @@ class PosixEnv : public CompositeEnvWrapper {
 
   unsigned int GetThreadPoolQueueLen(Priority pri = LOW) const override;
 
+  Status GetTestDirectory(std::string* result) override {
+    const char* env = getenv("TEST_TMPDIR");
+    if (env && env[0] != '\0') {
+      *result = env;
+    } else {
+      char buf[100];
+      snprintf(buf, sizeof(buf), "/tmp/rocksdbtest-%d", int(geteuid()));
+      *result = buf;
+    }
+    // Directory may already exist
+    CreateDir(*result);
+    return Status::OK();
+  }
+
   Status GetThreadList(std::vector<ThreadStatus>* thread_list) override {
     assert(thread_status_updater_);
     return thread_status_updater_->GetThreadList(thread_list);

env/env_test.cc

@@ -327,7 +327,6 @@ TEST_P(EnvPosixTestWithParam, UnSchedule) {
 // run in any order. The purpose of the test is unclear.
 #ifndef OS_WIN
 TEST_P(EnvPosixTestWithParam, RunMany) {
-  env_->SetBackgroundThreads(1, Env::LOW);
   std::atomic<int> last_id(0);
 
   struct CB {
@@ -1243,7 +1242,7 @@ TEST_F(EnvPosixTest, MultiReadNonAlignedLargeNum) {
     for (int so: start_offsets) {
       offsets.push_back(so);
     }
-    for (size_t i = 0; i + 1 < offsets.size(); i++) {
+    for (size_t i = 0; i < offsets.size() - 1; i++) {
       lens.push_back(static_cast<size_t>(rnd.Uniform(static_cast<int>(offsets[i + 1] - offsets[i])) + 1));
     }
     lens.push_back(static_cast<size_t>(rnd.Uniform(static_cast<int>(kTotalSize - offsets.back())) + 1));

env/io_posix.cc

@@ -189,19 +189,19 @@ bool IsSyncFileRangeSupported(int fd) {
 /*
  * DirectIOHelper
  */
+#ifndef NDEBUG
 namespace {
 
 bool IsSectorAligned(const size_t off, size_t sector_size) {
-  assert((sector_size & (sector_size - 1)) == 0);
-  return (off & (sector_size - 1)) == 0;
+  return off % sector_size == 0;
 }
 
-#ifndef NDEBUG
 bool IsSectorAligned(const void* ptr, size_t sector_size) {
   return uintptr_t(ptr) % sector_size == 0;
 }
-#endif
+
 }  // namespace
+#endif
 
 /*
  * PosixSequentialFile
@@ -276,7 +276,7 @@ IOStatus PosixSequentialFile::PositionedRead(uint64_t offset, size_t n,
     ptr += r;
     offset += r;
     left -= r;
-    if (!IsSectorAligned(r, GetRequiredBufferAlignment())) {
+    if (r % static_cast<ssize_t>(GetRequiredBufferAlignment()) != 0) {
       // Bytes reads don't fill sectors. Should only happen at the end
       // of the file.
       break;
@@ -416,7 +416,7 @@ Status LogicalBlockSizeCache::RefAndCacheLogicalBlockSize(
       v.size = dir_size->second;
     }
   }
-  return s;
+  return Status::OK();
 }
 
 void LogicalBlockSizeCache::UnrefAndTryRemoveCachedLogicalBlockSize(
@@ -711,22 +711,13 @@ IOStatus PosixRandomAccessFile::MultiRead(FSReadRequest* reqs,
           // comment
           // https://github.com/facebook/rocksdb/pull/6441#issuecomment-589843435
           // Fall back to pread in this case.
-          if (use_direct_io() &&
-              !IsSectorAligned(req_wrap->finished_len,
-                               GetRequiredBufferAlignment())) {
-            // Bytes reads don't fill sectors. Should only happen at the end
-            // of the file.
-            req->result = Slice(req->scratch, req_wrap->finished_len);
-            req->status = IOStatus::OK();
-          } else {
-            Slice tmp_slice;
-            req->status =
-                Read(req->offset + req_wrap->finished_len,
-                     req->len - req_wrap->finished_len, options, &tmp_slice,
-                     req->scratch + req_wrap->finished_len, dbg);
-            req->result =
-                Slice(req->scratch, req_wrap->finished_len + tmp_slice.size());
-          }
+          Slice tmp_slice;
+          req->status =
+              Read(req->offset + req_wrap->finished_len,
+                   req->len - req_wrap->finished_len, options, &tmp_slice,
+                   req->scratch + req_wrap->finished_len, dbg);
+          req->result =
+              Slice(req->scratch, req_wrap->finished_len + tmp_slice.size());
         } else if (bytes_read < req_wrap->iov.iov_len) {
           assert(bytes_read > 0);
           assert(bytes_read + req_wrap->finished_len < req->len);

env/io_posix_test.cc

@@ -67,7 +67,7 @@ TEST_F(LogicalBlockSizeCacheTest, Cache) {
   ASSERT_EQ(3, cache.GetLogicalBlockSize("/db2/sst2", 7));
   ASSERT_EQ(8, ncall);
 
-  ASSERT_OK(cache.RefAndCacheLogicalBlockSize({"/db"}));
+  cache.RefAndCacheLogicalBlockSize({"/db"});
   ASSERT_EQ(4, cache.Size());
   ASSERT_TRUE(cache.Contains("/"));
   ASSERT_TRUE(cache.Contains("/db1"));
@@ -104,7 +104,7 @@ TEST_F(LogicalBlockSizeCacheTest, Ref) {
   ASSERT_EQ(1, cache.GetLogicalBlockSize("/db/sst0", 1));
   ASSERT_EQ(1, ncall);
 
-  ASSERT_OK(cache.RefAndCacheLogicalBlockSize({"/db"}));
+  cache.RefAndCacheLogicalBlockSize({"/db"});
   ASSERT_EQ(2, ncall);
   ASSERT_EQ(1, cache.GetRefCount("/db"));
   // Block size for /db is cached. Ref count = 1.
@@ -112,7 +112,7 @@ TEST_F(LogicalBlockSizeCacheTest, Ref) {
   ASSERT_EQ(2, ncall);
 
   // Ref count = 2, but won't recompute the cached buffer size.
-  ASSERT_OK(cache.RefAndCacheLogicalBlockSize({"/db"}));
+  cache.RefAndCacheLogicalBlockSize({"/db"});
   ASSERT_EQ(2, cache.GetRefCount("/db"));
   ASSERT_EQ(2, ncall);
 

examples/multi_processes_example.cc

@@ -301,7 +301,7 @@ void RunSecondary() {
       std::string value;
       db->Get(ropts, key, &value);
     }
-    fprintf(stdout, "[process %ld] Point lookup thread finished\n", my_pid);
+    fprintf(stdout, "[process %ld] Point lookup thread finished\n");
   });
 
   uint64_t curr_key = 0;

file/delete_scheduler.cc

@@ -7,7 +7,6 @@
 
 #include "file/delete_scheduler.h"
 
-#include <cinttypes>
 #include <thread>
 #include <vector>
 
@@ -66,12 +65,6 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path,
     s = fs_->DeleteFile(file_path, IOOptions(), nullptr);
     if (s.ok()) {
       sst_file_manager_->OnDeleteFile(file_path);
-      ROCKS_LOG_INFO(info_log_,
-                     "Deleted file %s immediately, rate_bytes_per_sec %" PRIi64
-                     ", total_trash_size %" PRIu64 " max_trash_db_ratio %lf",
-                     file_path.c_str(), rate_bytes_per_sec_.load(),
-                     total_trash_size_.load(), max_trash_db_ratio_.load());
-      RecordTick(stats_.get(), FILES_DELETED_IMMEDIATELY);
     }
     return s;
   }
@@ -79,8 +72,6 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path,
   // Move file to trash
   std::string trash_file;
   s = MarkAsTrash(file_path, &trash_file);
-  ROCKS_LOG_INFO(info_log_, "Mark file: %s as trash -- %s", trash_file.c_str(),
-                 s.ToString().c_str());
 
   if (!s.ok()) {
     ROCKS_LOG_ERROR(info_log_, "Failed to mark %s as trash -- %s",
@@ -88,14 +79,10 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path,
     s = fs_->DeleteFile(file_path, IOOptions(), nullptr);
     if (s.ok()) {
       sst_file_manager_->OnDeleteFile(file_path);
-      ROCKS_LOG_INFO(info_log_, "Deleted file %s immediately",
-                     trash_file.c_str());
-      RecordTick(stats_.get(), FILES_DELETED_IMMEDIATELY);
     }
     return s;
   }
 
-  RecordTick(stats_.get(), FILES_MARKED_TRASH);
   // Update the total trash size
   uint64_t trash_file_size = 0;
   fs_->GetFileSize(trash_file, IOOptions(), &trash_file_size, nullptr);
@@ -223,8 +210,6 @@ void DeleteScheduler::BackgroundEmptyTrash() {
         current_delete_rate = rate_bytes_per_sec_.load();
         start_time = env_->NowMicros();
         total_deleted_bytes = 0;
-        ROCKS_LOG_INFO(info_log_, "rate_bytes_per_sec is changed to %" PRIi64,
-                       current_delete_rate);
       }
 
       // Get new file to delete
@@ -248,27 +233,19 @@ void DeleteScheduler::BackgroundEmptyTrash() {
         bg_errors_[path_in_trash] = s;
       }
 
-      // Apply penalty if necessary
-      uint64_t total_penalty;
+      // Apply penlty if necessary
+      uint64_t total_penlty;
       if (current_delete_rate > 0) {
         // rate limiting is enabled
-        total_penalty =
+        total_penlty =
             ((total_deleted_bytes * kMicrosInSecond) / current_delete_rate);
-        ROCKS_LOG_INFO(info_log_,
-                       "Rate limiting is enabled with penalty %" PRIu64
-                       "after deleting file %s",
-                       total_penalty, path_in_trash.c_str());
-        while (!closing_ && !cv_.TimedWait(start_time + total_penalty)) {
-        }
+        while (!closing_ && !cv_.TimedWait(start_time + total_penlty)) {}
       } else {
         // rate limiting is disabled
-        total_penalty = 0;
-        ROCKS_LOG_INFO(info_log_,
-                       "Rate limiting is disabled after deleting file %s",
-                       path_in_trash.c_str());
+        total_penlty = 0;
       }
       TEST_SYNC_POINT_CALLBACK("DeleteScheduler::BackgroundEmptyTrash:Wait",
-                               &total_penalty);
+                               &total_penlty);
 
       if (is_complete) {
         pending_files_--;
@@ -376,13 +353,9 @@ void DeleteScheduler::WaitForEmptyTrash() {
 }
 
 void DeleteScheduler::MaybeCreateBackgroundThread() {
-  if (bg_thread_ == nullptr && rate_bytes_per_sec_.load() > 0) {
+  if(bg_thread_ == nullptr && rate_bytes_per_sec_.load() > 0) {
     bg_thread_.reset(
         new port::Thread(&DeleteScheduler::BackgroundEmptyTrash, this));
-    ROCKS_LOG_INFO(info_log_,
-                   "Created background thread for deletion scheduler with "
-                   "rate_bytes_per_sec: %" PRIi64,
-                   rate_bytes_per_sec_.load());
   }
 }
 

file/delete_scheduler.h

@@ -83,10 +83,6 @@ class DeleteScheduler {
   static Status CleanupDirectory(Env* env, SstFileManagerImpl* sfm,
                                  const std::string& path);
 
-  void SetStatisticsPtr(const std::shared_ptr<Statistics>& stats) {
-    stats_ = stats;
-  }
-
  private:
   Status MarkAsTrash(const std::string& file_path, std::string* path_in_trash);
 
@@ -141,7 +137,6 @@ class DeleteScheduler {
   // immediately
   std::atomic<double> max_trash_db_ratio_;
   static const uint64_t kMicrosInSecond = 1000 * 1000LL;
-  std::shared_ptr<Statistics> stats_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

file/delete_scheduler_test.cc

@@ -32,7 +32,6 @@ class DeleteSchedulerTest : public testing::Test {
           ToString(i));
       DestroyAndCreateDir(dummy_files_dirs_.back());
     }
-    stats_ = ROCKSDB_NAMESPACE::CreateDBStatistics();
   }
 
   ~DeleteSchedulerTest() override {
@@ -100,7 +99,6 @@ class DeleteSchedulerTest : public testing::Test {
         new SstFileManagerImpl(env_, fs, nullptr, rate_bytes_per_sec_,
                                /* max_trash_db_ratio= */ 1.1, 128 * 1024));
     delete_scheduler_ = sst_file_mgr_->delete_scheduler();
-    sst_file_mgr_->SetStatisticsPtr(stats_);
   }
 
   Env* env_;
@@ -108,7 +106,6 @@ class DeleteSchedulerTest : public testing::Test {
   int64_t rate_bytes_per_sec_;
   DeleteScheduler* delete_scheduler_;
   std::unique_ptr<SstFileManagerImpl> sst_file_mgr_;
-  std::shared_ptr<Statistics> stats_;
 };
 
 // Test the basic functionality of DeleteScheduler (Rate Limiting).
@@ -185,8 +182,6 @@ TEST_F(DeleteSchedulerTest, BasicRateLimiting) {
     ASSERT_EQ(num_files, dir_synced);
 
     ASSERT_EQ(CountTrashFiles(), 0);
-    ASSERT_EQ(num_files, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
-    ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
   }
 }
@@ -224,9 +219,6 @@ TEST_F(DeleteSchedulerTest, MultiDirectoryDeletionsScheduled) {
     ASSERT_EQ(0, CountTrashFiles(i));
   }
 
-  ASSERT_EQ(kNumFiles, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
-  ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
-
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
 
@@ -309,10 +301,6 @@ TEST_F(DeleteSchedulerTest, RateLimitingMultiThreaded) {
 
     ASSERT_EQ(CountNormalFiles(), 0);
     ASSERT_EQ(CountTrashFiles(), 0);
-    ASSERT_EQ(num_files * thread_cnt,
-              stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
-    ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
-
     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
   }
 }
@@ -330,9 +318,8 @@ TEST_F(DeleteSchedulerTest, DisableRateLimiting) {
 
   rate_bytes_per_sec_ = 0;
   NewDeleteScheduler();
-  constexpr int num_files = 10;
 
-  for (int i = 0; i < num_files; i++) {
+  for (int i = 0; i < 10; i++) {
     // Every file we delete will be deleted immediately
     std::string dummy_file = NewDummyFile("dummy.data");
     ASSERT_OK(delete_scheduler_->DeleteFile(dummy_file, ""));
@@ -342,9 +329,6 @@ TEST_F(DeleteSchedulerTest, DisableRateLimiting) {
   }
 
   ASSERT_EQ(bg_delete_file, 0);
-  ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
-  ASSERT_EQ(num_files,
-            stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
 
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
@@ -381,8 +365,6 @@ TEST_F(DeleteSchedulerTest, ConflictNames) {
 
   auto bg_errors = delete_scheduler_->GetBackgroundErrors();
   ASSERT_EQ(bg_errors.size(), 0);
-  ASSERT_EQ(10, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
-  ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
 
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }
@@ -457,12 +439,9 @@ TEST_F(DeleteSchedulerTest, StartBGEmptyTrashMultipleTimes) {
 
     auto bg_errors = delete_scheduler_->GetBackgroundErrors();
     ASSERT_EQ(bg_errors.size(), 0);
-    ASSERT_EQ(10, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
-    ASSERT_EQ(0, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
   }
 
   ASSERT_EQ(bg_delete_file, 50);
-
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
 }
 
@@ -674,8 +653,6 @@ TEST_F(DeleteSchedulerTest, ImmediateDeleteOn25PercDBSize) {
   // When we end up with 26 files in trash we will start
   // deleting new files immediately
   ASSERT_EQ(fg_delete_file, 74);
-  ASSERT_EQ(26, stats_->getAndResetTickerCount(FILES_MARKED_TRASH));
-  ASSERT_EQ(74, stats_->getAndResetTickerCount(FILES_DELETED_IMMEDIATELY));
 
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
 }

file/file_prefetch_buffer.cc

@@ -17,6 +17,7 @@
 #include "monitoring/iostats_context_imp.h"
 #include "port/port.h"
 #include "test_util/sync_point.h"
+#include "test_util/testharness.h"
 #include "util/random.h"
 #include "util/rate_limiter.h"
 

file/file_util.cc

@@ -17,31 +17,30 @@
 namespace ROCKSDB_NAMESPACE {
 
 // Utility function to copy a file up to a specified length
-IOStatus CopyFile(FileSystem* fs, const std::string& source,
-                  const std::string& destination, uint64_t size,
-                  bool use_fsync) {
+Status CopyFile(FileSystem* fs, const std::string& source,
+                const std::string& destination, uint64_t size, bool use_fsync) {
   const FileOptions soptions;
-  IOStatus io_s;
+  Status s;
   std::unique_ptr<SequentialFileReader> src_reader;
   std::unique_ptr<WritableFileWriter> dest_writer;
 
   {
     std::unique_ptr<FSSequentialFile> srcfile;
-    io_s = fs->NewSequentialFile(source, soptions, &srcfile, nullptr);
-    if (!io_s.ok()) {
-      return io_s;
+    s = fs->NewSequentialFile(source, soptions, &srcfile, nullptr);
+    if (!s.ok()) {
+      return s;
     }
     std::unique_ptr<FSWritableFile> destfile;
-    io_s = fs->NewWritableFile(destination, soptions, &destfile, nullptr);
-    if (!io_s.ok()) {
-      return io_s;
+    s = fs->NewWritableFile(destination, soptions, &destfile, nullptr);
+    if (!s.ok()) {
+      return s;
     }
 
     if (size == 0) {
       // default argument means copy everything
-      io_s = fs->GetFileSize(source, IOOptions(), &size, nullptr);
-      if (!io_s.ok()) {
-        return io_s;
+      s = fs->GetFileSize(source, IOOptions(), &size, nullptr);
+      if (!s.ok()) {
+        return s;
       }
     }
     src_reader.reset(new SequentialFileReader(std::move(srcfile), source));
@@ -53,16 +52,16 @@ IOStatus CopyFile(FileSystem* fs, const std::string& source,
   Slice slice;
   while (size > 0) {
     size_t bytes_to_read = std::min(sizeof(buffer), static_cast<size_t>(size));
-    io_s = status_to_io_status(src_reader->Read(bytes_to_read, &slice, buffer));
-    if (!io_s.ok()) {
-      return io_s;
+    s = src_reader->Read(bytes_to_read, &slice, buffer);
+    if (!s.ok()) {
+      return s;
     }
     if (slice.size() == 0) {
-      return IOStatus::Corruption("file too small");
+      return Status::Corruption("file too small");
     }
-    io_s = dest_writer->Append(slice);
-    if (!io_s.ok()) {
-      return io_s;
+    s = dest_writer->Append(slice);
+    if (!s.ok()) {
+      return s;
     }
     size -= slice.size();
   }
@@ -70,22 +69,22 @@ IOStatus CopyFile(FileSystem* fs, const std::string& source,
 }
 
 // Utility function to create a file with the provided contents
-IOStatus CreateFile(FileSystem* fs, const std::string& destination,
-                    const std::string& contents, bool use_fsync) {
+Status CreateFile(FileSystem* fs, const std::string& destination,
+                  const std::string& contents, bool use_fsync) {
   const EnvOptions soptions;
-  IOStatus io_s;
+  Status s;
   std::unique_ptr<WritableFileWriter> dest_writer;
 
   std::unique_ptr<FSWritableFile> destfile;
-  io_s = fs->NewWritableFile(destination, soptions, &destfile, nullptr);
-  if (!io_s.ok()) {
-    return io_s;
+  s = fs->NewWritableFile(destination, soptions, &destfile, nullptr);
+  if (!s.ok()) {
+    return s;
   }
   dest_writer.reset(
       new WritableFileWriter(std::move(destfile), destination, soptions));
-  io_s = dest_writer->Append(Slice(contents));
-  if (!io_s.ok()) {
-    return io_s;
+  s = dest_writer->Append(Slice(contents));
+  if (!s.ok()) {
+    return s;
   }
   return dest_writer->Sync(use_fsync);
 }
@@ -122,68 +121,4 @@ bool IsWalDirSameAsDBPath(const ImmutableDBOptions* db_options) {
   return same;
 }
 
-IOStatus GenerateOneFileChecksum(FileSystem* fs, const std::string& file_path,
-                                 FileChecksumGenFactory* checksum_factory,
-                                 std::string* file_checksum,
-                                 std::string* file_checksum_func_name,
-                                 size_t verify_checksums_readahead_size,
-                                 bool allow_mmap_reads) {
-  if (checksum_factory == nullptr) {
-    return IOStatus::InvalidArgument("Checksum factory is invalid");
-  }
-  assert(file_checksum != nullptr);
-  assert(file_checksum_func_name != nullptr);
-
-  FileChecksumGenContext gen_context;
-  std::unique_ptr<FileChecksumGenerator> checksum_generator =
-      checksum_factory->CreateFileChecksumGenerator(gen_context);
-  uint64_t size;
-  IOStatus io_s;
-  std::unique_ptr<RandomAccessFileReader> reader;
-  {
-    std::unique_ptr<FSRandomAccessFile> r_file;
-    io_s = fs->NewRandomAccessFile(file_path, FileOptions(), &r_file, nullptr);
-    if (!io_s.ok()) {
-      return io_s;
-    }
-    io_s = fs->GetFileSize(file_path, IOOptions(), &size, nullptr);
-    if (!io_s.ok()) {
-      return io_s;
-    }
-    reader.reset(new RandomAccessFileReader(std::move(r_file), file_path));
-  }
-
-  // Found that 256 KB readahead size provides the best performance, based on
-  // experiments, for auto readahead. Experiment data is in PR #3282.
-  size_t default_max_read_ahead_size = 256 * 1024;
-  size_t readahead_size = (verify_checksums_readahead_size != 0)
-                              ? verify_checksums_readahead_size
-                              : default_max_read_ahead_size;
-
-  FilePrefetchBuffer prefetch_buffer(
-      reader.get(), readahead_size /* readadhead_size */,
-      readahead_size /* max_readahead_size */, !allow_mmap_reads /* enable */);
-
-  Slice slice;
-  uint64_t offset = 0;
-  while (size > 0) {
-    size_t bytes_to_read =
-        static_cast<size_t>(std::min(uint64_t{readahead_size}, size));
-    if (!prefetch_buffer.TryReadFromCache(offset, bytes_to_read, &slice,
-                                          false)) {
-      return IOStatus::Corruption("file read failed");
-    }
-    if (slice.size() == 0) {
-      return IOStatus::Corruption("file too small");
-    }
-    checksum_generator->Update(slice.data(), slice.size());
-    size -= slice.size();
-    offset += slice.size();
-  }
-  checksum_generator->Finalize();
-  *file_checksum = checksum_generator->GetChecksum();
-  *file_checksum_func_name = checksum_generator->Name();
-  return IOStatus::OK();
-}
-
 }  // namespace ROCKSDB_NAMESPACE