Skip CancelAllBackgroundWork if DBImpl is already closed (#6268 )

Summary: WritePreparedTxnDB calls CancelAllBackgroundWork in its destructor to avoid dangling references to it from background job's SnapshotChecker callback. However, if the DBImpl is already closed, the info log might be closed with it, which causes memory leak when CancelAllBackgroundWork tries to print to the info log. The patch fixes that by calling CancelAllBackgroundWork only if the db is not closed already. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6268 Differential Revision: D19303439 Pulled By: maysamyabandeh fbshipit-source-id: 4228a6be7e78d43c90630347baa89b008200bd15
parallel occ (#6240 )
2026-11-06 11:02:36 -05:00 · 2020-01-07 15:34:27 -08:00 · 2020-01-07 14:20:38 -08:00 · 2020-01-07 13:56:13 -08:00 · 2020-01-07 13:47:03 -08:00 · 2020-01-07 12:32:20 -08:00
286 changed files with 10149 additions and 2976 deletions
@@ -6,25 +6,34 @@ os:
 compiler:
  - clang
  - gcc
-osx_image: xcode8.3
+osx_image: xcode9.4
 jdk:
  - openjdk7
 cache:
  - ccache
-  - apt

 addons:
  apt:
    sources:
      - ubuntu-toolchain-r-test
    packages:
-      - curl
-      - g++-8
-      - libbz2-dev
      - libgflags-dev
+      - libbz2-dev
+      - liblz4-dev
      - libsnappy-dev
-      - mingw-w64
+      - liblzma-dev  # xv
+      - libzstd-dev
      - zlib1g-dev
+  homebrew:
+    update: true
+    packages:
+      - ccache
+      - gflags
+      - lz4
+      - snappy
+      - xz
+      - zstd
+
 env:
  - TEST_GROUP=platform_dependent # 16-18 minutes
  - TEST_GROUP=1 # 33-35 minutes
@@ -60,15 +69,17 @@ matrix:
  - os : osx
    compiler: gcc

-# https://docs.travis-ci.com/user/caching/#ccache-cache
 install:
  - if [ "${TRAVIS_OS_NAME}" == osx ]; then
-      brew install ccache zstd lz4 snappy xz;
      PATH=$PATH:/usr/local/opt/ccache/libexec;
    fi
  - if [ "${JOB_NAME}" == cmake-gcc8 ]; then
+      sudo apt-get install -y g++-8;
      CC=gcc-8 && CXX=g++-8;
    fi
+  - if [ "${JOB_NAME}" == cmake-mingw ]; then
+      sudo apt-get install -y mingw-w64 ;
+    fi
  - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then
      mkdir cmake-dist && curl -sfSL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH;
    fi
@@ -113,7 +124,7 @@ script:
      ;;
    cmake-mingw)
      sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix;
-      mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
+      mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
      ;;
    cmake*)
      mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release && make -j4 rocksdb rocksdbjni
@@ -45,6 +45,16 @@ if(POLICY CMP0042)
  cmake_policy(SET CMP0042 NEW)
 endif()

+if(NOT CMAKE_BUILD_TYPE)
+  if(EXISTS "${CMAKE_SOURCE_DIR}/.git")
+    set(default_build_type "Debug")
+  else()
+    set(default_build_type "RelWithDebInfo")
+  endif()
+  set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE STRING
+    "Default BUILD_TYPE is ${default_build_type}" FORCE)
+endif()
+
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
  set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
@@ -62,7 +72,7 @@ if (WITH_WINDOWS_UTF8_FILENAMES)
 endif()
 # third-party/folly is only validated to work on Linux and Windows for now.
 # So only turn it on there by default.
-if(CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_SYSTEM_NAME MATCHES "Windows")
+if(CMAKE_SYSTEM_NAME MATCHES "Linux|Windows")
  if(MSVC AND MSVC_VERSION LESS 1910)
    # Folly does not compile with MSVC older than VS2017
    option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF)
@@ -72,10 +82,12 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_SYSTEM_NAME MATCHES "Windows")
 else()
  option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF)
 endif()
+
+include(CMakeDependentOption)
+CMAKE_DEPENDENT_OPTION(WITH_GFLAGS "build with GFlags" ON
+  "NOT MSVC;NOT MINGW" OFF)
+
 if(MSVC)
-  # Defaults currently different for GFLAGS.
-  #  We will address find_package work a little later
-  option(WITH_GFLAGS "build with GFlags" OFF)
  option(WITH_XPRESS "build with windows built in compression" OFF)
  include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
 else()
@@ -92,14 +104,11 @@ else()
  endif()

  # No config file for this
-  option(WITH_GFLAGS "build with GFlags" ON)
  if(WITH_GFLAGS)
-    find_package(gflags)
-    if(gflags_FOUND)
-      add_definitions(-DGFLAGS=1)
-      include_directories(${gflags_INCLUDE_DIR})
-      list(APPEND THIRDPARTY_LIBS ${gflags_LIBRARIES})
-    endif()
+    find_package(gflags REQUIRED)
+    add_definitions(-DGFLAGS=1)
+    include_directories(${gflags_INCLUDE_DIR})
+    list(APPEND THIRDPARTY_LIBS gflags::gflags)
  endif()

  if(WITH_SNAPPY)
@@ -550,6 +559,7 @@ set(SOURCES
        env/env_chroot.cc
        env/env_encryption.cc
        env/env_hdfs.cc
+        env/file_system.cc
        env/mock_env.cc
        file/delete_scheduler.cc
        file/file_prefetch_buffer.cc
@@ -757,6 +767,7 @@ else()
  list(APPEND SOURCES
    port/port_posix.cc
    env/env_posix.cc
+    env/fs_posix.cc
    env/io_posix.cc)
 endif()

@@ -895,11 +906,15 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
  )
 endif()

-option(WITH_TESTS "build with tests" ON)
-# For test libraries, utilities, and exes that are build iff WITH_TESTS=ON and
-# in Debug mode. Add test only code that is not #ifdefed for Release here.
-if(WITH_TESTS AND CMAKE_BUILD_TYPE STREQUAL "Debug")
-  add_subdirectory(third-party/gtest-1.8.1/fused-src/gtest)
+add_subdirectory(third-party/gtest-1.8.1/fused-src/gtest)
+add_library(testharness STATIC
+  test_util/testharness.cc)
+target_link_libraries(testharness gtest)
+
+# Tests are excluded from Release builds
+CMAKE_DEPENDENT_OPTION(WITH_TESTS "build with tests" ON
+  "CMAKE_BUILD_TYPE STREQUAL Debug" OFF)
+if(WITH_TESTS)
  set(TESTS
        cache/cache_test.cc
        cache/lru_cache_test.cc
@@ -1015,6 +1030,7 @@ if(WITH_TESTS AND CMAKE_BUILD_TYPE STREQUAL "Debug")
        util/filelock_test.cc
        util/hash_test.cc
        util/heap_test.cc
+        util/random_test.cc
        util/rate_limiter_test.cc
        util/repeatable_thread_test.cc
        util/slice_transform_test.cc
@@ -1074,23 +1090,21 @@ if(WITH_TESTS AND CMAKE_BUILD_TYPE STREQUAL "Debug")
        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
        )

-  set(TEST_EXES ${TESTS})
-  foreach(sourcefile ${TEST_EXES})
+  foreach(sourcefile ${TESTS})
      get_filename_component(exename ${sourcefile} NAME_WE)
-      add_executable(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} ${sourcefile}
-        $<TARGET_OBJECTS:testharness>)
+      add_executable(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} ${sourcefile})
      set_target_properties(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX}
        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
        OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX}
        )
-      target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${ROCKSDB_LIB})
+      target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${ROCKSDB_LIB})
      if(NOT "${exename}" MATCHES "db_sanity_test")
        add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
        add_dependencies(check ${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX})
      endif()
-  endforeach(sourcefile ${TEST_EXES})
+  endforeach(sourcefile ${TESTS})

  if(WIN32)
    # C executables must link to a shared object
@@ -1105,50 +1119,51 @@ if(WITH_TESTS AND CMAKE_BUILD_TYPE STREQUAL "Debug")

  if(ROCKSDB_LIB_FOR_C)
    set(C_TESTS db/c_test.c)
-    set(C_TEST_EXES ${C_TESTS})
-
-    foreach(sourcefile ${C_TEST_EXES})
-        string(REPLACE ".c" "" exename ${sourcefile})
-        string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
-        add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
-        set_target_properties(${exename}${ARTIFACT_SUFFIX}
-          PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
-          EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
-          EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-          )
-        target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_LIB_FOR_C} testutillib${ARTIFACT_SUFFIX})
-        add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
-        add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
-    endforeach(sourcefile ${C_TEST_EXES})
+    # C executables must link to a shared object
+    add_executable(c_test db/c_test.c)
+    target_link_libraries(c_test ${ROCKSDB_SHARED_LIB} testharness)
+    add_test(NAME c_test COMMAND c_test${ARTIFACT_SUFFIX})
+    add_dependencies(check c_test)
  endif()
 endif()

 option(WITH_BENCHMARK_TOOLS "build with benchmarks" ON)
 if(WITH_BENCHMARK_TOOLS)
-  if(NOT TARGET gtest)
-    add_subdirectory(third-party/gtest-1.8.1/fused-src/gtest)
-  endif()
-  set(BENCHMARKS
-        cache/cache_bench.cc
-        memtable/memtablerep_bench.cc
-        db/range_del_aggregator_bench.cc
-        table/table_reader_bench.cc
-        util/filter_bench.cc
-        utilities/persistent_cache/hash_table_bench.cc
-  )
-  add_library(testharness OBJECT test_util/testharness.cc)
-  foreach(sourcefile ${BENCHMARKS})
-    get_filename_component(exename ${sourcefile} NAME_WE)
-    add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
-      $<TARGET_OBJECTS:testharness>)
-    target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${ROCKSDB_LIB})
-  endforeach(sourcefile ${BENCHMARKS})
-
-  add_executable(db_bench${ARTIFACT_SUFFIX}
+  add_executable(db_bench
    tools/db_bench.cc
-    tools/db_bench_tool.cc
-    $<TARGET_OBJECTS:testharness>)
-  target_link_libraries(db_bench${ARTIFACT_SUFFIX} gtest ${ROCKSDB_LIB})
+    tools/db_bench_tool.cc)
+  target_link_libraries(db_bench
+    ${ROCKSDB_LIB})
+
+  add_executable(cache_bench
+    cache/cache_bench.cc)
+  target_link_libraries(cache_bench
+    ${ROCKSDB_LIB})
+
+  add_executable(memtablerep_bench
+    memtable/memtablerep_bench.cc)
+  target_link_libraries(memtablerep_bench
+    ${ROCKSDB_LIB})
+
+  add_executable(range_del_aggregator_bench
+    db/range_del_aggregator_bench.cc)
+  target_link_libraries(range_del_aggregator_bench
+    ${ROCKSDB_LIB})
+
+  add_executable(table_reader_bench
+    table/table_reader_bench.cc)
+  target_link_libraries(table_reader_bench
+    ${ROCKSDB_LIB} testharness)
+
+  add_executable(filter_bench
+    util/filter_bench.cc)
+  target_link_libraries(filter_bench
+    ${ROCKSDB_LIB})
+
+  add_executable(hash_table_bench
+    utilities/persistent_cache/hash_table_bench.cc)
+  target_link_libraries(hash_table_bench
+    ${ROCKSDB_LIB})
 endif()

 option(WITH_TOOLS "build with tools" ON)
@@ -1,7 +1,26 @@
 # Rocksdb Change Log
 ## Unreleased
-### Buf Fixes
+### Public API Change
+* Added a rocksdb::FileSystem class in include/rocksdb/file_system.h to encapsulate file creation/read/write operations, and an option DBOptions::file_system to allow a user to pass in an instance of rocksdb::FileSystem. If its a non-null value, this will take precendence over DBOptions::env for file operations. A new API rocksdb::FileSystem::Default() returns a platform default object. The DBOptions::env option and Env::Default() API will continue to be used for threading and other OS related functions, and where DBOptions::file_system is not specified, for file operations. For storage developers who are accustomed to rocksdb::Env, the interface in rocksdb::FileSystem is new and will probably undergo some changes as more storage systems are ported to it from rocksdb::Env. As of now, no env other than Posix has been ported to the new interface.
+* A new rocksdb::NewSstFileManager() API that allows the caller to pass in separate Env and FileSystem objects.
+* Changed Java API for RocksDB.keyMayExist functions to use Holder<byte[]> instead of StringBuilder, so that retrieved values need not decode to Strings.
+* A new `OptimisticTransactionDBOptions` Option that allows users to configure occ validation policy. The default policy changes from kValidateSerial to kValidateParallel to reduce mutex contention.
+
+### Bug Fixes
 * Fix a bug that can cause unnecessary bg thread to be scheduled(#6104).
+* Fix a bug in which a snapshot read could be affected by a DeleteRange after the snapshot (#6062).
+* Fix crash caused by concurrent CF iterations and drops(#6147).
+* Fixed two performance issues related to memtable history trimming. First, a new SuperVersion is now created only if some memtables were actually trimmed. Second, trimming is only scheduled if there is at least one flushed memtable that is kept in memory for the purposes of transaction conflict checking.
+* Fix a bug in WriteBatchWithIndex::MultiGetFromBatchAndDB, which is called by Transaction::MultiGet, that causes due to stale pointer access when the number of keys is > 32
+* BlobDB no longer updates the SST to blob file mapping upon failed compactions.
+* Fixed a bug where BlobDB was comparing the `ColumnFamilyHandle` pointers themselves instead of only the column family IDs when checking whether an API call uses the default column family or not.
+* Fix a race condition for cfd->log_number_ between manifest switch and memtable switch (PR 6249) when number of column families is greater than 1.
+
+### New Features
+* It is now possible to enable periodic compactions for the base DB when using BlobDB.
+* BlobDB now garbage collects non-TTL blobs when `enable_garbage_collection` is set to `true` in `BlobDBOptions`. Garbage collection is performed during compaction: any valid blobs located in the oldest N files (where N is the number of non-TTL blob files multiplied by the value of `BlobDBOptions::garbage_collection_cutoff`) encountered during compaction get relocated to new blob files, and old blob files are dropped once they are no longer needed. Note: we recommend enabling periodic compactions for the base DB when using this feature to deal with the case when some old blob files are kept alive by SSTs that otherwise do not get picked for compaction.
+* `db_bench` now supports the `garbage_collection_cutoff` option for BlobDB.
+
 ## 6.6.0 (11/25/2019)
 ### Bug Fixes
 * Fix data corruption caused by output of intra-L0 compaction on ingested file not being placed in correct order in L0.
@@ -460,6 +460,7 @@ TESTS = \
 	env_test \
 	env_logger_test \
 	hash_test \
+	random_test \
 	thread_local_test \
 	rate_limiter_test \
 	perf_context_test \
@@ -744,7 +745,7 @@ endif  # PLATFORM_SHARED_EXT
 	dbg rocksdbjavastatic rocksdbjava install install-static install-shared uninstall \
 	analyze tools tools_lib \
 	blackbox_crash_test_with_atomic_flush whitebox_crash_test_with_atomic_flush  \
-	blackbox_crash_test_with_txn whitebox_crash_test_with_txn 
+	blackbox_crash_test_with_txn whitebox_crash_test_with_txn


 all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)
@@ -938,6 +939,7 @@ check: all
 	fi
 	rm -rf $(TMPD)
 ifneq ($(PLATFORM), OS_AIX)
+	python tools/check_all_python.py
 ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
 	python tools/ldb_test.py
 	sh tools/rocksdb_dump_test.sh
@@ -1095,6 +1097,9 @@ parallel_check: $(TESTS)
 		TMPD=$(TMPD) J=$(J) db_test=0 parloop;

 analyze: clean
+	USE_CLANG=1 $(MAKE) analyze_incremental
+
+analyze_incremental:
 	$(CLANG_SCAN_BUILD) --use-analyzer=$(CLANG_ANALYZER) \
 		--use-c++=$(CXX) --use-cc=$(CC) --status-bugs \
 		-o $(CURDIR)/scan_build_report \
@@ -1245,6 +1250,9 @@ coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
 hash_test: util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

+random_test: util/random_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 option_change_migration_test: utilities/option_change_migration/option_change_migration_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -2047,6 +2055,7 @@ jtest_run:

 jtest: rocksdbjava
 	cd java;$(MAKE) sample;$(MAKE) test;
+	python tools/check_all_python.py # TODO peterd: find a better place for this check in CI targets

 jdb_bench:
 	cd java;$(MAKE) db_bench;
@@ -181,6 +181,8 @@ cpp_library(
        "env/env_encryption.cc",
        "env/env_hdfs.cc",
        "env/env_posix.cc",
+        "env/file_system.cc",
+        "env/fs_posix.cc",
        "env/io_posix.cc",
        "env/mock_env.cc",
        "file/delete_scheduler.cc",
@@ -1245,6 +1247,13 @@ ROCKS_TESTS = [
        [],
        [],
    ],
+    [
+        "random_test",
+        "util/random_test.cc",
+        "serial",
+        [],
+        [],
+    ],
    [
        "range_del_aggregator_test",
        "db/range_del_aggregator_test.cc",
@@ -13,9 +13,14 @@ then
  echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
  echo "You can download clang-format-diff.py by running: "
  echo "    curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}"
-  echo "You can download clang-format by running: "
+  echo "You can download clang-format by running:"
  echo "    brew install clang-format"
+  echo "  Or"
+  echo "    apt install clang-format"
+  echo "  This might work too:"
+  echo "    yum install git-clang-format"
  echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}"
+  echo "and make sure ${CLANG_FORMAT_DIFF} is executable."
  exit 128
 fi

@@ -0,0 +1,29 @@
+# - Find gflags library
+# Find the gflags includes and library
+#
+# gflags_INCLUDE_DIR - where to find gflags.h.
+# gflags_LIBRARIES - List of libraries when using gflags.
+# gflags_FOUND - True if gflags found.
+
+find_path(GFLAGS_INCLUDE_DIR
+  NAMES gflags/gflags.h)
+
+find_library(GFLAGS_LIBRARIES
+  NAMES gflags)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(gflags
+  DEFAULT_MSG GFLAGS_LIBRARIES GFLAGS_INCLUDE_DIR)
+
+mark_as_advanced(
+  GFLAGS_LIBRARIES
+  GFLAGS_INCLUDE_DIR)
+
+if(gflags_FOUND AND NOT (TARGET gflags::gflags))
+  add_library(gflags::gflags UNKNOWN IMPORTED)
+  set_target_properties(gflags::gflags
+    PROPERTIES
+      IMPORTED_LOCATION ${GFLAGS_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${GFLAGS_INCLUDE_DIR}
+      IMPORTED_LINK_INTERFACE_LANGUAGES "CXX")
+endif()
@@ -64,7 +64,7 @@ Status ArenaWrappedDBIter::Refresh() {
    arena_.~Arena();
    new (&arena_) Arena();

-    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_->mutex());
+    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_);
    if (read_callback_) {
      read_callback_->Refresh(latest_seq);
    }
@@ -66,8 +66,9 @@ TableBuilder* NewTableBuilder(
 }

 Status BuildTable(
-    const std::string& dbname, Env* env, const ImmutableCFOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
+    const std::string& dbname, Env* env, FileSystem* fs,
+    const ImmutableCFOptions& ioptions,
+    const MutableCFOptions& mutable_cf_options, const FileOptions& file_options,
    TableCache* table_cache, InternalIterator* iter,
    std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
        range_del_iters,
@@ -115,12 +116,12 @@ Status BuildTable(
    compression_opts_for_flush.max_dict_bytes = 0;
    compression_opts_for_flush.zstd_max_train_bytes = 0;
    {
-      std::unique_ptr<WritableFile> file;
+      std::unique_ptr<FSWritableFile> file;
 #ifndef NDEBUG
-      bool use_direct_writes = env_options.use_direct_writes;
+      bool use_direct_writes = file_options.use_direct_writes;
      TEST_SYNC_POINT_CALLBACK("BuildTable:create_file", &use_direct_writes);
 #endif  // !NDEBUG
-      s = NewWritableFile(env, fname, &file, env_options);
+      s = NewWritableFile(fs, fname, &file, file_options);
      if (!s.ok()) {
        EventHelpers::LogAndNotifyTableFileCreationFinished(
            event_logger, ioptions.listeners, dbname, column_family_name, fname,
@@ -131,7 +132,7 @@ Status BuildTable(
      file->SetWriteLifeTimeHint(write_hint);

      file_writer.reset(
-          new WritableFileWriter(std::move(file), fname, env_options, env,
+          new WritableFileWriter(std::move(file), fname, file_options, env,
                                 ioptions.statistics, ioptions.listeners));
      builder = NewTableBuilder(
          ioptions, mutable_cf_options, internal_comparator,
@@ -218,7 +219,7 @@ Status BuildTable(
      // we will regrad this verification as user reads since the goal is
      // to cache it here for further user reads
      std::unique_ptr<InternalIterator> it(table_cache->NewIterator(
-          ReadOptions(), env_options, internal_comparator, *meta,
+          ReadOptions(), file_options, internal_comparator, *meta,
          nullptr /* range_del_agg */,
          mutable_cf_options.prefix_extractor.get(), nullptr,
          (internal_stats == nullptr) ? nullptr
@@ -241,7 +242,7 @@ Status BuildTable(
  }

  if (!s.ok() || meta->fd.GetFileSize() == 0) {
-    env->DeleteFile(fname);
+    fs->DeleteFile(fname, IOOptions(), nullptr);
  }

  if (meta->fd.GetFileSize() == 0) {
@@ -62,8 +62,9 @@ TableBuilder* NewTableBuilder(
 // @param column_family_name Name of the column family that is also identified
 //    by column_family_id, or empty string if unknown.
 extern Status BuildTable(
-    const std::string& dbname, Env* env, const ImmutableCFOptions& options,
-    const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
+    const std::string& dbname, Env* env, FileSystem* fs,
+    const ImmutableCFOptions& options,
+    const MutableCFOptions& mutable_cf_options, const FileOptions& file_options,
    TableCache* table_cache, InternalIterator* iter,
    std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
        range_del_iters,
@@ -852,6 +852,17 @@ void rocksdb_delete_cf(
        Slice(key, keylen)));
 }

+void rocksdb_delete_range_cf(rocksdb_t* db,
+                             const rocksdb_writeoptions_t* options,
+                             rocksdb_column_family_handle_t* column_family,
+                             const char* start_key, size_t start_key_len,
+                             const char* end_key, size_t end_key_len,
+                             char** errptr) {
+  SaveError(errptr, db->rep->DeleteRange(options->rep, column_family->rep,
+                                         Slice(start_key, start_key_len),
+                                         Slice(end_key, end_key_len)));
+}
+
 void rocksdb_merge(
    rocksdb_t* db,
    const rocksdb_writeoptions_t* options,
@@ -1244,6 +1244,15 @@ int main(int argc, char** argv) {
    rocksdb_put_cf(db, woptions, handles[1], "foo", 3, "hello", 5, &err);
    CheckNoError(err);

+    rocksdb_put_cf(db, woptions, handles[1], "foobar1", 7, "hello1", 6, &err);
+    CheckNoError(err);
+    rocksdb_put_cf(db, woptions, handles[1], "foobar2", 7, "hello2", 6, &err);
+    CheckNoError(err);
+    rocksdb_put_cf(db, woptions, handles[1], "foobar3", 7, "hello3", 6, &err);
+    CheckNoError(err);
+    rocksdb_put_cf(db, woptions, handles[1], "foobar4", 7, "hello4", 6, &err);
+    CheckNoError(err);
+
    rocksdb_flushoptions_t *flush_options = rocksdb_flushoptions_create();
    rocksdb_flushoptions_set_wait(flush_options, 1);
    rocksdb_flush_cf(db, flush_options, handles[1], &err);
@@ -1256,6 +1265,10 @@ int main(int argc, char** argv) {
    rocksdb_delete_cf(db, woptions, handles[1], "foo", 3, &err);
    CheckNoError(err);

+    rocksdb_delete_range_cf(db, woptions, handles[1], "foobar2", 7, "foobar4",
+                            7, &err);
+    CheckNoError(err);
+
    CheckGetCF(db, roptions, handles[1], "foo", NULL);
    CheckPinGetCF(db, roptions, handles[1], "foo", NULL);

@@ -1308,7 +1321,7 @@ int main(int argc, char** argv) {
    for (i = 0; rocksdb_iter_valid(iter) != 0; rocksdb_iter_next(iter)) {
      i++;
    }
-    CheckCondition(i == 1);
+    CheckCondition(i == 3);
    rocksdb_iter_get_error(iter, &err);
    CheckNoError(err);
    rocksdb_iter_destroy(iter);
@@ -1332,7 +1345,7 @@ int main(int argc, char** argv) {
    for (i = 0; rocksdb_iter_valid(iter) != 0; rocksdb_iter_next(iter)) {
      i++;
    }
-    CheckCondition(i == 1);
+    CheckCondition(i == 3);
    rocksdb_iter_get_error(iter, &err);
    CheckNoError(err);
    rocksdb_iter_destroy(iter);
@@ -60,11 +60,8 @@ ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
    ColumnFamilyOptions initial_cf_options_copy = cfd_->initial_cf_options();
    JobContext job_context(0);
    mutex_->Lock();
-    if (cfd_->Unref()) {
-      bool dropped = cfd_->IsDropped();
-
-      delete cfd_;
-
+    bool dropped = cfd_->IsDropped();
+    if (cfd_->UnrefAndTryDelete()) {
      if (dropped) {
        db_->FindObsoleteFiles(&job_context, false, true);
      }
@@ -163,6 +160,11 @@ Status CheckConcurrentWritesSupported(const ColumnFamilyOptions& cf_options) {
    return Status::InvalidArgument(
        "Memtable doesn't concurrent writes (allow_concurrent_memtable_write)");
  }
+  if (cf_options.max_successive_merges != 0) {
+    return Status::InvalidArgument(
+        "max_successive_merges > 0 is incompatible "
+        "with concurrent writes (allow_concurrent_memtable_write)");
+  }
  return Status::OK();
 }

@@ -439,13 +441,18 @@ void SuperVersion::Cleanup() {
    to_delete.push_back(m);
  }
  current->Unref();
+  if (cfd->Unref()) {
+    delete cfd;
+  }
 }

-void SuperVersion::Init(MemTable* new_mem, MemTableListVersion* new_imm,
-                        Version* new_current) {
+void SuperVersion::Init(ColumnFamilyData* new_cfd, MemTable* new_mem,
+                        MemTableListVersion* new_imm, Version* new_current) {
+  cfd = new_cfd;
  mem = new_mem;
  imm = new_imm;
  current = new_current;
+  cfd->Ref();
  mem->Ref();
  imm->Ref();
  current->Ref();
@@ -473,7 +480,7 @@ ColumnFamilyData::ColumnFamilyData(
    uint32_t id, const std::string& name, Version* _dummy_versions,
    Cache* _table_cache, WriteBufferManager* write_buffer_manager,
    const ColumnFamilyOptions& cf_options, const ImmutableDBOptions& db_options,
-    const EnvOptions& env_options, ColumnFamilySet* column_family_set,
+    const FileOptions& file_options, ColumnFamilySet* column_family_set,
    BlockCacheTracer* const block_cache_tracer)
    : id_(id),
      name_(name),
@@ -515,7 +522,7 @@ ColumnFamilyData::ColumnFamilyData(
  if (_dummy_versions != nullptr) {
    internal_stats_.reset(
        new InternalStats(ioptions_.num_levels, db_options.env, this));
-    table_cache_.reset(new TableCache(ioptions_, env_options, _table_cache,
+    table_cache_.reset(new TableCache(ioptions_, file_options, _table_cache,
                                      block_cache_tracer));
    if (ioptions_.compaction_style == kCompactionStyleLevel) {
      compaction_picker_.reset(
@@ -581,21 +588,7 @@ ColumnFamilyData::~ColumnFamilyData() {
  // compaction_queue_ and we destroyed it
  assert(!queued_for_flush_);
  assert(!queued_for_compaction_);
-
-  if (super_version_ != nullptr) {
-    // Release SuperVersion reference kept in ThreadLocalPtr.
-    // This must be done outside of mutex_ since unref handler can lock mutex.
-    super_version_->db_mutex->Unlock();
-    local_sv_.reset();
-    super_version_->db_mutex->Lock();
-
-    bool is_last_reference __attribute__((__unused__));
-    is_last_reference = super_version_->Unref();
-    assert(is_last_reference);
-    super_version_->Cleanup();
-    delete super_version_;
-    super_version_ = nullptr;
-  }
+  assert(super_version_ == nullptr);

  if (dummy_versions_ != nullptr) {
    // List must be empty
@@ -615,6 +608,36 @@ ColumnFamilyData::~ColumnFamilyData() {
  }
 }

+bool ColumnFamilyData::UnrefAndTryDelete() {
+  int old_refs = refs_.fetch_sub(1);
+  assert(old_refs > 0);
+
+  if (old_refs == 1) {
+    assert(super_version_ == nullptr);
+    delete this;
+    return true;
+  }
+
+  if (old_refs == 2 && super_version_ != nullptr) {
+    // Only the super_version_ holds me
+    SuperVersion* sv = super_version_;
+    super_version_ = nullptr;
+    // Release SuperVersion reference kept in ThreadLocalPtr.
+    // This must be done outside of mutex_ since unref handler can lock mutex.
+    sv->db_mutex->Unlock();
+    local_sv_.reset();
+    sv->db_mutex->Lock();
+
+    if (sv->Unref()) {
+      // May delete this ColumnFamilyData after calling Cleanup()
+      sv->Cleanup();
+      delete sv;
+      return true;
+    }
+  }
+  return false;
+}
+
 void ColumnFamilyData::SetDropped() {
  // can't drop default CF
  assert(id_ != 0);
@@ -949,8 +972,8 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
  return write_stall_condition;
 }

-const EnvOptions* ColumnFamilyData::soptions() const {
-  return &(column_family_set_->env_options_);
+const FileOptions* ColumnFamilyData::soptions() const {
+  return &(column_family_set_->file_options_);
 }

 void ColumnFamilyData::SetCurrent(Version* current_version) {
@@ -1080,9 +1103,8 @@ Compaction* ColumnFamilyData::CompactRange(
  return result;
 }

-SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(
-    InstrumentedMutex* db_mutex) {
-  SuperVersion* sv = GetThreadLocalSuperVersion(db_mutex);
+SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(DBImpl* db) {
+  SuperVersion* sv = GetThreadLocalSuperVersion(db);
  sv->Ref();
  if (!ReturnThreadLocalSuperVersion(sv)) {
    // This Unref() corresponds to the Ref() in GetThreadLocalSuperVersion()
@@ -1094,8 +1116,7 @@ SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(
  return sv;
 }

-SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(
-    InstrumentedMutex* db_mutex) {
+SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(DBImpl* db) {
  // The SuperVersion is cached in thread local storage to avoid acquiring
  // mutex when SuperVersion does not change since the last use. When a new
  // SuperVersion is installed, the compaction or flush thread cleans up
@@ -1122,16 +1143,21 @@ SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(

    if (sv && sv->Unref()) {
      RecordTick(ioptions_.statistics, NUMBER_SUPERVERSION_CLEANUPS);
-      db_mutex->Lock();
+      db->mutex()->Lock();
      // NOTE: underlying resources held by superversion (sst files) might
      // not be released until the next background job.
      sv->Cleanup();
-      sv_to_delete = sv;
+      if (db->immutable_db_options().avoid_unnecessary_blocking_io) {
+        db->AddSuperVersionsToFreeQueue(sv);
+        db->SchedulePurge();
+      } else {
+        sv_to_delete = sv;
+      }
    } else {
-      db_mutex->Lock();
+      db->mutex()->Lock();
    }
    sv = super_version_->Ref();
-    db_mutex->Unlock();
+    db->mutex()->Unlock();

    delete sv_to_delete;
  }
@@ -1169,7 +1195,7 @@ void ColumnFamilyData::InstallSuperVersion(
  SuperVersion* new_superversion = sv_context->new_superversion.release();
  new_superversion->db_mutex = db_mutex;
  new_superversion->mutable_cf_options = mutable_cf_options;
-  new_superversion->Init(mem_, imm_.current(), current_);
+  new_superversion->Init(this, mem_, imm_.current(), current_);
  SuperVersion* old_superversion = super_version_;
  super_version_ = new_superversion;
  ++super_version_number_;
@@ -1285,6 +1311,10 @@ Env::WriteLifeTimeHint ColumnFamilyData::CalculateSSTWriteHint(int level) {
  // L1: medium, L2: long, ...
  if (level - base_level >= 2) {
    return Env::WLTH_EXTREME;
+  } else if (level < base_level) {
+    // There is no restriction which prevents level passed in to be smaller
+    // than base_level.
+    return Env::WLTH_MEDIUM;
  }
  return static_cast<Env::WriteLifeTimeHint>(level - base_level +
                            static_cast<int>(Env::WLTH_MEDIUM));
@@ -1317,7 +1347,7 @@ Directory* ColumnFamilyData::GetDataDir(size_t path_id) const {

 ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
                                 const ImmutableDBOptions* db_options,
-                                 const EnvOptions& env_options,
+                                 const FileOptions& file_options,
                                 Cache* table_cache,
                                 WriteBufferManager* write_buffer_manager,
                                 WriteController* write_controller,
@@ -1325,11 +1355,11 @@ ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
    : max_column_family_(0),
      dummy_cfd_(new ColumnFamilyData(
          0, "", nullptr, nullptr, nullptr, ColumnFamilyOptions(), *db_options,
-          env_options, nullptr, block_cache_tracer)),
+          file_options, nullptr, block_cache_tracer)),
      default_cfd_cache_(nullptr),
      db_name_(dbname),
      db_options_(db_options),
-      env_options_(env_options),
+      file_options_(file_options),
      table_cache_(table_cache),
      write_buffer_manager_(write_buffer_manager),
      write_controller_(write_controller),
@@ -1344,14 +1374,12 @@ ColumnFamilySet::~ColumnFamilySet() {
    // cfd destructor will delete itself from column_family_data_
    auto cfd = column_family_data_.begin()->second;
    bool last_ref __attribute__((__unused__));
-    last_ref = cfd->Unref();
+    last_ref = cfd->UnrefAndTryDelete();
    assert(last_ref);
-    delete cfd;
  }
  bool dummy_last_ref __attribute__((__unused__));
-  dummy_last_ref = dummy_cfd_->Unref();
+  dummy_last_ref = dummy_cfd_->UnrefAndTryDelete();
  assert(dummy_last_ref);
-  delete dummy_cfd_;
 }

 ColumnFamilyData* ColumnFamilySet::GetDefault() const {
@@ -1401,7 +1429,7 @@ ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
  assert(column_families_.find(name) == column_families_.end());
  ColumnFamilyData* new_cfd = new ColumnFamilyData(
      id, name, dummy_versions, table_cache_, write_buffer_manager_, options,
-      *db_options_, env_options_, this, block_cache_tracer_);
+      *db_options_, file_options_, this, block_cache_tracer_);
  column_families_.insert({name, id});
  column_family_data_.insert({id, new_cfd});
  max_column_family_ = std::max(max_column_family_, id);
@@ -198,6 +198,7 @@ class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
 struct SuperVersion {
  // Accessing members of this class is not thread-safe and requires external
  // synchronization (ie db mutex held or on write thread).
+  ColumnFamilyData* cfd;
  MemTable* mem;
  MemTableListVersion* imm;
  Version* current;
@@ -221,8 +222,8 @@ struct SuperVersion {
  // that needs to be deleted in to_delete vector. Unrefing those
  // objects needs to be done in the mutex
  void Cleanup();
-  void Init(MemTable* new_mem, MemTableListVersion* new_imm,
-            Version* new_current);
+  void Init(ColumnFamilyData* new_cfd, MemTable* new_mem,
+            MemTableListVersion* new_imm, Version* new_current);

  // The value of dummy is not actually used. kSVInUse takes its address as a
  // mark in the thread local storage to indicate the SuperVersion is in use
@@ -288,6 +289,11 @@ class ColumnFamilyData {
    return old_refs == 1;
  }

+  // UnrefAndTryDelete() decreases the reference count and do free if needed,
+  // return true if this is freed else false, UnrefAndTryDelete() can only
+  // be called while holding a DB mutex, or during single-threaded recovery.
+  bool UnrefAndTryDelete();
+
  // SetDropped() can only be called under following conditions:
  // 1) Holding a DB mutex,
  // 2) from single-threaded write thread, AND
@@ -318,7 +324,7 @@ class ColumnFamilyData {
  }
  FlushReason GetFlushReason() const { return flush_reason_; }
  // thread-safe
-  const EnvOptions* soptions() const;
+  const FileOptions* soptions() const;
  const ImmutableCFOptions* ioptions() const { return &ioptions_; }
  // REQUIRES: DB mutex held
  // This returns the MutableCFOptions used by current SuperVersion
@@ -430,11 +436,11 @@ class ColumnFamilyData {
  SuperVersion* GetSuperVersion() { return super_version_; }
  // thread-safe
  // Return a already referenced SuperVersion to be used safely.
-  SuperVersion* GetReferencedSuperVersion(InstrumentedMutex* db_mutex);
+  SuperVersion* GetReferencedSuperVersion(DBImpl* db);
  // thread-safe
  // Get SuperVersion stored in thread local storage. If it does not exist,
  // get a reference from a current SuperVersion.
-  SuperVersion* GetThreadLocalSuperVersion(InstrumentedMutex* db_mutex);
+  SuperVersion* GetThreadLocalSuperVersion(DBImpl* db);
  // Try to return SuperVersion back to thread local storage. Retrun true on
  // success and false on failure. It fails when the thread local storage
  // contains anything other than SuperVersion::kSVInUse flag.
@@ -504,7 +510,7 @@ class ColumnFamilyData {
                   WriteBufferManager* write_buffer_manager,
                   const ColumnFamilyOptions& options,
                   const ImmutableDBOptions& db_options,
-                   const EnvOptions& env_options,
+                   const FileOptions& file_options,
                   ColumnFamilySet* column_family_set,
                   BlockCacheTracer* const block_cache_tracer);

@@ -632,7 +638,7 @@ class ColumnFamilySet {

  ColumnFamilySet(const std::string& dbname,
                  const ImmutableDBOptions* db_options,
-                  const EnvOptions& env_options, Cache* table_cache,
+                  const FileOptions& file_options, Cache* table_cache,
                  WriteBufferManager* write_buffer_manager,
                  WriteController* write_controller,
                  BlockCacheTracer* const block_cache_tracer);
@@ -690,7 +696,7 @@ class ColumnFamilySet {

  const std::string db_name_;
  const ImmutableDBOptions* const db_options_;
-  const EnvOptions env_options_;
+  const FileOptions file_options_;
  Cache* table_cache_;
  WriteBufferManager* write_buffer_manager_;
  WriteController* write_controller_;
@@ -67,9 +67,9 @@ class ColumnFamilyTestBase : public testing::Test {
 #ifndef ROCKSDB_LITE
    const char* test_env_uri = getenv("TEST_ENV_URI");
    if (test_env_uri) {
-      Status s = ObjectRegistry::NewInstance()->NewSharedObject(test_env_uri,
-                                                                &env_guard_);
-      base_env = env_guard_.get();
+      Env* test_env = nullptr;
+      Status s = Env::LoadEnv(test_env_uri, &test_env, &env_guard_);
+      base_env = test_env;
      EXPECT_OK(s);
      EXPECT_NE(Env::Default(), base_env);
    }
@@ -2365,6 +2365,45 @@ TEST_P(ColumnFamilyTest, ReadDroppedColumnFamily) {
  }
 }

+TEST_P(ColumnFamilyTest, LiveIteratorWithDroppedColumnFamily) {
+  db_options_.create_missing_column_families = true;
+  db_options_.max_open_files = 20;
+  // delete obsolete files always
+  db_options_.delete_obsolete_files_period_micros = 0;
+  Open({"default", "one", "two"});
+  ColumnFamilyOptions options;
+  options.level0_file_num_compaction_trigger = 100;
+  options.level0_slowdown_writes_trigger = 200;
+  options.level0_stop_writes_trigger = 200;
+  options.write_buffer_size = 100000;  // small write buffer size
+  Reopen({options, options, options});
+
+  // 1MB should create ~10 files for each CF
+  int kKeysNum = 10000;
+  PutRandomData(1, kKeysNum, 100);
+
+  {
+    std::unique_ptr<Iterator> iterator(
+        db_->NewIterator(ReadOptions(), handles_[1]));
+    iterator->SeekToFirst();
+
+    DropColumnFamilies({1});
+
+    // Make sure iterator created can still be used.
+    int count = 0;
+    for (; iterator->Valid(); iterator->Next()) {
+      ASSERT_OK(iterator->status());
+      ++count;
+    }
+    ASSERT_OK(iterator->status());
+    ASSERT_EQ(count, kKeysNum);
+  }
+
+  Reopen();
+  Close();
+  Destroy();
+}
+
 TEST_P(ColumnFamilyTest, FlushAndDropRaceCondition) {
  db_options_.create_missing_column_families = true;
  Open({"default", "one"});
@@ -275,9 +275,7 @@ Compaction::~Compaction() {
    input_version_->Unref();
  }
  if (cfd_ != nullptr) {
-    if (cfd_->Unref()) {
-      delete cfd_;
-    }
+    cfd_->UnrefAndTryDelete();
  }
 }

@@ -639,28 +639,48 @@ void CompactionIterator::NextFromInput() {
 }

 void CompactionIterator::PrepareOutput() {
-  // Zeroing out the sequence number leads to better compression.
-  // If this is the bottommost level (no files in lower levels)
-  // and the earliest snapshot is larger than this seqno
-  // and the userkey differs from the last userkey in compaction
-  // then we can squash the seqno to zero.
-  //
-  // This is safe for TransactionDB write-conflict checking since transactions
-  // only care about sequence number larger than any active snapshots.
-  //
-  // Can we do the same for levels above bottom level as long as
-  // KeyNotExistsBeyondOutputLevel() return true?
-  if ((compaction_ != nullptr && !compaction_->allow_ingest_behind()) &&
-      ikeyNotNeededForIncrementalSnapshot() && bottommost_level_ && valid_ &&
-      IN_EARLIEST_SNAPSHOT(ikey_.sequence) && ikey_.type != kTypeMerge) {
-    assert(ikey_.type != kTypeDeletion && ikey_.type != kTypeSingleDeletion);
-    if (ikey_.type == kTypeDeletion || ikey_.type == kTypeSingleDeletion) {
-      ROCKS_LOG_FATAL(info_log_,
-                      "Unexpected key type %d for seq-zero optimization",
-                      ikey_.type);
+  if (valid_) {
+    if (compaction_filter_ && ikey_.type == kTypeBlobIndex) {
+      const auto blob_decision = compaction_filter_->PrepareBlobOutput(
+          user_key(), value_, &compaction_filter_value_);
+
+      if (blob_decision == CompactionFilter::BlobDecision::kCorruption) {
+        status_ = Status::Corruption(
+            "Corrupted blob reference encountered during GC");
+        valid_ = false;
+      } else if (blob_decision == CompactionFilter::BlobDecision::kIOError) {
+        status_ = Status::IOError("Could not relocate blob during GC");
+        valid_ = false;
+      } else if (blob_decision ==
+                 CompactionFilter::BlobDecision::kChangeValue) {
+        value_ = compaction_filter_value_;
+      }
+    }
+
+    // Zeroing out the sequence number leads to better compression.
+    // If this is the bottommost level (no files in lower levels)
+    // and the earliest snapshot is larger than this seqno
+    // and the userkey differs from the last userkey in compaction
+    // then we can squash the seqno to zero.
+    //
+    // This is safe for TransactionDB write-conflict checking since transactions
+    // only care about sequence number larger than any active snapshots.
+    //
+    // Can we do the same for levels above bottom level as long as
+    // KeyNotExistsBeyondOutputLevel() return true?
+    if (valid_ && compaction_ != nullptr &&
+        !compaction_->allow_ingest_behind() &&
+        ikeyNotNeededForIncrementalSnapshot() && bottommost_level_ &&
+        IN_EARLIEST_SNAPSHOT(ikey_.sequence) && ikey_.type != kTypeMerge) {
+      assert(ikey_.type != kTypeDeletion && ikey_.type != kTypeSingleDeletion);
+      if (ikey_.type == kTypeDeletion || ikey_.type == kTypeSingleDeletion) {
+        ROCKS_LOG_FATAL(info_log_,
+                        "Unexpected key type %d for seq-zero optimization",
+                        ikey_.type);
+      }
+      ikey_.sequence = 0;
+      current_key_.UpdateInternalKey(0, ikey_.type);
    }
-    ikey_.sequence = 0;
-    current_key_.UpdateInternalKey(0, ikey_.type);
  }
 }

@@ -301,7 +301,7 @@ void CompactionJob::AggregateStatistics() {

 CompactionJob::CompactionJob(
    int job_id, Compaction* compaction, const ImmutableDBOptions& db_options,
-    const EnvOptions env_options, VersionSet* versions,
+    const FileOptions& file_options, VersionSet* versions,
    const std::atomic<bool>* shutting_down,
    const SequenceNumber preserve_deletes_seqnum, LogBuffer* log_buffer,
    Directory* db_directory, Directory* output_directory, Statistics* stats,
@@ -318,10 +318,11 @@ CompactionJob::CompactionJob(
      compaction_stats_(compaction->compaction_reason(), 1),
      dbname_(dbname),
      db_options_(db_options),
-      env_options_(env_options),
+      file_options_(file_options),
      env_(db_options.env),
-      env_options_for_read_(
-          env_->OptimizeForCompactionTableRead(env_options, db_options_)),
+      fs_(db_options.fs.get()),
+      file_options_for_read_(
+          fs_->OptimizeForCompactionTableRead(file_options, db_options_)),
      versions_(versions),
      shutting_down_(shutting_down),
      manual_compaction_paused_(manual_compaction_paused),
@@ -647,7 +648,7 @@ Status CompactionJob::Run() {
        // we will regard this verification as user reads since the goal is
        // to cache it here for further user reads
        InternalIterator* iter = cfd->table_cache()->NewIterator(
-            ReadOptions(), env_options_, cfd->internal_comparator(),
+            ReadOptions(), file_options_, cfd->internal_comparator(),
            *files_meta[file_idx], /*range_del_agg=*/nullptr, prefix_extractor,
            /*table_reader_ptr=*/nullptr,
            cfd->internal_stats()->GetFileReadHist(
@@ -836,7 +837,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  // Although the v2 aggregator is what the level iterator(s) know about,
  // the AddTombstones calls will be propagated down to the v1 aggregator.
  std::unique_ptr<InternalIterator> input(versions_->MakeInputIterator(
-      sub_compact->compaction, &range_del_agg, env_options_for_read_));
+      sub_compact->compaction, &range_del_agg, file_options_for_read_));

  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_COMPACTION_PROCESS_KV);
@@ -1457,13 +1458,13 @@ Status CompactionJob::OpenCompactionOutputFile(
      TableFileCreationReason::kCompaction);
 #endif  // !ROCKSDB_LITE
  // Make the output file
-  std::unique_ptr<WritableFile> writable_file;
+  std::unique_ptr<FSWritableFile> writable_file;
 #ifndef NDEBUG
-  bool syncpoint_arg = env_options_.use_direct_writes;
+  bool syncpoint_arg = file_options_.use_direct_writes;
  TEST_SYNC_POINT_CALLBACK("CompactionJob::OpenCompactionOutputFile",
                           &syncpoint_arg);
 #endif
-  Status s = NewWritableFile(env_, fname, &writable_file, env_options_);
+  Status s = NewWritableFile(fs_, fname, &writable_file, file_options_);
  if (!s.ok()) {
    ROCKS_LOG_ERROR(
        db_options_.info_log,
@@ -1506,14 +1507,14 @@ Status CompactionJob::OpenCompactionOutputFile(
    sub_compact->outputs.push_back(out);
  }

-  writable_file->SetIOPriority(Env::IO_LOW);
+  writable_file->SetIOPriority(Env::IOPriority::IO_LOW);
  writable_file->SetWriteLifeTimeHint(write_hint_);
  writable_file->SetPreallocationBlockSize(static_cast<size_t>(
      sub_compact->compaction->OutputFilePreallocationSize()));
  const auto& listeners =
      sub_compact->compaction->immutable_cf_options()->listeners;
  sub_compact->outfile.reset(
-      new WritableFileWriter(std::move(writable_file), fname, env_options_,
+      new WritableFileWriter(std::move(writable_file), fname, file_options_,
                             env_, db_options_.statistics.get(), listeners));

  // If the Column family flag is to only optimize filters for hits,
@@ -64,7 +64,7 @@ class CompactionJob {
 public:
  CompactionJob(int job_id, Compaction* compaction,
                const ImmutableDBOptions& db_options,
-                const EnvOptions env_options, VersionSet* versions,
+                const FileOptions& file_options, VersionSet* versions,
                const std::atomic<bool>* shutting_down,
                const SequenceNumber preserve_deletes_seqnum,
                LogBuffer* log_buffer, Directory* db_directory,
@@ -150,11 +150,12 @@ class CompactionJob {
  // DBImpl state
  const std::string& dbname_;
  const ImmutableDBOptions& db_options_;
-  const EnvOptions env_options_;
+  const FileOptions file_options_;

  Env* env_;
+  FileSystem* fs_;
  // env_option optimized for compaction table reads
-  EnvOptions env_options_for_read_;
+  FileOptions file_options_for_read_;
  VersionSet* versions_;
  const std::atomic<bool>* shutting_down_;
  const std::atomic<bool>* manual_compaction_paused_;
@@ -72,6 +72,7 @@ class CompactionJobTest : public testing::Test {
 public:
  CompactionJobTest()
      : env_(Env::Default()),
+        fs_(std::make_shared<LegacyFileSystemWrapper>(env_)),
        dbname_(test::PerThreadDBPath("compaction_job_test")),
        db_options_(),
        mutable_cf_options_(cf_options_),
@@ -86,6 +87,8 @@ class CompactionJobTest : public testing::Test {
        mock_table_factory_(new mock::MockTableFactory()),
        error_handler_(nullptr, db_options_, &mutex_) {
    EXPECT_OK(env_->CreateDirIfMissing(dbname_));
+    db_options_.env = env_;
+    db_options_.fs = fs_;
    db_options_.db_paths.emplace_back(dbname_,
                                      std::numeric_limits<uint64_t>::max());
  }
@@ -267,8 +270,8 @@ class CompactionJobTest : public testing::Test {
    Status s = env_->NewWritableFile(
        manifest, &file, env_->OptimizeForManifestWrite(env_options_));
    ASSERT_OK(s);
-    std::unique_ptr<WritableFileWriter> file_writer(
-        new WritableFileWriter(std::move(file), manifest, env_options_));
+    std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
+        NewLegacyWritableFileWrapper(std::move(file)), manifest, env_options_));
    {
      log::Writer log(std::move(file_writer), 0, false);
      std::string record;
@@ -360,6 +363,7 @@ class CompactionJobTest : public testing::Test {
  }

  Env* env_;
+  std::shared_ptr<FileSystem> fs_;
  std::string dbname_;
  EnvOptions env_options_;
  ImmutableDBOptions db_options_;
@@ -41,14 +41,16 @@ Status VerifySstFileChecksum(const Options& options,
                             const EnvOptions& env_options,
                             const ReadOptions& read_options,
                             const std::string& file_path) {
-  std::unique_ptr<RandomAccessFile> file;
+  std::unique_ptr<FSRandomAccessFile> file;
  uint64_t file_size;
  InternalKeyComparator internal_comparator(options.comparator);
  ImmutableCFOptions ioptions(options);

-  Status s = ioptions.env->NewRandomAccessFile(file_path, &file, env_options);
+  Status s = ioptions.fs->NewRandomAccessFile(file_path,
+                                              FileOptions(env_options),
+                                              &file, nullptr);
  if (s.ok()) {
-    s = ioptions.env->GetFileSize(file_path, &file_size);
+    s = ioptions.fs->GetFileSize(file_path, IOOptions(), &file_size, nullptr);
  } else {
    return s;
  }
@@ -20,6 +20,7 @@
 #include "db/db_test_util.h"
 #include "db/log_format.h"
 #include "db/version_set.h"
+#include "env/composite_env_wrapper.h"
 #include "file/filename.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/convenience.h"
@@ -189,6 +190,7 @@ class CorruptionTest : public testing::Test {
    ASSERT_TRUE(s.ok()) << s.ToString();
    Options options;
    EnvOptions env_options;
+    options.file_system.reset(new LegacyFileSystemWrapper(options.env));
    ASSERT_NOK(VerifySstFileChecksum(options, env_options, fname));
  }

@@ -539,7 +541,8 @@ TEST_F(CorruptionTest, RangeDeletionCorrupted) {
  std::unique_ptr<RandomAccessFile> file;
  ASSERT_OK(options_.env->NewRandomAccessFile(filename, &file, EnvOptions()));
  std::unique_ptr<RandomAccessFileReader> file_reader(
-      new RandomAccessFileReader(std::move(file), filename));
+      new RandomAccessFileReader(NewLegacyRandomAccessFileWrapper(file),
+                                 filename));

  uint64_t file_size;
  ASSERT_OK(options_.env->GetFileSize(filename, &file_size));
@@ -6,7 +6,6 @@
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-// #include <iostream>
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
 #include "rocksdb/perf_context.h"
@@ -525,6 +524,7 @@ TEST_F(DBBasicTest, Snapshot) {
    ASSERT_EQ(1U, GetNumSnapshots());
    uint64_t time_snap1 = GetTimeOldestSnapshots();
    ASSERT_GT(time_snap1, 0U);
+    ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber());
    Put(0, "foo", "0v2");
    Put(1, "foo", "1v2");

@@ -533,6 +533,7 @@ TEST_F(DBBasicTest, Snapshot) {
    const Snapshot* s2 = db_->GetSnapshot();
    ASSERT_EQ(2U, GetNumSnapshots());
    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
+    ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber());
    Put(0, "foo", "0v3");
    Put(1, "foo", "1v3");

@@ -540,6 +541,7 @@ TEST_F(DBBasicTest, Snapshot) {
      ManagedSnapshot s3(db_);
      ASSERT_EQ(3U, GetNumSnapshots());
      ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
+      ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber());

      Put(0, "foo", "0v4");
      Put(1, "foo", "1v4");
@@ -555,6 +557,7 @@ TEST_F(DBBasicTest, Snapshot) {

    ASSERT_EQ(2U, GetNumSnapshots());
    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
+    ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber());
    ASSERT_EQ("0v1", Get(0, "foo", s1));
    ASSERT_EQ("1v1", Get(1, "foo", s1));
    ASSERT_EQ("0v2", Get(0, "foo", s2));
@@ -569,9 +572,11 @@ TEST_F(DBBasicTest, Snapshot) {
    ASSERT_EQ("1v4", Get(1, "foo"));
    ASSERT_EQ(1U, GetNumSnapshots());
    ASSERT_LT(time_snap1, GetTimeOldestSnapshots());
+    ASSERT_EQ(GetSequenceOldestSnapshots(), s2->GetSequenceNumber());

    db_->ReleaseSnapshot(s2);
    ASSERT_EQ(0U, GetNumSnapshots());
+    ASSERT_EQ(GetSequenceOldestSnapshots(), 0);
    ASSERT_EQ("0v4", Get(0, "foo"));
    ASSERT_EQ("1v4", Get(1, "foo"));
  } while (ChangeOptions());
@@ -1706,6 +1711,14 @@ class DBBasicTestWithParallelIO
      assert(Put(Key(i), values_[i]) == Status::OK());
    }
    Flush();
+
+    for (int i = 0; i < 100; ++i) {
+      // block cannot gain space by compression
+      uncompressable_values_.emplace_back(RandomString(&rnd, 256) + '\0');
+      std::string tmp_key = "a" + Key(i);
+      assert(Put(tmp_key, uncompressable_values_[i]) == Status::OK());
+    }
+    Flush();
  }

  bool CheckValue(int i, const std::string& value) {
@@ -1715,6 +1728,13 @@ class DBBasicTestWithParallelIO
    return false;
  }

+  bool CheckUncompressableValue(int i, const std::string& value) {
+    if (uncompressable_values_[i].compare(value) == 0) {
+      return true;
+    }
+    return false;
+  }
+
  int num_lookups() { return uncompressed_cache_->num_lookups(); }
  int num_found() { return uncompressed_cache_->num_found(); }
  int num_inserts() { return uncompressed_cache_->num_inserts(); }
@@ -1864,6 +1884,7 @@ class DBBasicTestWithParallelIO
  std::shared_ptr<MyBlockCache> uncompressed_cache_;
  bool compression_enabled_;
  std::vector<std::string> values_;
+  std::vector<std::string> uncompressable_values_;
  bool fill_cache_;
 };

@@ -1928,8 +1949,66 @@ TEST_P(DBBasicTestWithParallelIO, MultiGet) {
    ASSERT_OK(statuses[i]);
    ASSERT_TRUE(CheckValue(key_ints[i], values[i].ToString()));
  }
-  expected_reads += (read_from_cache ? 2 : 4);
+  if (compression_enabled() && !has_compressed_cache()) {
+    expected_reads += (read_from_cache ? 2 : 3);
+  } else {
+    expected_reads += (read_from_cache ? 2 : 4);
+  }
  ASSERT_EQ(env_->random_read_counter_.Read(), expected_reads);
+
+  keys.resize(10);
+  statuses.resize(10);
+  std::vector<int> key_uncmp{1, 2, 15, 16, 55, 81, 82, 83, 84, 85};
+  for (size_t i = 0; i < key_uncmp.size(); ++i) {
+    key_data[i] = "a" + Key(key_uncmp[i]);
+    keys[i] = Slice(key_data[i]);
+    statuses[i] = Status::OK();
+    values[i].Reset();
+  }
+  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
+                     keys.data(), values.data(), statuses.data(), true);
+  for (size_t i = 0; i < key_uncmp.size(); ++i) {
+    ASSERT_OK(statuses[i]);
+    ASSERT_TRUE(CheckUncompressableValue(key_uncmp[i], values[i].ToString()));
+  }
+  if (compression_enabled() && !has_compressed_cache()) {
+    expected_reads += (read_from_cache ? 3 : 3);
+  } else {
+    expected_reads += (read_from_cache ? 4 : 4);
+  }
+  ASSERT_EQ(env_->random_read_counter_.Read(), expected_reads);
+
+  keys.resize(5);
+  statuses.resize(5);
+  std::vector<int> key_tr{1, 2, 15, 16, 55};
+  for (size_t i = 0; i < key_tr.size(); ++i) {
+    key_data[i] = "a" + Key(key_tr[i]);
+    keys[i] = Slice(key_data[i]);
+    statuses[i] = Status::OK();
+    values[i].Reset();
+  }
+  dbfull()->MultiGet(ro, dbfull()->DefaultColumnFamily(), keys.size(),
+                     keys.data(), values.data(), statuses.data(), true);
+  for (size_t i = 0; i < key_tr.size(); ++i) {
+    ASSERT_OK(statuses[i]);
+    ASSERT_TRUE(CheckUncompressableValue(key_tr[i], values[i].ToString()));
+  }
+  if (compression_enabled() && !has_compressed_cache()) {
+    expected_reads += (read_from_cache ? 0 : 2);
+    ASSERT_EQ(env_->random_read_counter_.Read(), expected_reads);
+  } else {
+    if (has_uncompressed_cache()) {
+      expected_reads += (read_from_cache ? 0 : 3);
+      ASSERT_EQ(env_->random_read_counter_.Read(), expected_reads);
+    } else {
+      // A rare case, even we enable the block compression but some of data
+      // blocks are not compressed due to content. If user only enable the
+      // compressed cache, the uncompressed blocks will not tbe cached, and
+      // block reads will be triggered. The number of reads is related to
+      // the compression algorithm.
+      ASSERT_TRUE(env_->random_read_counter_.Read() >= expected_reads);
+    }
+  }
 }

 INSTANTIATE_TEST_CASE_P(
@@ -10,6 +10,7 @@
 #include "cache/lru_cache.h"
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
+#include "util/compression.h"

 namespace rocksdb {

@@ -685,16 +686,18 @@ TEST_F(DBBlockCacheTest, CacheCompressionDict) {

  // Try all the available libraries that support dictionary compression
  std::vector<CompressionType> compression_types;
-#ifdef ZLIB
-  compression_types.push_back(kZlibCompression);
-#endif  // ZLIB
-#if LZ4_VERSION_NUMBER >= 10400
-  compression_types.push_back(kLZ4Compression);
-  compression_types.push_back(kLZ4HCCompression);
-#endif  // LZ4_VERSION_NUMBER >= 10400
-#if ZSTD_VERSION_NUMBER >= 500
-  compression_types.push_back(kZSTD);
-#endif  // ZSTD_VERSION_NUMBER >= 500
+  if (Zlib_Supported()) {
+    compression_types.push_back(kZlibCompression);
+  }
+  if (LZ4_Supported()) {
+    compression_types.push_back(kLZ4Compression);
+    compression_types.push_back(kLZ4HCCompression);
+  }
+  if (ZSTD_Supported()) {
+    compression_types.push_back(kZSTD);
+  } else if (ZSTDNotFinal_Supported()) {
+    compression_types.push_back(kZSTDNotFinalCompression);
+  }
  Random rnd(301);
  for (auto compression_type : compression_types) {
    Options options = CurrentOptions();
@@ -1503,6 +1503,37 @@ TEST_F(DBCompactionTest, DISABLED_ManualPartialFill) {
  }
 }

+TEST_F(DBCompactionTest, ManualCompactionWithUnorderedWrite) {
+  rocksdb::SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::WriteImpl:UnorderedWriteAfterWriteWAL",
+        "DBCompactionTest::ManualCompactionWithUnorderedWrite:WaitWriteWAL"},
+       {"DBImpl::WaitForPendingWrites:BeforeBlock",
+        "DBImpl::WriteImpl:BeforeUnorderedWriteMemtable"}});
+
+  Options options = CurrentOptions();
+  options.unordered_write = true;
+  DestroyAndReopen(options);
+  Put("foo", "v1");
+  ASSERT_OK(Flush());
+
+  Put("bar", "v1");
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+  port::Thread writer([&]() { Put("foo", "v2"); });
+
+  TEST_SYNC_POINT(
+      "DBCompactionTest::ManualCompactionWithUnorderedWrite:WaitWriteWAL");
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+
+  writer.join();
+  ASSERT_EQ(Get("foo"), "v2");
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+
+  Reopen(options);
+  ASSERT_EQ(Get("foo"), "v2");
+}
+
 TEST_F(DBCompactionTest, DeleteFileRange) {
  Options options = CurrentOptions();
  options.write_buffer_size = 10 * 1024 * 1024;
@@ -102,7 +102,7 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
        TEST_SYNC_POINT("DBImpl::GetLiveFiles:1");
        TEST_SYNC_POINT("DBImpl::GetLiveFiles:2");
        mutex_.Lock();
-        cfd->Unref();
+        cfd->UnrefAndTryDelete();
        if (!status.ok()) {
          break;
        }
@@ -52,6 +52,7 @@
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
 #include "db/write_callback.h"
+#include "env/composite_env_wrapper.h"
 #include "file/file_util.h"
 #include "file/filename.h"
 #include "file/random_access_file_reader.h"
@@ -102,6 +103,7 @@
 #include "util/string_util.h"

 namespace rocksdb {
+
 const std::string kDefaultColumnFamilyName("default");
 const std::string kPersistentStatsColumnFamilyName(
    "___rocksdb_stats_history___");
@@ -145,10 +147,11 @@ void DumpSupportInfo(Logger* logger) {

 DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
               const bool seq_per_batch, const bool batch_per_txn)
-    : env_(options.env),
-      dbname_(dbname),
+    : dbname_(dbname),
      own_info_log_(options.info_log == nullptr),
      initial_db_options_(SanitizeOptions(dbname, options)),
+      env_(initial_db_options_.env),
+      fs_(initial_db_options_.file_system),
      immutable_db_options_(initial_db_options_),
      mutable_db_options_(initial_db_options_),
      stats_(immutable_db_options_.statistics.get()),
@@ -156,9 +159,9 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
             immutable_db_options_.use_adaptive_mutex),
      default_cf_handle_(nullptr),
      max_total_in_memory_state_(0),
-      env_options_(BuildDBOptions(immutable_db_options_, mutable_db_options_)),
-      env_options_for_compaction_(env_->OptimizeForCompactionTableWrite(
-          env_options_, immutable_db_options_)),
+      file_options_(BuildDBOptions(immutable_db_options_, mutable_db_options_)),
+      file_options_for_compaction_(fs_->OptimizeForCompactionTableWrite(
+          file_options_, immutable_db_options_)),
      seq_per_batch_(seq_per_batch),
      batch_per_txn_(batch_per_txn),
      db_lock_(nullptr),
@@ -194,7 +197,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
      unable_to_release_oldest_log_(false),
      num_running_ingest_file_(0),
 #ifndef ROCKSDB_LITE
-      wal_manager_(immutable_db_options_, env_options_, seq_per_batch),
+      wal_manager_(immutable_db_options_, file_options_, seq_per_batch),
 #endif  // ROCKSDB_LITE
      event_logger_(immutable_db_options_.info_log.get()),
      bg_work_paused_(0),
@@ -240,7 +243,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
  co.metadata_charge_policy = kDontChargeCacheMetadata;
  table_cache_ = NewLRUCache(co);

-  versions_.reset(new VersionSet(dbname_, &immutable_db_options_, env_options_,
+  versions_.reset(new VersionSet(dbname_, &immutable_db_options_, file_options_,
                                 table_cache_.get(), write_buffer_manager_,
                                 &write_controller_, &block_cache_tracer_));
  column_family_memtables_.reset(
@@ -330,7 +333,7 @@ Status DBImpl::ResumeImpl() {
        mutex_.Unlock();
        s = FlushMemTable(cfd, flush_opts, FlushReason::kErrorRecovery);
        mutex_.Lock();
-        cfd->Unref();
+        cfd->UnrefAndTryDelete();
        if (!s.ok()) {
          break;
        }
@@ -418,7 +421,7 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
          mutex_.Unlock();
          FlushMemTable(cfd, FlushOptions(), FlushReason::kShutDown);
          mutex_.Lock();
-          cfd->Unref();
+          cfd->UnrefAndTryDelete();
        }
      }
    }
@@ -475,17 +478,12 @@ Status DBImpl::CloseHelper() {
  while (!flush_queue_.empty()) {
    const FlushRequest& flush_req = PopFirstFromFlushQueue();
    for (const auto& iter : flush_req) {
-      ColumnFamilyData* cfd = iter.first;
-      if (cfd->Unref()) {
-        delete cfd;
-      }
+      iter.first->UnrefAndTryDelete();
    }
  }
  while (!compaction_queue_.empty()) {
    auto cfd = PopFirstFromCompactionQueue();
-    if (cfd->Unref()) {
-      delete cfd;
-    }
+    cfd->UnrefAndTryDelete();
  }

  if (default_cf_handle_ != nullptr || persist_stats_cf_handle_ != nullptr) {
@@ -592,6 +590,7 @@ Status DBImpl::CloseHelper() {
      ret = s;
    }
  }
+
  if (ret.IsAborted()) {
    // Reserve IsAborted() error for those where users didn't release
    // certain resource and they can release them and come back and
@@ -872,7 +871,7 @@ Status DBImpl::TablesRangeTombstoneSummary(ColumnFamilyHandle* column_family,
          column_family);
  ColumnFamilyData* cfd = cfh->cfd();

-  SuperVersion* super_version = cfd->GetReferencedSuperVersion(&mutex_);
+  SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
  Version* version = super_version->current;

  Status s =
@@ -888,7 +887,6 @@ void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) {
      AddToLogsToFreeQueue(l);
    }
    job_context->logs_to_free.clear();
-    SchedulePurge();
  }
 }

@@ -1053,14 +1051,14 @@ Status DBImpl::SetDBOptions(
      wal_changed = mutable_db_options_.wal_bytes_per_sync !=
                    new_options.wal_bytes_per_sync;
      mutable_db_options_ = new_options;
-      env_options_for_compaction_ = EnvOptions(new_db_options);
-      env_options_for_compaction_ = env_->OptimizeForCompactionTableWrite(
-          env_options_for_compaction_, immutable_db_options_);
-      versions_->ChangeEnvOptions(mutable_db_options_);
+      file_options_for_compaction_ = FileOptions(new_db_options);
+      file_options_for_compaction_ = fs_->OptimizeForCompactionTableWrite(
+          file_options_for_compaction_, immutable_db_options_);
+      versions_->ChangeFileOptions(mutable_db_options_);
      //TODO(xiez): clarify why apply optimize for read to write options
-      env_options_for_compaction_ = env_->OptimizeForCompactionTableRead(
-          env_options_for_compaction_, immutable_db_options_);
-      env_options_for_compaction_.compaction_readahead_size =
+      file_options_for_compaction_ = fs_->OptimizeForCompactionTableRead(
+          file_options_for_compaction_, immutable_db_options_);
+      file_options_for_compaction_.compaction_readahead_size =
          mutable_db_options_.compaction_readahead_size;
      WriteThread::Writer w;
      write_thread_.EnterUnbatched(&w, &mutex_);
@@ -1315,19 +1313,34 @@ void DBImpl::BackgroundCallPurge() {
    delete log_writer;
    mutex_.Lock();
  }
-  for (const auto& file : purge_files_) {
-    const PurgeFileInfo& purge_file = file.second;
+  while (!superversions_to_free_queue_.empty()) {
+    assert(!superversions_to_free_queue_.empty());
+    SuperVersion* sv = superversions_to_free_queue_.front();
+    superversions_to_free_queue_.pop_front();
+    mutex_.Unlock();
+    delete sv;
+    mutex_.Lock();
+  }
+
+  // Can't use iterator to go over purge_files_ because inside the loop we're
+  // unlocking the mutex that protects purge_files_.
+  while (!purge_files_.empty()) {
+    auto it = purge_files_.begin();
+    // Need to make a copy of the PurgeFilesInfo before unlocking the mutex.
+    PurgeFileInfo purge_file = it->second;
+
    const std::string& fname = purge_file.fname;
    const std::string& dir_to_sync = purge_file.dir_to_sync;
    FileType type = purge_file.type;
    uint64_t number = purge_file.number;
    int job_id = purge_file.job_id;

+    purge_files_.erase(it);
+
    mutex_.Unlock();
    DeleteObsoleteFileImpl(job_id, fname, dir_to_sync, type, number);
    mutex_.Lock();
  }
-  purge_files_.clear();

  bg_purge_scheduled_--;

@@ -1367,10 +1380,14 @@ static void CleanupIteratorState(void* arg1, void* /*arg2*/) {
    state->db->FindObsoleteFiles(&job_context, false, true);
    if (state->background_purge) {
      state->db->ScheduleBgLogWriterClose(&job_context);
+      state->db->AddSuperVersionsToFreeQueue(state->super_version);
+      state->db->SchedulePurge();
    }
    state->mu->Unlock();

-    delete state->super_version;
+    if (!state->background_purge) {
+      delete state->super_version;
+    }
    if (job_context.HaveSomethingToDelete()) {
      if (state->background_purge) {
        // PurgeObsoleteFiles here does not delete files. Instead, it adds the
@@ -1427,7 +1444,7 @@ InternalIterator* DBImpl::NewInternalIterator(const ReadOptions& read_options,
  if (s.ok()) {
    // Collect iterators for files in L0 - Ln
    if (read_options.read_tier != kMemtableTier) {
-      super_version->current->AddIterators(read_options, env_options_,
+      super_version->current->AddIterators(read_options, file_options_,
                                           &merge_iter_builder, range_del_agg);
    }
    internal_iter = merge_iter_builder.Finish();
@@ -2445,7 +2462,7 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
    result = nullptr;

 #else
-    SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+    SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
    auto iter = new ForwardIterator(this, read_options, cfd, sv);
    result = NewDBIterator(
        env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
@@ -2471,7 +2488,7 @@ ArenaWrappedDBIter* DBImpl::NewIteratorImpl(const ReadOptions& read_options,
                                            ReadCallback* read_callback,
                                            bool allow_blob,
                                            bool allow_refresh) {
-  SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+  SuperVersion* sv = cfd->GetReferencedSuperVersion(this);

  // Try to generate a DB iterator tree in continuous memory area to be
  // cache friendly. Here is an example of result:
@@ -2550,7 +2567,7 @@ Status DBImpl::NewIterators(
 #else
    for (auto cfh : column_families) {
      auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
-      SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+      SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
      auto iter = new ForwardIterator(this, read_options, cfd, sv);
      iterators->push_back(NewDBIterator(
          env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
@@ -2722,6 +2739,15 @@ const std::string& DBImpl::GetName() const { return dbname_; }

 Env* DBImpl::GetEnv() const { return env_; }

+FileSystem* DB::GetFileSystem() const {
+  static LegacyFileSystemWrapper fs_wrap(GetEnv());
+  return &fs_wrap;
+}
+
+FileSystem* DBImpl::GetFileSystem() const {
+  return immutable_db_options_.fs.get();
+}
+
 Options DBImpl::GetOptions(ColumnFamilyHandle* column_family) const {
  InstrumentedMutexLock l(&mutex_);
  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
@@ -2877,7 +2903,7 @@ bool DBImpl::GetAggregatedIntProperty(const Slice& property,

 SuperVersion* DBImpl::GetAndRefSuperVersion(ColumnFamilyData* cfd) {
  // TODO(ljin): consider using GetReferencedSuperVersion() directly
-  return cfd->GetThreadLocalSuperVersion(&mutex_);
+  return cfd->GetThreadLocalSuperVersion(this);
 }

 // REQUIRED: this function should only be called on the write thread or if the
@@ -2895,11 +2921,19 @@ SuperVersion* DBImpl::GetAndRefSuperVersion(uint32_t column_family_id) {
 void DBImpl::CleanupSuperVersion(SuperVersion* sv) {
  // Release SuperVersion
  if (sv->Unref()) {
+    bool defer_purge =
+            immutable_db_options().avoid_unnecessary_blocking_io;
    {
      InstrumentedMutexLock l(&mutex_);
      sv->Cleanup();
+      if (defer_purge) {
+        AddSuperVersionsToFreeQueue(sv);
+        SchedulePurge();
+      }
+    }
+    if (!defer_purge) {
+      delete sv;
    }
-    delete sv;
    RecordTick(stats_, NUMBER_SUPERVERSION_CLEANUPS);
  }
  RecordTick(stats_, NUMBER_SUPERVERSION_RELEASES);
@@ -3294,12 +3328,12 @@ Status DBImpl::GetDbIdentity(std::string& identity) const {

 Status DBImpl::GetDbIdentityFromIdentityFile(std::string* identity) const {
  std::string idfilename = IdentityFileName(dbname_);
-  const EnvOptions soptions;
+  const FileOptions soptions;
  std::unique_ptr<SequentialFileReader> id_file_reader;
  Status s;
  {
-    std::unique_ptr<SequentialFile> idfile;
-    s = env_->NewSequentialFile(idfilename, &idfile, soptions);
+    std::unique_ptr<FSSequentialFile> idfile;
+    s = fs_->NewSequentialFile(idfilename, soptions, &idfile, nullptr);
    if (!s.ok()) {
      return s;
    }
@@ -3308,7 +3342,7 @@ Status DBImpl::GetDbIdentityFromIdentityFile(std::string* identity) const {
  }

  uint64_t file_size;
-  s = env_->GetFileSize(idfilename, &file_size);
+  s = fs_->GetFileSize(idfilename, IOOptions(), &file_size, nullptr);
  if (!s.ok()) {
    return s;
  }
@@ -3382,7 +3416,12 @@ Status DBImpl::Close() {
 Status DB::ListColumnFamilies(const DBOptions& db_options,
                              const std::string& name,
                              std::vector<std::string>* column_families) {
-  return VersionSet::ListColumnFamilies(column_families, name, db_options.env);
+  FileSystem* fs = db_options.file_system.get();
+  LegacyFileSystemWrapper legacy_fs(db_options.env);
+  if (!fs) {
+    fs = &legacy_fs;
+  }
+  return VersionSet::ListColumnFamilies(column_families, name, fs);
 }

 Snapshot::~Snapshot() {}
@@ -3547,8 +3586,8 @@ Status DBImpl::WriteOptionsFile(bool need_mutex_lock,

  std::string file_name =
      TempOptionsFileName(GetName(), versions_->NewFileNumber());
-  Status s =
-      PersistRocksDBOptions(db_options, cf_names, cf_opts, file_name, GetEnv());
+  Status s = PersistRocksDBOptions(db_options, cf_names, cf_opts, file_name,
+                                   GetFileSystem());

  if (s.ok()) {
    s = RenameTempFileToOptionsFile(file_name);
@@ -3892,7 +3931,7 @@ Status DBImpl::IngestExternalFiles(
  for (const auto& arg : args) {
    auto* cfd = static_cast<ColumnFamilyHandleImpl*>(arg.column_family)->cfd();
    ingestion_jobs.emplace_back(
-        env_, versions_.get(), cfd, immutable_db_options_, env_options_,
+        env_, versions_.get(), cfd, immutable_db_options_, file_options_,
        &snapshots_, arg.options, &directories_, &event_logger_);
  }
  std::vector<std::pair<bool, Status>> exec_results;
@@ -3905,7 +3944,7 @@ Status DBImpl::IngestExternalFiles(
    start_file_number += args[i - 1].external_files.size();
    auto* cfd =
        static_cast<ColumnFamilyHandleImpl*>(args[i].column_family)->cfd();
-    SuperVersion* super_version = cfd->GetReferencedSuperVersion(&mutex_);
+    SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
    exec_results[i].second = ingestion_jobs[i].Prepare(
        args[i].external_files, start_file_number, super_version);
    exec_results[i].first = true;
@@ -3916,7 +3955,7 @@ Status DBImpl::IngestExternalFiles(
  {
    auto* cfd =
        static_cast<ColumnFamilyHandleImpl*>(args[0].column_family)->cfd();
-    SuperVersion* super_version = cfd->GetReferencedSuperVersion(&mutex_);
+    SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
    exec_results[0].second = ingestion_jobs[0].Prepare(
        args[0].external_files, next_file_number, super_version);
    exec_results[0].first = true;
@@ -3958,6 +3997,13 @@ Status DBImpl::IngestExternalFiles(
      nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
    }

+    // When unordered_write is enabled, the keys are writing to memtable in an
+    // unordered way. If the ingestion job checks memtable key range before the
+    // key landing in memtable, the ingestion job may skip the necessary
+    // memtable flush.
+    // So wait here to ensure there is no pending write to memtable.
+    WaitForPendingWrites();
+
    num_running_ingest_file_ += static_cast<int>(num_cfs);
    TEST_SYNC_POINT("DBImpl::IngestExternalFile:AfterIncIngestFileCounter");

@@ -4149,7 +4195,7 @@ Status DBImpl::CreateColumnFamilyWithImport(
  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(*handle);
  auto cfd = cfh->cfd();
  ImportColumnFamilyJob import_job(env_, versions_.get(), cfd,
-                                   immutable_db_options_, env_options_,
+                                   immutable_db_options_, file_options_,
                                   import_options, metadata.files);

  SuperVersionContext dummy_sv_ctx(/* create_superversion */ true);
@@ -4185,7 +4231,7 @@ Status DBImpl::CreateColumnFamilyWithImport(
  dummy_sv_ctx.Clean();

  if (status.ok()) {
-    SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+    SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
    status = import_job.Prepare(next_file_number, sv);
    CleanupSuperVersion(sv);
  }
@@ -4262,7 +4308,7 @@ Status DBImpl::VerifyChecksum(const ReadOptions& read_options) {
  }
  std::vector<SuperVersion*> sv_list;
  for (auto cfd : cfd_list) {
-    sv_list.push_back(cfd->GetReferencedSuperVersion(&mutex_));
+    sv_list.push_back(cfd->GetReferencedSuperVersion(this));
  }
  for (auto& sv : sv_list) {
    VersionStorageInfo* vstorage = sv->current->storage_info();
@@ -4279,7 +4325,7 @@ Status DBImpl::VerifyChecksum(const ReadOptions& read_options) {
        const auto& fd = vstorage->LevelFilesBrief(i).files[j].fd;
        std::string fname = TableFileName(cfd->ioptions()->cf_paths,
                                          fd.GetNumber(), fd.GetPathId());
-        s = rocksdb::VerifySstFileChecksum(opts, env_options_, read_options,
+        s = rocksdb::VerifySstFileChecksum(opts, file_options_, read_options,
                                           fname);
      }
    }
@@ -4287,16 +4333,25 @@ Status DBImpl::VerifyChecksum(const ReadOptions& read_options) {
      break;
    }
  }
+  bool defer_purge =
+          immutable_db_options().avoid_unnecessary_blocking_io;
  {
    InstrumentedMutexLock l(&mutex_);
    for (auto sv : sv_list) {
      if (sv && sv->Unref()) {
        sv->Cleanup();
-        delete sv;
+        if (defer_purge) {
+          AddSuperVersionsToFreeQueue(sv);
+        } else {
+          delete sv;
+        }
      }
    }
+    if (defer_purge) {
+      SchedulePurge();
+    }
    for (auto cfd : cfd_list) {
-      cfd->Unref();
+      cfd->UnrefAndTryDelete();
    }
  }
  return s;
@@ -306,6 +306,7 @@ class DBImpl : public DB {
      ColumnFamilyHandle* column_family) override;
  virtual const std::string& GetName() const override;
  virtual Env* GetEnv() const override;
+  virtual FileSystem* GetFileSystem() const override;
  using DB::GetOptions;
  virtual Options GetOptions(ColumnFamilyHandle* column_family) const override;
  using DB::GetDBOptions;
@@ -798,6 +799,10 @@ class DBImpl : public DB {
    logs_to_free_queue_.push_back(log_writer);
  }

+  void AddSuperVersionsToFreeQueue(SuperVersion* sv) {
+    superversions_to_free_queue_.push_back(sv);
+  }
+
  void SetSnapshotChecker(SnapshotChecker* snapshot_checker);

  // Fill JobContext with snapshot information needed by flush and compaction.
@@ -941,13 +946,14 @@ class DBImpl : public DB {
 #endif  // NDEBUG

 protected:
-  Env* const env_;
  const std::string dbname_;
  std::string db_id_;
  std::unique_ptr<VersionSet> versions_;
  // Flag to check whether we allocated and own the info log file
  bool own_info_log_;
  const DBOptions initial_db_options_;
+  Env* const env_;
+  std::shared_ptr<FileSystem> fs_;
  const ImmutableDBOptions immutable_db_options_;
  MutableDBOptions mutable_db_options_;
  Statistics* stats_;
@@ -975,10 +981,10 @@ class DBImpl : public DB {
  bool single_column_family_mode_;

  // The options to access storage files
-  const EnvOptions env_options_;
+  const FileOptions file_options_;

  // Additonal options for compaction and flush
-  EnvOptions env_options_for_compaction_;
+  FileOptions file_options_for_compaction_;

  std::unique_ptr<ColumnFamilyMemTablesImpl> column_family_memtables_;

@@ -1195,7 +1201,7 @@ class DBImpl : public DB {
  };

  // PurgeFileInfo is a structure to hold information of files to be deleted in
-  // purge_queue_
+  // purge_files_
  struct PurgeFileInfo {
    std::string fname;
    std::string dir_to_sync;
@@ -1416,6 +1422,7 @@ class DBImpl : public DB {

  inline void WaitForPendingWrites() {
    mutex_.AssertHeld();
+    TEST_SYNC_POINT("DBImpl::WaitForPendingWrites:BeforeBlock");
    // In case of pipelined write is enabled, wait for all pending memtable
    // writers.
    if (immutable_db_options_.enable_pipelined_write) {
@@ -1888,6 +1895,7 @@ class DBImpl : public DB {

  // A queue to store log writers to close
  std::deque<log::Writer*> logs_to_free_queue_;
+  std::deque<SuperVersion*> superversions_to_free_queue_;
  int unscheduled_flushes_;
  int unscheduled_compactions_;

@@ -148,7 +148,7 @@ Status DBImpl::FlushMemTableToOutputFile(

  FlushJob flush_job(
      dbname_, cfd, immutable_db_options_, mutable_cf_options,
-      nullptr /* memtable_id */, env_options_for_compaction_, versions_.get(),
+      nullptr /* memtable_id */, file_options_for_compaction_, versions_.get(),
      &mutex_, &shutting_down_, snapshot_seqs, earliest_write_conflict_snapshot,
      snapshot_checker, job_context, log_buffer, directories_.GetDbDir(),
      GetDataDir(cfd, 0U),
@@ -332,7 +332,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    const uint64_t* max_memtable_id = &(bg_flush_args[i].max_memtable_id_);
    jobs.emplace_back(new FlushJob(
        dbname_, cfd, immutable_db_options_, mutable_cf_options,
-        max_memtable_id, env_options_for_compaction_, versions_.get(), &mutex_,
+        max_memtable_id, file_options_for_compaction_, versions_.get(), &mutex_,
        &shutting_down_, snapshot_seqs, earliest_write_conflict_snapshot,
        snapshot_checker, job_context, log_buffer, directories_.GetDbDir(),
        data_dir, GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
@@ -659,7 +659,7 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
    // one/both sides of the interval are unbounded. But it requires more
    // changes to RangesOverlapWithMemtables.
    Range range(*begin, *end);
-    SuperVersion* super_version = cfd->GetReferencedSuperVersion(&mutex_);
+    SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
    cfd->RangesOverlapWithMemtables({range}, super_version, &flush_needed);
    CleanupSuperVersion(super_version);
  }
@@ -968,7 +968,7 @@ Status DBImpl::CompactFilesImpl(
  CompactionJobStats compaction_job_stats;
  CompactionJob compaction_job(
      job_context->job_id, c.get(), immutable_db_options_,
-      env_options_for_compaction_, versions_.get(), &shutting_down_,
+      file_options_for_compaction_, versions_.get(), &shutting_down_,
      preserve_deletes_seqnum_.load(), log_buffer, directories_.GetDbDir(),
      GetDataDir(c->column_family_data(), c->output_path_id()), stats_, &mutex_,
      &error_handler_, snapshot_seqs, earliest_write_conflict_snapshot,
@@ -1540,6 +1540,7 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
        nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
      }
    }
+    WaitForPendingWrites();

    if (!cfd->mem()->IsEmpty() || !cached_recoverable_state_empty_.load()) {
      s = SwitchMemtable(cfd, &context);
@@ -1617,19 +1618,16 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
    }
    s = WaitForFlushMemTables(cfds, flush_memtable_ids,
                              (flush_reason == FlushReason::kErrorRecovery));
+    InstrumentedMutexLock lock_guard(&mutex_);
    for (auto* tmp_cfd : cfds) {
-      if (tmp_cfd->Unref()) {
-        // Only one thread can reach here.
-        InstrumentedMutexLock lock_guard(&mutex_);
-        delete tmp_cfd;
-      }
+      tmp_cfd->UnrefAndTryDelete();
    }
  }
-  TEST_SYNC_POINT("FlushMemTableFinished");
+  TEST_SYNC_POINT("DBImpl::FlushMemTable:FlushMemTableFinished");
  return s;
 }

-// Flush all elments in 'column_family_datas'
+// Flush all elements in 'column_family_datas'
 // and atomically record the result to the MANIFEST.
 Status DBImpl::AtomicFlushMemTables(
    const autovector<ColumnFamilyData*>& column_family_datas,
@@ -1665,6 +1663,7 @@ Status DBImpl::AtomicFlushMemTables(
        nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
      }
    }
+    WaitForPendingWrites();

    for (auto cfd : column_family_datas) {
      if (cfd->IsDropped()) {
@@ -1681,7 +1680,7 @@ Status DBImpl::AtomicFlushMemTables(
      }
      cfd->Ref();
      s = SwitchMemtable(cfd, &context);
-      cfd->Unref();
+      cfd->UnrefAndTryDelete();
      if (!s.ok()) {
        break;
      }
@@ -1721,12 +1720,9 @@ Status DBImpl::AtomicFlushMemTables(
    }
    s = WaitForFlushMemTables(cfds, flush_memtable_ids,
                              (flush_reason == FlushReason::kErrorRecovery));
+    InstrumentedMutexLock lock_guard(&mutex_);
    for (auto* cfd : cfds) {
-      if (cfd->Unref()) {
-        // Only one thread can reach here.
-        InstrumentedMutexLock lock_guard(&mutex_);
-        delete cfd;
-      }
+      cfd->UnrefAndTryDelete();
    }
  }
  return s;
@@ -2207,16 +2203,13 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
    *reason = bg_flush_args[0].cfd_->GetFlushReason();
    for (auto& arg : bg_flush_args) {
      ColumnFamilyData* cfd = arg.cfd_;
-      if (cfd->Unref()) {
-        delete cfd;
+      if (cfd->UnrefAndTryDelete()) {
        arg.cfd_ = nullptr;
      }
    }
  }
  for (auto cfd : column_families_not_to_flush) {
-    if (cfd->Unref()) {
-      delete cfd;
-    }
+    cfd->UnrefAndTryDelete();
  }
  return status;
 }
@@ -2545,10 +2538,9 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    // reference).
    // This will all happen under a mutex so we don't have to be afraid of
    // somebody else deleting it.
-    if (cfd->Unref()) {
+    if (cfd->UnrefAndTryDelete()) {
      // This was the last reference of the column family, so no need to
      // compact.
-      delete cfd;
      return Status::OK();
    }

@@ -2755,7 +2747,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    assert(is_snapshot_supported_ || snapshots_.empty());
    CompactionJob compaction_job(
        job_context->job_id, c.get(), immutable_db_options_,
-        env_options_for_compaction_, versions_.get(), &shutting_down_,
+        file_options_for_compaction_, versions_.get(), &shutting_down_,
        preserve_deletes_seqnum_.load(), log_buffer, directories_.GetDbDir(),
        GetDataDir(c->column_family_data(), c->output_path_id()), stats_,
        &mutex_, &error_handler_, snapshot_seqs,
@@ -12,6 +12,7 @@

 #include "db/builder.h"
 #include "db/error_handler.h"
+#include "env/composite_env_wrapper.h"
 #include "file/read_write_util.h"
 #include "file/sst_file_manager_impl.h"
 #include "file/writable_file_writer.h"
@@ -34,6 +35,18 @@ Options SanitizeOptions(const std::string& dbname, const Options& src) {
 DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
  DBOptions result(src);

+  if (result.file_system == nullptr) {
+    if (result.env == Env::Default()) {
+      result.file_system = FileSystem::Default();
+    } else {
+      result.file_system.reset(new LegacyFileSystemWrapper(result.env));
+    }
+  } else {
+    if (result.env == nullptr) {
+      result.env = Env::Default();
+    }
+  }
+
  // result.max_open_files means an "infinite" open files.
  if (result.max_open_files != -1) {
    int max_max_open_files = port::GetMaxOpenFiles();
@@ -254,16 +267,16 @@ Status DBImpl::NewDB() {
  ROCKS_LOG_INFO(immutable_db_options_.info_log, "Creating manifest 1 \n");
  const std::string manifest = DescriptorFileName(dbname_, 1);
  {
-    std::unique_ptr<WritableFile> file;
-    EnvOptions env_options = env_->OptimizeForManifestWrite(env_options_);
-    s = NewWritableFile(env_, manifest, &file, env_options);
+    std::unique_ptr<FSWritableFile> file;
+    FileOptions file_options = fs_->OptimizeForManifestWrite(file_options_);
+    s = NewWritableFile(fs_.get(), manifest, &file, file_options);
    if (!s.ok()) {
      return s;
    }
    file->SetPreallocationBlockSize(
        immutable_db_options_.manifest_preallocation_size);
    std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
-        std::move(file), manifest, env_options, env_, nullptr /* stats */,
+        std::move(file), manifest, file_options, env_, nullptr /* stats */,
        immutable_db_options_.listeners));
    log::Writer log(std::move(file_writer), 0, false);
    std::string record;
@@ -277,7 +290,7 @@ Status DBImpl::NewDB() {
    // Make "CURRENT" file that points to the new manifest file.
    s = SetCurrentFile(env_, dbname_, 1, directories_.GetDbDir());
  } else {
-    env_->DeleteFile(manifest);
+    fs_->DeleteFile(manifest, IOOptions(), nullptr);
  }
  return s;
 }
@@ -350,7 +363,8 @@ Status DBImpl::Recover(
      return s;
    }

-    s = env_->FileExists(CurrentFileName(dbname_));
+    std::string current_fname = CurrentFileName(dbname_);
+    s = env_->FileExists(current_fname);
    if (s.IsNotFound()) {
      if (immutable_db_options_.create_if_missing) {
        // Has to be called only after Identity File creation is successful
@@ -363,7 +377,7 @@ Status DBImpl::Recover(
        }
      } else {
        return Status::InvalidArgument(
-            dbname_, "does not exist (create_if_missing is false)");
+            current_fname, "does not exist (create_if_missing is false)");
      }
    } else if (s.ok()) {
      if (immutable_db_options_.error_if_exists) {
@@ -375,20 +389,20 @@ Status DBImpl::Recover(
      assert(s.IsIOError());
      return s;
    }
-    // Verify compatibility of env_options_ and filesystem
+    // Verify compatibility of file_options_ and filesystem
    {
-      std::unique_ptr<RandomAccessFile> idfile;
-      EnvOptions customized_env(env_options_);
-      customized_env.use_direct_reads |=
+      std::unique_ptr<FSRandomAccessFile> idfile;
+      FileOptions customized_fs(file_options_);
+      customized_fs.use_direct_reads |=
          immutable_db_options_.use_direct_io_for_flush_and_compaction;
-      s = env_->NewRandomAccessFile(CurrentFileName(dbname_), &idfile,
-                                    customized_env);
+      s = fs_->NewRandomAccessFile(current_fname, customized_fs, &idfile,
+                                   nullptr);
      if (!s.ok()) {
        std::string error_str = s.ToString();
        // Check if unsupported Direct I/O is the root cause
-        customized_env.use_direct_reads = false;
-        s = env_->NewRandomAccessFile(CurrentFileName(dbname_), &idfile,
-                                      customized_env);
+        customized_fs.use_direct_reads = false;
+        s = fs_->NewRandomAccessFile(current_fname, customized_fs, &idfile,
+                                     nullptr);
        if (s.ok()) {
          return Status::InvalidArgument(
              "Direct I/O is not supported by the specified DB.");
@@ -408,7 +422,7 @@ Status DBImpl::Recover(
  // the very first time.
  if (db_id_.empty()) {
    // Check for the IDENTITY file and create it if not there.
-    s = env_->FileExists(IdentityFileName(dbname_));
+    s = fs_->FileExists(IdentityFileName(dbname_), IOOptions(), nullptr);
    // Typically Identity file is created in NewDB() and for some reason if
    // it is no longer available then at this point DB ID is not in Identity
    // file or Manifest.
@@ -748,9 +762,10 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,

    std::unique_ptr<SequentialFileReader> file_reader;
    {
-      std::unique_ptr<SequentialFile> file;
-      status = env_->NewSequentialFile(fname, &file,
-                                       env_->OptimizeForLogRead(env_options_));
+      std::unique_ptr<FSSequentialFile> file;
+      status = fs_->NewSequentialFile(fname,
+                                      fs_->OptimizeForLogRead(file_options_),
+                                      &file, nullptr);
      if (!status.ok()) {
        MaybeIgnoreError(&status);
        if (!status.ok()) {
@@ -920,7 +935,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
        ColumnFamilyData* cfd;

        while ((cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) {
-          cfd->Unref();
+          cfd->UnrefAndTryDelete();
          // If this asserts, it means that InsertInto failed in
          // filtering updates to already-flushed column families
          assert(cfd->GetLogNumber() <= log_number);
@@ -1119,17 +1134,18 @@ Status DBImpl::RestoreAliveLogFiles(const std::vector<uint64_t>& log_numbers) {
    // preallocated space are not needed anymore. It is likely only the last
    // log has such preallocated space, so we only truncate for the last log.
    if (log_number == log_numbers.back()) {
-      std::unique_ptr<WritableFile> last_log;
-      Status truncate_status = env_->ReopenWritableFile(
-          fname, &last_log,
-          env_->OptimizeForLogWrite(
-              env_options_,
-              BuildDBOptions(immutable_db_options_, mutable_db_options_)));
+      std::unique_ptr<FSWritableFile> last_log;
+      Status truncate_status = fs_->ReopenWritableFile(
+          fname,
+          fs_->OptimizeForLogWrite(
+              file_options_,
+              BuildDBOptions(immutable_db_options_, mutable_db_options_)),
+          &last_log, nullptr);
      if (truncate_status.ok()) {
-        truncate_status = last_log->Truncate(log.size);
+        truncate_status = last_log->Truncate(log.size, IOOptions(), nullptr);
      }
      if (truncate_status.ok()) {
-        truncate_status = last_log->Close();
+        truncate_status = last_log->Close(IOOptions(), nullptr);
      }
      // Not a critical error if fail to truncate.
      if (!truncate_status.ok()) {
@@ -1196,8 +1212,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
        range_del_iters.emplace_back(range_del_iter);
      }
      s = BuildTable(
-          dbname_, env_, *cfd->ioptions(), mutable_cf_options,
-          env_options_for_compaction_, cfd->table_cache(), iter.get(),
+          dbname_, env_, fs_.get(), *cfd->ioptions(), mutable_cf_options,
+          file_options_for_compaction_, cfd->table_cache(), iter.get(),
          std::move(range_del_iters), &meta, cfd->internal_comparator(),
          cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
          snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker,
@@ -1280,12 +1296,12 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
 Status DBImpl::CreateWAL(uint64_t log_file_num, uint64_t recycle_log_number,
                         size_t preallocate_block_size, log::Writer** new_log) {
  Status s;
-  std::unique_ptr<WritableFile> lfile;
+  std::unique_ptr<FSWritableFile> lfile;

  DBOptions db_options =
      BuildDBOptions(immutable_db_options_, mutable_db_options_);
-  EnvOptions opt_env_options =
-      env_->OptimizeForLogWrite(env_options_, db_options);
+  FileOptions opt_file_options =
+      fs_->OptimizeForLogWrite(file_options_, db_options);
  std::string log_fname =
      LogFileName(immutable_db_options_.wal_dir, log_file_num);

@@ -1295,10 +1311,10 @@ Status DBImpl::CreateWAL(uint64_t log_file_num, uint64_t recycle_log_number,
                   recycle_log_number);
    std::string old_log_fname =
        LogFileName(immutable_db_options_.wal_dir, recycle_log_number);
-    s = env_->ReuseWritableFile(log_fname, old_log_fname, &lfile,
-                                opt_env_options);
+    s = fs_->ReuseWritableFile(log_fname, old_log_fname, opt_file_options,
+                               &lfile, /*dbg=*/nullptr);
  } else {
-    s = NewWritableFile(env_, log_fname, &lfile, opt_env_options);
+    s = NewWritableFile(fs_.get(), log_fname, &lfile, opt_file_options);
  }

  if (s.ok()) {
@@ -1307,7 +1323,7 @@ Status DBImpl::CreateWAL(uint64_t log_file_num, uint64_t recycle_log_number,

    const auto& listeners = immutable_db_options_.listeners;
    std::unique_ptr<WritableFileWriter> file_writer(
-        new WritableFileWriter(std::move(lfile), log_fname, opt_env_options,
+        new WritableFileWriter(std::move(lfile), log_fname, opt_file_options,
                               env_, nullptr /* stats */, listeners));
    *new_log = new log::Writer(std::move(file_writer), log_file_num,
                               immutable_db_options_.recycle_log_file_num > 0,
@@ -144,9 +144,10 @@ Status DBImplSecondary::MaybeInitLogReader(

    std::unique_ptr<SequentialFileReader> file_reader;
    {
-      std::unique_ptr<SequentialFile> file;
-      Status status = env_->NewSequentialFile(
-          fname, &file, env_->OptimizeForLogRead(env_options_));
+      std::unique_ptr<FSSequentialFile> file;
+      Status status = fs_->NewSequentialFile(
+          fname, fs_->OptimizeForLogRead(file_options_), &file,
+          nullptr);
      if (!status.ok()) {
        *log_reader = nullptr;
        return status;
@@ -406,7 +407,7 @@ ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl(
    const ReadOptions& read_options, ColumnFamilyData* cfd,
    SequenceNumber snapshot, ReadCallback* read_callback) {
  assert(nullptr != cfd);
-  SuperVersion* super_version = cfd->GetReferencedSuperVersion(&mutex_);
+  SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
  auto db_iter = NewArenaWrappedDbIterator(
      env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options,
      snapshot,
@@ -600,7 +601,7 @@ Status DB::OpenAsSecondary(
  handles->clear();
  DBImplSecondary* impl = new DBImplSecondary(tmp_opts, dbname);
  impl->versions_.reset(new ReactiveVersionSet(
-      dbname, &impl->immutable_db_options_, impl->env_options_,
+      dbname, &impl->immutable_db_options_, impl->file_options_,
      impl->table_cache_.get(), impl->write_buffer_manager_,
      &impl->write_controller_));
  impl->column_family_memtables_.reset(
@@ -131,6 +131,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
                              log_used, log_ref, &seq, sub_batch_cnt,
                              pre_release_callback, kDoAssignOrder,
                              kDoPublishLastSeq, disable_memtable);
+    TEST_SYNC_POINT("DBImpl::WriteImpl:UnorderedWriteAfterWriteWAL");
    if (!status.ok()) {
      return status;
    }
@@ -138,6 +139,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
      *seq_used = seq;
    }
    if (!disable_memtable) {
+      TEST_SYNC_POINT("DBImpl::WriteImpl:BeforeUnorderedWriteMemtable");
      status = UnorderedWriteMemtable(write_options, my_batch, callback,
                                      log_ref, seq, sub_batch_cnt);
    }
@@ -1254,7 +1256,7 @@ Status DBImpl::SwitchWAL(WriteContext* write_context) {
  for (const auto cfd : cfds) {
    cfd->Ref();
    status = SwitchMemtable(cfd, write_context);
-    cfd->Unref();
+    cfd->UnrefAndTryDelete();
    if (!status.ok()) {
      break;
    }
@@ -1333,7 +1335,7 @@ Status DBImpl::HandleWriteBufferFull(WriteContext* write_context) {
    }
    cfd->Ref();
    status = SwitchMemtable(cfd, write_context);
-    cfd->Unref();
+    cfd->UnrefAndTryDelete();
    if (!status.ok()) {
      break;
    }
@@ -1516,15 +1518,16 @@ Status DBImpl::TrimMemtableHistory(WriteContext* context) {
  for (auto& cfd : cfds) {
    autovector<MemTable*> to_delete;
    cfd->imm()->TrimHistory(&to_delete, cfd->mem()->ApproximateMemoryUsage());
-    for (auto m : to_delete) {
-      delete m;
+    if (!to_delete.empty()) {
+      for (auto m : to_delete) {
+        delete m;
+      }
+      context->superversion_context.NewSuperVersion();
+      assert(context->superversion_context.new_superversion.get() != nullptr);
+      cfd->InstallSuperVersion(&context->superversion_context, &mutex_);
    }
-    context->superversion_context.NewSuperVersion();
-    assert(context->superversion_context.new_superversion.get() != nullptr);
-    cfd->InstallSuperVersion(&context->superversion_context, &mutex_);

-    if (cfd->Unref()) {
-      delete cfd;
+    if (cfd->UnrefAndTryDelete()) {
      cfd = nullptr;
    }
  }
@@ -1556,8 +1559,7 @@ Status DBImpl::ScheduleFlushes(WriteContext* context) {
    if (!cfd->mem()->IsEmpty()) {
      status = SwitchMemtable(cfd, context);
    }
-    if (cfd->Unref()) {
-      delete cfd;
+    if (cfd->UnrefAndTryDelete()) {
      cfd = nullptr;
    }
    if (!status.ok()) {
@@ -804,6 +804,50 @@ TEST_P(DBIteratorTest, IteratorPinsRef) {
  } while (ChangeCompactOptions());
 }

+TEST_P(DBIteratorTest, IteratorDeleteAfterCfDelete) {
+  CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
+
+  Put(1, "foo", "delete-cf-then-delete-iter");
+  Put(1, "hello", "value2");
+
+  ColumnFamilyHandle* cf = handles_[1];
+  ReadOptions ro;
+
+  auto* iter = db_->NewIterator(ro, cf);
+  iter->SeekToFirst();
+  ASSERT_EQ(IterStatus(iter), "foo->delete-cf-then-delete-iter");
+
+  // delete CF handle
+  db_->DestroyColumnFamilyHandle(cf);
+  handles_.erase(std::begin(handles_) + 1);
+
+  // delete Iterator after CF handle is deleted
+  iter->Next();
+  ASSERT_EQ(IterStatus(iter), "hello->value2");
+  delete iter;
+}
+
+TEST_P(DBIteratorTest, IteratorDeleteAfterCfDrop) {
+  CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
+
+  Put(1, "foo", "drop-cf-then-delete-iter");
+
+  ReadOptions ro;
+  ColumnFamilyHandle* cf = handles_[1];
+
+  auto* iter = db_->NewIterator(ro, cf);
+  iter->SeekToFirst();
+  ASSERT_EQ(IterStatus(iter), "foo->drop-cf-then-delete-iter");
+
+  // drop and delete CF
+  db_->DropColumnFamily(cf);
+  db_->DestroyColumnFamilyHandle(cf);
+  handles_.erase(std::begin(handles_) + 1);
+
+  // delete Iterator after CF handle is dropped
+  delete iter;
+}
+
 // SetOptions not defined in ROCKSDB LITE
 #ifndef ROCKSDB_LITE
 TEST_P(DBIteratorTest, DBIteratorBoundTest) {
@@ -1297,19 +1341,19 @@ class DBIteratorTestForPinnedData : public DBIteratorTest {

      // Insert data to true_data map and to DB
      true_data[k] = v;
-      if (rnd.OneIn(static_cast<int>(100.0 / merge_percentage))) {
+      if (rnd.PercentTrue(merge_percentage)) {
        ASSERT_OK(db_->Merge(WriteOptions(), k, v));
      } else {
        ASSERT_OK(Put(k, v));
      }

      // Pick random keys to be used to test Seek()
-      if (rnd.OneIn(static_cast<int>(100.0 / seeks_percentage))) {
+      if (rnd.PercentTrue(seeks_percentage)) {
        random_keys.push_back(k);
      }

      // Delete some random keys
-      if (rnd.OneIn(static_cast<int>(100.0 / delete_percentage))) {
+      if (rnd.PercentTrue(delete_percentage)) {
        deleted_keys.push_back(k);
        true_data.erase(k);
        ASSERT_OK(Delete(k));
@@ -1867,7 +1911,7 @@ TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocksRandomized) {
  // make sure that we dont merge them while flushing but actually
  // merge them in the read path
  for (int i = 0; i < kNumKeys; i++) {
-    if (rnd.OneIn(static_cast<int>(100.0 / kNoMergeOpPercentage))) {
+    if (rnd.PercentTrue(kNoMergeOpPercentage)) {
      // Dont give merge operations for some keys
      continue;
    }
@@ -1883,7 +1927,7 @@ TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocksRandomized) {
  ASSERT_OK(Flush());

  for (int i = 0; i < kNumKeys; i++) {
-    if (rnd.OneIn(static_cast<int>(100.0 / kDeletePercentage))) {
+    if (rnd.PercentTrue(kDeletePercentage)) {
      gen_key = Key(i);

      ASSERT_OK(Delete(gen_key));
@@ -150,6 +150,8 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnWrite) {
  options.create_if_missing = true;
  options.merge_operator.reset(new TestPutOperator());
  options.max_successive_merges = 3;
+  // allow_concurrent_memtable_write is incompatible with max_successive_merges
+  options.allow_concurrent_memtable_write = false;
  options.env = env_;
  Reopen(options);
  ASSERT_OK(Merge("k1", "v1"));
@@ -1427,6 +1427,47 @@ TEST_F(DBRangeDelTest, SnapshotPreventsDroppedKeys) {
  db_->ReleaseSnapshot(snapshot);
 }

+TEST_F(DBRangeDelTest, SnapshotPreventsDroppedKeysInImmMemTables) {
+  const int kFileBytes = 1 << 20;
+
+  Options options = CurrentOptions();
+  options.compression = kNoCompression;
+  options.disable_auto_compactions = true;
+  options.target_file_size_base = kFileBytes;
+  Reopen(options);
+
+  // block flush thread -> pin immtables in memory
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->LoadDependency({
+      {"SnapshotPreventsDroppedKeysInImmMemTables:AfterNewIterator",
+       "DBImpl::BGWorkFlush"},
+  });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(Put(Key(0), "a"));
+  std::unique_ptr<const Snapshot, std::function<void(const Snapshot*)>>
+      snapshot(db_->GetSnapshot(),
+               [this](const Snapshot* s) { db_->ReleaseSnapshot(s); });
+
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
+                             Key(10)));
+
+  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
+
+  ReadOptions read_opts;
+  read_opts.snapshot = snapshot.get();
+  std::unique_ptr<Iterator> iter(db_->NewIterator(read_opts));
+
+  TEST_SYNC_POINT("SnapshotPreventsDroppedKeysInImmMemTables:AfterNewIterator");
+
+  iter->SeekToFirst();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(Key(0), iter->key());
+
+  iter->Next();
+  ASSERT_FALSE(iter->Valid());
+}
+
 TEST_F(DBRangeDelTest, RangeTombstoneWrittenToMinimalSsts) {
  // Adapted from
  // https://github.com/cockroachdb/cockroach/blob/de8b3ea603dd1592d9dc26443c2cc92c356fbc2f/pkg/storage/engine/rocksdb_test.go#L1267-L1398.
@@ -1705,6 +1705,7 @@ TEST_F(DBTest, Snapshot) {
    ASSERT_EQ(1U, GetNumSnapshots());
    uint64_t time_snap1 = GetTimeOldestSnapshots();
    ASSERT_GT(time_snap1, 0U);
+    ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber());
    Put(0, "foo", "0v2");
    Put(1, "foo", "1v2");

@@ -1713,6 +1714,7 @@ TEST_F(DBTest, Snapshot) {
    const Snapshot* s2 = db_->GetSnapshot();
    ASSERT_EQ(2U, GetNumSnapshots());
    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
+    ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber());
    Put(0, "foo", "0v3");
    Put(1, "foo", "1v3");

@@ -1720,6 +1722,7 @@ TEST_F(DBTest, Snapshot) {
      ManagedSnapshot s3(db_);
      ASSERT_EQ(3U, GetNumSnapshots());
      ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
+      ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber());

      Put(0, "foo", "0v4");
      Put(1, "foo", "1v4");
@@ -1735,6 +1738,7 @@ TEST_F(DBTest, Snapshot) {

    ASSERT_EQ(2U, GetNumSnapshots());
    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
+    ASSERT_EQ(GetSequenceOldestSnapshots(), s1->GetSequenceNumber());
    ASSERT_EQ("0v1", Get(0, "foo", s1));
    ASSERT_EQ("1v1", Get(1, "foo", s1));
    ASSERT_EQ("0v2", Get(0, "foo", s2));
@@ -1749,9 +1753,11 @@ TEST_F(DBTest, Snapshot) {
    ASSERT_EQ("1v4", Get(1, "foo"));
    ASSERT_EQ(1U, GetNumSnapshots());
    ASSERT_LT(time_snap1, GetTimeOldestSnapshots());
+    ASSERT_EQ(GetSequenceOldestSnapshots(), s2->GetSequenceNumber());

    db_->ReleaseSnapshot(s2);
    ASSERT_EQ(0U, GetNumSnapshots());
+    ASSERT_EQ(GetSequenceOldestSnapshots(), 0);
    ASSERT_EQ("0v4", Get(0, "foo"));
    ASSERT_EQ("1v4", Get(1, "foo"));
  } while (ChangeOptions());
@@ -153,6 +153,8 @@ TEST_F(DBTest2, MaxSuccessiveMergesChangeWithDBRecovery) {
  options.create_if_missing = true;
  options.statistics = rocksdb::CreateDBStatistics();
  options.max_successive_merges = 3;
+  // allow_concurrent_memtable_write is incompatible with max_successive_merges
+  options.allow_concurrent_memtable_write = false;
  options.merge_operator = MergeOperators::CreatePutOperator();
  options.disable_auto_compactions = true;
  DestroyAndReopen(options);
@@ -4210,6 +4212,62 @@ TEST_F(DBTest2, SeekFileRangeDeleteTail) {
  }
  db_->ReleaseSnapshot(s1);
 }
+
+TEST_F(DBTest2, BackgroundPurgeTest) {
+  Options options = CurrentOptions();
+  options.write_buffer_manager = std::make_shared<rocksdb::WriteBufferManager>(1 << 20);
+  options.avoid_unnecessary_blocking_io = true;
+  DestroyAndReopen(options);
+  size_t base_value = options.write_buffer_manager->memory_usage();
+
+  ASSERT_OK(Put("a", "a"));
+  Iterator* iter = db_->NewIterator(ReadOptions());
+  ASSERT_OK(Flush());
+  size_t value = options.write_buffer_manager->memory_usage();
+  ASSERT_GT(value, base_value);
+
+  db_->GetEnv()->SetBackgroundThreads(1, Env::Priority::HIGH);
+  test::SleepingBackgroundTask sleeping_task_after;
+  db_->GetEnv()->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                          &sleeping_task_after, Env::Priority::HIGH);
+  delete iter;
+
+  Env::Default()->SleepForMicroseconds(100000);
+  value = options.write_buffer_manager->memory_usage();
+  ASSERT_GT(value, base_value);
+
+  sleeping_task_after.WakeUp();
+  sleeping_task_after.WaitUntilDone();
+
+  test::SleepingBackgroundTask sleeping_task_after2;
+  db_->GetEnv()->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                          &sleeping_task_after2, Env::Priority::HIGH);
+  sleeping_task_after2.WakeUp();
+  sleeping_task_after2.WaitUntilDone();
+
+  value = options.write_buffer_manager->memory_usage();
+  ASSERT_EQ(base_value, value);
+}
+
+TEST_F(DBTest2, SwitchMemtableRaceWithNewManifest) {
+  Options options = CurrentOptions();
+  DestroyAndReopen(options);
+  options.max_manifest_file_size = 10;
+  options.create_if_missing = true;
+  CreateAndReopenWithCF({"pikachu"}, options);
+  ASSERT_EQ(2, handles_.size());
+
+  ASSERT_OK(Put("foo", "value"));
+  const int kL0Files = options.level0_file_num_compaction_trigger;
+  for (int i = 0; i < kL0Files; ++i) {
+    ASSERT_OK(Put(/*cf=*/1, "a", std::to_string(i)));
+    ASSERT_OK(Flush(/*cf=*/1));
+  }
+
+  port::Thread thread([&]() { ASSERT_OK(Flush()); });
+  ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  thread.join();
+}
 }  // namespace rocksdb

 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
@@ -53,9 +53,9 @@ DBTestBase::DBTestBase(const std::string path)
 #ifndef ROCKSDB_LITE
  const char* test_env_uri = getenv("TEST_ENV_URI");
  if (test_env_uri) {
-    Status s = ObjectRegistry::NewInstance()->NewSharedObject(test_env_uri,
-                                                              &env_guard_);
-    base_env = env_guard_.get();
+    Env* test_env = nullptr;
+    Status s = Env::LoadEnv(test_env_uri, &test_env, &env_guard_);
+    base_env = test_env;
    EXPECT_OK(s);
    EXPECT_NE(Env::Default(), base_env);
  }
@@ -867,6 +867,13 @@ uint64_t DBTestBase::GetTimeOldestSnapshots() {
  return int_num;
 }

+uint64_t DBTestBase::GetSequenceOldestSnapshots() {
+  uint64_t int_num;
+  EXPECT_TRUE(
+      dbfull()->GetIntProperty("rocksdb.oldest-snapshot-sequence", &int_num));
+  return int_num;
+}
+
 // Return a string that contains all key,value pairs in order,
 // formatted like "(k1->v1)(k2->v2)".
 std::string DBTestBase::Contents(int cf) {
@@ -860,6 +860,8 @@ class DBTestBase : public testing::Test {

  uint64_t GetTimeOldestSnapshots();

+  uint64_t GetSequenceOldestSnapshots();
+
  // Return a string that contains all key,value pairs in order,
  // formatted like "(k1->v1)(k2->v2)".
  std::string Contents(int cf = 0);
@@ -8,6 +8,7 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

 #include "db/db_test_util.h"
+#include "env/composite_env_wrapper.h"
 #include "options/options_helper.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
@@ -902,8 +903,8 @@ class RecoveryTestHelper {
      std::string fname = LogFileName(test->dbname_, current_log_number);
      std::unique_ptr<WritableFile> file;
      ASSERT_OK(db_options.env->NewWritableFile(fname, &file, env_options));
-      std::unique_ptr<WritableFileWriter> file_writer(
-          new WritableFileWriter(std::move(file), fname, env_options));
+      std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
+          NewLegacyWritableFileWrapper(std::move(file)), fname, env_options));
      current_log_writer.reset(
          new log::Writer(std::move(file_writer), current_log_number,
                          db_options.recycle_log_file_num > 0));
@@ -183,7 +183,8 @@ TEST_F(DBErrorHandlingTest, CompactionWriteError) {
      );
  listener->EnableAutoRecovery(false);
  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"FlushMemTableFinished", "BackgroundCallCompaction:0"}});
+      {{"DBImpl::FlushMemTable:FlushMemTableFinished",
+        "BackgroundCallCompaction:0"}});
  rocksdb::SyncPoint::GetInstance()->SetCallBack(
      "BackgroundCallCompaction:0", [&](void *) {
      fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
@@ -219,7 +220,8 @@ TEST_F(DBErrorHandlingTest, CorruptionError) {
  ASSERT_EQ(s, Status::OK());

  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"FlushMemTableFinished", "BackgroundCallCompaction:0"}});
+      {{"DBImpl::FlushMemTable:FlushMemTableFinished",
+        "BackgroundCallCompaction:0"}});
  rocksdb::SyncPoint::GetInstance()->SetCallBack(
      "BackgroundCallCompaction:0", [&](void *) {
      fault_env->SetFilesystemActive(false, Status::Corruption("Corruption"));
@@ -101,14 +101,15 @@ Status ExternalSstFileIngestionJob::Prepare(
        TableFileName(cfd_->ioptions()->cf_paths, f.fd.GetNumber(),
                      f.fd.GetPathId());
    if (ingestion_options_.move_files) {
-      status = env_->LinkFile(path_outside_db, path_inside_db);
+      status =
+          fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr);
      if (status.ok()) {
        // It is unsafe to assume application had sync the file and file
        // directory before ingest the file. For integrity of RocksDB we need
        // to sync the file.
-        std::unique_ptr<WritableFile> file_to_sync;
-        status = env_->ReopenWritableFile(path_inside_db, &file_to_sync,
-                                          env_options_);
+        std::unique_ptr<FSWritableFile> file_to_sync;
+        status = fs_->ReopenWritableFile(path_inside_db, env_options_,
+                                         &file_to_sync, nullptr);
        if (status.ok()) {
          TEST_SYNC_POINT(
              "ExternalSstFileIngestionJob::BeforeSyncIngestedFile");
@@ -133,7 +134,7 @@ Status ExternalSstFileIngestionJob::Prepare(
      TEST_SYNC_POINT_CALLBACK("ExternalSstFileIngestionJob::Prepare:CopyFile",
                               nullptr);
      // CopyFile also sync the new file.
-      status = CopyFile(env_, path_outside_db, path_inside_db, 0,
+      status = CopyFile(fs_, path_outside_db, path_inside_db, 0,
                        db_options_.use_fsync);
    }
    TEST_SYNC_POINT("ExternalSstFileIngestionJob::Prepare:FileAdded");
@@ -360,17 +361,19 @@ Status ExternalSstFileIngestionJob::GetIngestedFileInfo(
  file_to_ingest->external_file_path = external_file;

  // Get external file size
-  Status status = env_->GetFileSize(external_file, &file_to_ingest->file_size);
+  Status status = fs_->GetFileSize(external_file, IOOptions(),
+                                   &file_to_ingest->file_size, nullptr);
  if (!status.ok()) {
    return status;
  }

  // Create TableReader for external file
  std::unique_ptr<TableReader> table_reader;
-  std::unique_ptr<RandomAccessFile> sst_file;
+  std::unique_ptr<FSRandomAccessFile> sst_file;
  std::unique_ptr<RandomAccessFileReader> sst_file_reader;

-  status = env_->NewRandomAccessFile(external_file, &sst_file, env_options_);
+  status = fs_->NewRandomAccessFile(external_file, env_options_,
+                                    &sst_file, nullptr);
  if (!status.ok()) {
    return status;
  }
@@ -651,13 +654,15 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
    // Determine if we can write global_seqno to a given offset of file.
    // If the file system does not support random write, then we should not.
    // Otherwise we should.
-    std::unique_ptr<RandomRWFile> rwfile;
-    Status status = env_->NewRandomRWFile(file_to_ingest->internal_file_path,
-                                          &rwfile, env_options_);
+    std::unique_ptr<FSRandomRWFile> rwfile;
+    Status status =
+        fs_->NewRandomRWFile(file_to_ingest->internal_file_path, env_options_,
+                             &rwfile, nullptr);
    if (status.ok()) {
      std::string seqno_val;
      PutFixed64(&seqno_val, seqno);
-      status = rwfile->Write(file_to_ingest->global_seqno_offset, seqno_val);
+      status = rwfile->Write(file_to_ingest->global_seqno_offset, seqno_val,
+                             IOOptions(), nullptr);
      if (status.ok()) {
        TEST_SYNC_POINT("ExternalSstFileIngestionJob::BeforeSyncGlobalSeqno");
        status = SyncIngestedFile(rwfile.get());
@@ -715,9 +720,9 @@ template <typename TWritableFile>
 Status ExternalSstFileIngestionJob::SyncIngestedFile(TWritableFile* file) {
  assert(file != nullptr);
  if (db_options_.use_fsync) {
-    return file->Fsync();
+    return file->Fsync(IOOptions(), nullptr);
  } else {
-    return file->Sync();
+    return file->Sync(IOOptions(), nullptr);
  }
 }

@@ -74,6 +74,7 @@ class ExternalSstFileIngestionJob {
      const IngestExternalFileOptions& ingestion_options,
      Directories* directories, EventLogger* event_logger)
      : env_(env),
+        fs_(db_options.fs.get()),
        versions_(versions),
        cfd_(cfd),
        db_options_(db_options),
@@ -158,6 +159,7 @@ class ExternalSstFileIngestionJob {
  Status SyncIngestedFile(TWritableFile* file);

  Env* env_;
+  FileSystem* fs_;
  VersionSet* versions_;
  ColumnFamilyData* cfd_;
  const ImmutableDBOptions& db_options_;
@@ -1700,6 +1700,35 @@ TEST_F(ExternalSSTFileTest, SstFileWriterNonSharedKeys) {
  ASSERT_OK(DeprecatedAddFile({file_path}));
 }

+TEST_F(ExternalSSTFileTest, WithUnorderedWrite) {
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::WriteImpl:UnorderedWriteAfterWriteWAL",
+        "ExternalSSTFileTest::WithUnorderedWrite:WaitWriteWAL"},
+       {"DBImpl::WaitForPendingWrites:BeforeBlock",
+        "DBImpl::WriteImpl:BeforeUnorderedWriteMemtable"}});
+  SyncPoint::GetInstance()->SetCallBack(
+      "DBImpl::IngestExternalFile:NeedFlush", [&](void* need_flush) {
+        ASSERT_TRUE(*reinterpret_cast<bool*>(need_flush));
+      });
+
+  Options options = CurrentOptions();
+  options.unordered_write = true;
+  DestroyAndReopen(options);
+  Put("foo", "v1");
+  SyncPoint::GetInstance()->EnableProcessing();
+  port::Thread writer([&]() { Put("bar", "v2"); });
+
+  TEST_SYNC_POINT("ExternalSSTFileTest::WithUnorderedWrite:WaitWriteWAL");
+  ASSERT_OK(GenerateAndAddExternalFile(options, {{"bar", "v3"}}, -1,
+                                       true /* allow_global_seqno */));
+  ASSERT_EQ(Get("bar"), "v3");
+
+  writer.join();
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
 TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoRandomized) {
  Options options = CurrentOptions();
  options.IncreaseParallelism(20);
@@ -86,7 +86,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
                   const ImmutableDBOptions& db_options,
                   const MutableCFOptions& mutable_cf_options,
                   const uint64_t* max_memtable_id,
-                   const EnvOptions& env_options, VersionSet* versions,
+                   const FileOptions& file_options, VersionSet* versions,
                   InstrumentedMutex* db_mutex,
                   std::atomic<bool>* shutting_down,
                   std::vector<SequenceNumber> existing_snapshots,
@@ -103,7 +103,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
      db_options_(db_options),
      mutable_cf_options_(mutable_cf_options),
      max_memtable_id_(max_memtable_id),
-      env_options_(env_options),
+      file_options_(file_options),
      versions_(versions),
      db_mutex_(db_mutex),
      shutting_down_(shutting_down),
@@ -249,7 +249,8 @@ Status FlushJob::Run(LogsWithPrepTracker* prep_tracker,
  }
  RecordFlushIOStats();

-  auto stream = event_logger_->LogToBuffer(log_buffer_);
+  // When measure_io_stats_ is true, the default 512 bytes is not enough.
+  auto stream = event_logger_->LogToBuffer(log_buffer_, 1024);
  stream << "job" << job_context_->job_id << "event"
         << "flush_finished";
  stream << "output_compression"
@@ -371,8 +372,8 @@ Status FlushJob::WriteLevel0Table() {
      meta_.file_creation_time = current_time;

      s = BuildTable(
-          dbname_, db_options_.env, *cfd_->ioptions(), mutable_cf_options_,
-          env_options_, cfd_->table_cache(), iter.get(),
+          dbname_, db_options_.env, db_options_.fs.get(), *cfd_->ioptions(),
+          mutable_cf_options_, file_options_, cfd_->table_cache(), iter.get(),
          std::move(range_del_iters), &meta_, cfd_->internal_comparator(),
          cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
          cfd_->GetName(), existing_snapshots_,
@@ -61,7 +61,7 @@ class FlushJob {
  FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
           const ImmutableDBOptions& db_options,
           const MutableCFOptions& mutable_cf_options,
-           const uint64_t* max_memtable_id, const EnvOptions& env_options,
+           const uint64_t* max_memtable_id, const FileOptions& file_options,
           VersionSet* versions, InstrumentedMutex* db_mutex,
           std::atomic<bool>* shutting_down,
           std::vector<SequenceNumber> existing_snapshots,
@@ -108,7 +108,7 @@ class FlushJob {
  // equal to *max_memtable_id_ will be selected for flush. If null, then all
  // memtables in the column family will be selected.
  const uint64_t* max_memtable_id_;
-  const EnvOptions env_options_;
+  const FileOptions file_options_;
  VersionSet* versions_;
  InstrumentedMutex* db_mutex_;
  std::atomic<bool>* shutting_down_;
@@ -30,16 +30,13 @@ class FlushJobTest : public testing::Test {
 public:
  FlushJobTest()
      : env_(Env::Default()),
+        fs_(std::make_shared<LegacyFileSystemWrapper>(env_)),
        dbname_(test::PerThreadDBPath("flush_job_test")),
        options_(),
        db_options_(options_),
        column_family_names_({kDefaultColumnFamilyName, "foo", "bar"}),
        table_cache_(NewLRUCache(50000, 16)),
        write_buffer_manager_(db_options_.db_write_buffer_size),
-        versions_(new VersionSet(dbname_, &db_options_, env_options_,
-                                 table_cache_.get(), &write_buffer_manager_,
-                                 &write_controller_,
-                                 /*block_cache_tracer=*/nullptr)),
        shutting_down_(false),
        mock_table_factory_(new mock::MockTableFactory()) {
    EXPECT_OK(env_->CreateDirIfMissing(dbname_));
@@ -54,6 +51,12 @@ class FlushJobTest : public testing::Test {
      column_families.emplace_back(cf_name, cf_options_);
    }

+    db_options_.env = env_;
+    db_options_.fs = fs_;
+    versions_.reset(new VersionSet(dbname_, &db_options_, env_options_,
+                                   table_cache_.get(), &write_buffer_manager_,
+                                   &write_controller_,
+                                   /*block_cache_tracer=*/nullptr));
    EXPECT_OK(versions_->Recover(column_families, false));
  }

@@ -88,8 +91,8 @@ class FlushJobTest : public testing::Test {
    Status s = env_->NewWritableFile(
        manifest, &file, env_->OptimizeForManifestWrite(env_options_));
    ASSERT_OK(s);
-    std::unique_ptr<WritableFileWriter> file_writer(
-        new WritableFileWriter(std::move(file), manifest, EnvOptions()));
+    std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
+        NewLegacyWritableFileWrapper(std::move(file)), manifest, EnvOptions()));
    {
      log::Writer log(std::move(file_writer), 0, false);
      std::string record;
@@ -109,6 +112,7 @@ class FlushJobTest : public testing::Test {
  }

  Env* env_;
+  std::shared_ptr<FileSystem> fs_;
  std::string dbname_;
  EnvOptions env_options_;
  Options options_;
@@ -60,9 +60,7 @@ ColumnFamilyData* FlushScheduler::TakeNextColumnFamily() {
    }

    // no longer relevant, retry
-    if (cfd->Unref()) {
-      delete cfd;
-    }
+    cfd->UnrefAndTryDelete();
  }
 }

@@ -80,9 +78,7 @@ bool FlushScheduler::Empty() {
 void FlushScheduler::Clear() {
  ColumnFamilyData* cfd;
  while ((cfd = TakeNextColumnFamily()) != nullptr) {
-    if (cfd->Unref()) {
-      delete cfd;
-    }
+    cfd->UnrefAndTryDelete();
  }
  assert(head_.load(std::memory_order_relaxed) == nullptr);
 }
@@ -239,9 +239,13 @@ void ForwardIterator::SVCleanup(DBImpl* db, SuperVersion* sv,
    db->FindObsoleteFiles(&job_context, false, true);
    if (background_purge_on_iterator_cleanup) {
      db->ScheduleBgLogWriterClose(&job_context);
+      db->AddSuperVersionsToFreeQueue(sv);
+      db->SchedulePurge();
    }
    db->mutex_.Unlock();
-    delete sv;
+    if (!background_purge_on_iterator_cleanup) {
+      delete sv;
+    }
    if (job_context.HaveSomethingToDelete()) {
      db->PurgeObsoleteFiles(job_context, background_purge_on_iterator_cleanup);
    }
@@ -614,7 +618,7 @@ void ForwardIterator::RebuildIterators(bool refresh_sv) {
  Cleanup(refresh_sv);
  if (refresh_sv) {
    // New
-    sv_ = cfd_->GetReferencedSuperVersion(&(db_->mutex_));
+    sv_ = cfd_->GetReferencedSuperVersion(db_);
  }
  ReadRangeDelAggregator range_del_agg(&cfd_->internal_comparator(),
                                       kMaxSequenceNumber /* upper_bound */);
@@ -668,7 +672,7 @@ void ForwardIterator::RebuildIterators(bool refresh_sv) {
 void ForwardIterator::RenewIterators() {
  SuperVersion* svnew;
  assert(sv_);
-  svnew = cfd_->GetReferencedSuperVersion(&(db_->mutex_));
+  svnew = cfd_->GetReferencedSuperVersion(db_);

  if (mutable_iter_ != nullptr) {
    DeleteIterator(mutable_iter_, true /* is_arena */);
@@ -92,14 +92,15 @@ Status ImportColumnFamilyJob::Prepare(uint64_t next_file_number,
        cfd_->ioptions()->cf_paths, f.fd.GetNumber(), f.fd.GetPathId());

    if (hardlink_files) {
-      status = env_->LinkFile(path_outside_db, path_inside_db);
+      status =
+          fs_->LinkFile(path_outside_db, path_inside_db, IOOptions(), nullptr);
      if (status.IsNotSupported()) {
        // Original file is on a different FS, use copy instead of hard linking
        hardlink_files = false;
      }
    }
    if (!hardlink_files) {
-      status = CopyFile(env_, path_outside_db, path_inside_db, 0,
+      status = CopyFile(fs_, path_outside_db, path_inside_db, 0,
                        db_options_.use_fsync);
    }
    if (!status.ok()) {
@@ -115,7 +116,8 @@ Status ImportColumnFamilyJob::Prepare(uint64_t next_file_number,
      if (f.internal_file_path.empty()) {
        break;
      }
-      const auto s = env_->DeleteFile(f.internal_file_path);
+      const auto s =
+          fs_->DeleteFile(f.internal_file_path, IOOptions(), nullptr);
      if (!s.ok()) {
        ROCKS_LOG_WARN(db_options_.info_log,
                       "AddFile() clean up for file %s failed : %s",
@@ -168,7 +170,8 @@ void ImportColumnFamilyJob::Cleanup(const Status& status) {
  if (!status.ok()) {
    // We failed to add files to the database remove all the files we copied.
    for (const auto& f : files_to_import_) {
-      const auto s = env_->DeleteFile(f.internal_file_path);
+      const auto s =
+          fs_->DeleteFile(f.internal_file_path, IOOptions(), nullptr);
      if (!s.ok()) {
        ROCKS_LOG_WARN(db_options_.info_log,
                       "AddFile() clean up for file %s failed : %s",
@@ -178,7 +181,8 @@ void ImportColumnFamilyJob::Cleanup(const Status& status) {
  } else if (status.ok() && import_options_.move_files) {
    // The files were moved and added successfully, remove original file links
    for (IngestedFileInfo& f : files_to_import_) {
-      const auto s = env_->DeleteFile(f.external_file_path);
+      const auto s =
+          fs_->DeleteFile(f.external_file_path, IOOptions(), nullptr);
      if (!s.ok()) {
        ROCKS_LOG_WARN(
            db_options_.info_log,
@@ -196,17 +200,19 @@ Status ImportColumnFamilyJob::GetIngestedFileInfo(
  file_to_import->external_file_path = external_file;

  // Get external file size
-  auto status = env_->GetFileSize(external_file, &file_to_import->file_size);
+  Status status = fs_->GetFileSize(external_file, IOOptions(),
+                                   &file_to_import->file_size, nullptr);
  if (!status.ok()) {
    return status;
  }

  // Create TableReader for external file
  std::unique_ptr<TableReader> table_reader;
-  std::unique_ptr<RandomAccessFile> sst_file;
+  std::unique_ptr<FSRandomAccessFile> sst_file;
  std::unique_ptr<RandomAccessFileReader> sst_file_reader;

-  status = env_->NewRandomAccessFile(external_file, &sst_file, env_options_);
+  status = fs_->NewRandomAccessFile(external_file, env_options_,
+                                    &sst_file, nullptr);
  if (!status.ok()) {
    return status;
  }
@@ -29,6 +29,7 @@ class ImportColumnFamilyJob {
        versions_(versions),
        cfd_(cfd),
        db_options_(db_options),
+        fs_(db_options_.fs.get()),
        env_options_(env_options),
        import_options_(import_options),
        metadata_(metadata) {}
@@ -60,6 +61,7 @@ class ImportColumnFamilyJob {
  VersionSet* versions_;
  ColumnFamilyData* cfd_;
  const ImmutableDBOptions& db_options_;
+  FileSystem* fs_;
  const EnvOptions& env_options_;
  autovector<IngestedFileInfo> files_to_import_;
  VersionEdit edit_;
@@ -231,6 +231,7 @@ static const std::string is_file_deletions_enabled =
    "is-file-deletions-enabled";
 static const std::string num_snapshots = "num-snapshots";
 static const std::string oldest_snapshot_time = "oldest-snapshot-time";
+static const std::string oldest_snapshot_sequence = "oldest-snapshot-sequence";
 static const std::string num_live_versions = "num-live-versions";
 static const std::string current_version_number =
    "current-super-version-number";
@@ -309,6 +310,8 @@ const std::string DB::Properties::kNumSnapshots =
    rocksdb_prefix + num_snapshots;
 const std::string DB::Properties::kOldestSnapshotTime =
    rocksdb_prefix + oldest_snapshot_time;
+const std::string DB::Properties::kOldestSnapshotSequence =
+    rocksdb_prefix + oldest_snapshot_sequence;
 const std::string DB::Properties::kNumLiveVersions =
    rocksdb_prefix + num_live_versions;
 const std::string DB::Properties::kCurrentSuperVersionNumber =
@@ -427,6 +430,9 @@ const std::unordered_map<std::string, DBPropertyInfo>
        {DB::Properties::kOldestSnapshotTime,
         {false, nullptr, &InternalStats::HandleOldestSnapshotTime, nullptr,
          nullptr}},
+        {DB::Properties::kOldestSnapshotSequence,
+         {false, nullptr, &InternalStats::HandleOldestSnapshotSequence, nullptr,
+          nullptr}},
        {DB::Properties::kNumLiveVersions,
         {false, nullptr, &InternalStats::HandleNumLiveVersions, nullptr,
          nullptr}},
@@ -772,6 +778,12 @@ bool InternalStats::HandleOldestSnapshotTime(uint64_t* value, DBImpl* db,
  return true;
 }

+bool InternalStats::HandleOldestSnapshotSequence(uint64_t* value, DBImpl* db,
+                                                 Version* /*version*/) {
+  *value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotSequence());
+  return true;
+}
+
 bool InternalStats::HandleNumLiveVersions(uint64_t* value, DBImpl* /*db*/,
                                          Version* /*version*/) {
  *value = cfd_->GetNumLiveVersions();
@@ -553,6 +553,8 @@ class InternalStats {
  bool HandleEstimateNumKeys(uint64_t* value, DBImpl* db, Version* version);
  bool HandleNumSnapshots(uint64_t* value, DBImpl* db, Version* version);
  bool HandleOldestSnapshotTime(uint64_t* value, DBImpl* db, Version* version);
+  bool HandleOldestSnapshotSequence(uint64_t* value, DBImpl* db,
+                                    Version* version);
  bool HandleNumLiveVersions(uint64_t* value, DBImpl* db, Version* version);
  bool HandleCurrentSuperVersionNumber(uint64_t* value, DBImpl* db,
                                       Version* version);
@@ -9,6 +9,7 @@

 #include "db/log_reader.h"
 #include "db/log_writer.h"
+#include "env/composite_env_wrapper.h"
 #include "file/sequence_file_reader.h"
 #include "file/writable_file_writer.h"
 #include "rocksdb/env.h"
@@ -120,6 +121,13 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
    }
  };

+  inline StringSource* GetStringSourceFromLegacyReader(
+      SequentialFileReader* reader) {
+    LegacySequentialFileWrapper* file =
+        static_cast<LegacySequentialFileWrapper*>(reader->file());
+    return static_cast<StringSource*>(file->target());
+  }
+
  class ReportCollector : public Reader::Reporter {
   public:
    size_t dropped_bytes_;
@@ -133,21 +141,19 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
  };

  std::string& dest_contents() {
-    auto dest =
-      dynamic_cast<test::StringSink*>(writer_.file()->writable_file());
+    auto dest = test::GetStringSinkFromLegacyWriter(writer_.file());
    assert(dest);
    return dest->contents_;
  }

  const std::string& dest_contents() const {
-    auto dest =
-      dynamic_cast<const test::StringSink*>(writer_.file()->writable_file());
+    auto dest = test::GetStringSinkFromLegacyWriter(writer_.file());
    assert(dest);
    return dest->contents_;
  }

  void reset_source_contents() {
-    auto src = dynamic_cast<StringSource*>(reader_->file()->file());
+    auto src = GetStringSourceFromLegacyReader(reader_->file());
    assert(src);
    src->contents_ = dest_contents();
  }
@@ -214,8 +220,7 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
  }

  void ShrinkSize(int bytes) {
-    auto dest =
-      dynamic_cast<test::StringSink*>(writer_.file()->writable_file());
+    auto dest = test::GetStringSinkFromLegacyWriter(writer_.file());
    assert(dest);
    dest->Drop(bytes);
  }
@@ -230,7 +235,7 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
  }

  void ForceError(size_t position = 0) {
-    auto src = dynamic_cast<StringSource*>(reader_->file()->file());
+    auto src = GetStringSourceFromLegacyReader(reader_->file());
    src->force_error_ = true;
    src->force_error_position_ = position;
  }
@@ -244,13 +249,13 @@ class LogTest : public ::testing::TestWithParam<std::tuple<int, bool>> {
  }

  void ForceEOF(size_t position = 0) {
-    auto src = dynamic_cast<StringSource*>(reader_->file()->file());
+    auto src = GetStringSourceFromLegacyReader(reader_->file());
    src->force_eof_ = true;
    src->force_eof_position_ = position;
  }

  void UnmarkEOF() {
-    auto src = dynamic_cast<StringSource*>(reader_->file()->file());
+    auto src = GetStringSourceFromLegacyReader(reader_->file());
    src->returned_partial_ = false;
    reader_->UnmarkEOF();
  }
@@ -752,8 +757,9 @@ class RetriableLogTest : public ::testing::TestWithParam<int> {
      s = env_->NewWritableFile(log_file_, &writable_file, env_options_);
    }
    if (s.ok()) {
-      writer_.reset(new WritableFileWriter(std::move(writable_file), log_file_,
-                                           env_options_));
+      writer_.reset(new WritableFileWriter(
+          NewLegacyWritableFileWrapper(std::move(writable_file)), log_file_,
+          env_options_));
      assert(writer_ != nullptr);
    }
    std::unique_ptr<SequentialFile> seq_file;
@@ -761,7 +767,8 @@ class RetriableLogTest : public ::testing::TestWithParam<int> {
      s = env_->NewSequentialFile(log_file_, &seq_file, env_options_);
    }
    if (s.ok()) {
-      reader_.reset(new SequentialFileReader(std::move(seq_file), log_file_));
+      reader_.reset(new SequentialFileReader(
+          NewLegacySequentialFileWrapper(seq_file), log_file_));
      assert(reader_ != nullptr);
      log_reader_.reset(new FragmentBufferedReader(
          nullptr, std::move(reader_), &report_, true /* checksum */,
@@ -772,8 +779,7 @@ class RetriableLogTest : public ::testing::TestWithParam<int> {
  }

  std::string contents() {
-    auto file =
-        dynamic_cast<test::StringSink*>(log_writer_->file()->writable_file());
+    auto file = test::GetStringSinkFromLegacyWriter(log_writer_->file());
    assert(file != nullptr);
    return file->contents_;
  }
@@ -138,7 +138,7 @@ class MemTable {

  // As a cheap version of `ApproximateMemoryUsage()`, this function doens't
  // require external synchronization. The value may be less accurate though
-  size_t ApproximateMemoryUsageFast() {
+  size_t ApproximateMemoryUsageFast() const {
    return approximate_memory_usage_.load(std::memory_order_relaxed);
  }

@@ -186,11 +186,14 @@ Status MemTableListVersion::AddRangeTombstoneIterators(
    const ReadOptions& read_opts, Arena* /*arena*/,
    RangeDelAggregator* range_del_agg) {
  assert(range_del_agg != nullptr);
+  // Except for snapshot read, using kMaxSequenceNumber is OK because these
+  // are immutable memtables.
+  SequenceNumber read_seq = read_opts.snapshot != nullptr
+                                ? read_opts.snapshot->GetSequenceNumber()
+                                : kMaxSequenceNumber;
  for (auto& m : memlist_) {
-    // Using kMaxSequenceNumber is OK because these are immutable memtables.
    std::unique_ptr<FragmentedRangeTombstoneIterator> range_del_iter(
-        m->NewRangeTombstoneIterator(read_opts,
-                                     kMaxSequenceNumber /* read_seq */));
+        m->NewRangeTombstoneIterator(read_opts, read_seq));
    range_del_agg->AddTombstones(std::move(range_del_iter));
  }
  return Status::OK();
@@ -277,7 +280,7 @@ void MemTableListVersion::Remove(MemTable* m,
 }

 // return the total memory usage assuming the oldest flushed memtable is dropped
-size_t MemTableListVersion::ApproximateMemoryUsageExcludingLast() {
+size_t MemTableListVersion::ApproximateMemoryUsageExcludingLast() const {
  size_t total_memtable_size = 0;
  for (auto& memtable : memlist_) {
    total_memtable_size += memtable->ApproximateMemoryUsage();
@@ -500,7 +503,7 @@ Status MemTableList::TryInstallMemtableFlushResults(
                           cfd->GetName().c_str(), m->file_number_, mem_id);
          assert(m->file_number_ > 0);
          current_->Remove(m, to_delete);
-          UpdateMemoryUsageExcludingLast();
+          UpdateCachedValuesFromMemTableListVersion();
          ResetTrimHistoryNeeded();
          ++mem_id;
        }
@@ -541,14 +544,14 @@ void MemTableList::Add(MemTable* m, autovector<MemTable*>* to_delete) {
  if (num_flush_not_started_ == 1) {
    imm_flush_needed.store(true, std::memory_order_release);
  }
-  UpdateMemoryUsageExcludingLast();
+  UpdateCachedValuesFromMemTableListVersion();
  ResetTrimHistoryNeeded();
 }

 void MemTableList::TrimHistory(autovector<MemTable*>* to_delete, size_t usage) {
  InstallNewVersion();
  current_->TrimHistory(to_delete, usage);
-  UpdateMemoryUsageExcludingLast();
+  UpdateCachedValuesFromMemTableListVersion();
  ResetTrimHistoryNeeded();
 }

@@ -563,18 +566,25 @@ size_t MemTableList::ApproximateUnflushedMemTablesMemoryUsage() {

 size_t MemTableList::ApproximateMemoryUsage() { return current_memory_usage_; }

-size_t MemTableList::ApproximateMemoryUsageExcludingLast() {
-  size_t usage =
+size_t MemTableList::ApproximateMemoryUsageExcludingLast() const {
+  const size_t usage =
      current_memory_usage_excluding_last_.load(std::memory_order_relaxed);
  return usage;
 }

-// Update current_memory_usage_excluding_last_, need to call whenever state
-// changes for MemtableListVersion (whenever InstallNewVersion() is called)
-void MemTableList::UpdateMemoryUsageExcludingLast() {
-  size_t total_memtable_size = current_->ApproximateMemoryUsageExcludingLast();
+bool MemTableList::HasHistory() const {
+  const bool has_history = current_has_history_.load(std::memory_order_relaxed);
+  return has_history;
+}
+
+void MemTableList::UpdateCachedValuesFromMemTableListVersion() {
+  const size_t total_memtable_size =
+      current_->ApproximateMemoryUsageExcludingLast();
  current_memory_usage_excluding_last_.store(total_memtable_size,
                                             std::memory_order_relaxed);
+
+  const bool has_history = current_->HasHistory();
+  current_has_history_.store(has_history, std::memory_order_relaxed);
 }

 uint64_t MemTableList::ApproximateOldestKeyTime() const {
@@ -704,7 +714,7 @@ Status InstallMemtableAtomicFlushResults(
                         cfds[i]->GetName().c_str(), m->GetFileNumber(),
                         mem_id);
        imm->current_->Remove(m, to_delete);
-        imm->UpdateMemoryUsageExcludingLast();
+        imm->UpdateCachedValuesFromMemTableListVersion();
        imm->ResetTrimHistoryNeeded();
      }
    }
@@ -754,7 +764,7 @@ void MemTableList::RemoveOldMemTables(uint64_t log_number,
    }
  }

-  UpdateMemoryUsageExcludingLast();
+  UpdateCachedValuesFromMemTableListVersion();
  ResetTrimHistoryNeeded();
 }

@@ -162,7 +162,11 @@ class MemTableListVersion {
  // excluding the last MemTable in memlist_history_. The reason for excluding
  // the last MemTable is to see if dropping the last MemTable will keep total
  // memory usage above or equal to max_write_buffer_size_to_maintain_
-  size_t ApproximateMemoryUsageExcludingLast();
+  size_t ApproximateMemoryUsageExcludingLast() const;
+
+  // Whether this version contains flushed memtables that are only kept around
+  // for transaction conflict checking.
+  bool HasHistory() const { return !memlist_history_.empty(); }

  bool MemtableLimitExceeded(size_t usage);

@@ -211,7 +215,8 @@ class MemTableList {
        commit_in_progress_(false),
        flush_requested_(false),
        current_memory_usage_(0),
-        current_memory_usage_excluding_last_(0) {
+        current_memory_usage_excluding_last_(0),
+        current_has_history_(false) {
    current_->Ref();
  }

@@ -266,11 +271,16 @@ class MemTableList {
  // Returns an estimate of the number of bytes of data in use.
  size_t ApproximateMemoryUsage();

-  // Returns the cached current_memory_usage_excluding_last_ value
-  size_t ApproximateMemoryUsageExcludingLast();
+  // Returns the cached current_memory_usage_excluding_last_ value.
+  size_t ApproximateMemoryUsageExcludingLast() const;

-  // Update current_memory_usage_excluding_last_ from MemtableListVersion
-  void UpdateMemoryUsageExcludingLast();
+  // Returns the cached current_has_history_ value.
+  bool HasHistory() const;
+
+  // Updates current_memory_usage_excluding_last_ and current_has_history_
+  // from MemTableListVersion. Must be called whenever InstallNewVersion is
+  // called.
+  void UpdateCachedValuesFromMemTableListVersion();

  // `usage` is the current size of the mutable Memtable. When
  // max_write_buffer_size_to_maintain is used, total size of mutable and
@@ -388,7 +398,11 @@ class MemTableList {
  // The current memory usage.
  size_t current_memory_usage_;

+  // Cached value of current_->ApproximateMemoryUsageExcludingLast().
  std::atomic<size_t> current_memory_usage_excluding_last_;
+
+  // Cached value of current_->HasHistory().
+  std::atomic<bool> current_has_history_;
 };

 // Installs memtable atomic flush results.
@@ -92,6 +92,7 @@ class MemTableListTest : public testing::Test {
    CreateDB();
    // Create a mock VersionSet
    DBOptions db_options;
+    db_options.file_system = FileSystem::Default();
    ImmutableDBOptions immutable_db_options(db_options);
    EnvOptions env_options;
    std::shared_ptr<Cache> table_cache(NewLRUCache(50000, 16));
@@ -138,6 +139,8 @@ class MemTableListTest : public testing::Test {
    CreateDB();
    // Create a mock VersionSet
    DBOptions db_options;
+    db_options.file_system.reset(new LegacyFileSystemWrapper(db_options.env));
+
    ImmutableDBOptions immutable_db_options(db_options);
    EnvOptions env_options;
    std::shared_ptr<Cache> table_cache(NewLRUCache(50000, 16));
@@ -78,6 +78,9 @@ std::shared_ptr<DB> OpenDb(const std::string& dbname, const bool ttl = false,
  options.create_if_missing = true;
  options.merge_operator = std::make_shared<CountMergeOperator>();
  options.max_successive_merges = max_successive_merges;
+  if (max_successive_merges > 0) {
+    options.allow_concurrent_memtable_write = false;
+  }
  Status s;
  DestroyDB(dbname, Options());
 // DBWithTTL is not supported in ROCKSDB_LITE
@@ -71,6 +71,7 @@
 #include "db/table_cache.h"
 #include "db/version_edit.h"
 #include "db/write_batch_internal.h"
+#include "env/composite_env_wrapper.h"
 #include "file/filename.h"
 #include "file/writable_file_writer.h"
 #include "options/cf_options.h"
@@ -348,8 +349,8 @@ class Repairer {
    if (!status.ok()) {
      return status;
    }
-    std::unique_ptr<SequentialFileReader> lfile_reader(
-        new SequentialFileReader(std::move(lfile), logname));
+    std::unique_ptr<SequentialFileReader> lfile_reader(new SequentialFileReader(
+        NewLegacySequentialFileWrapper(lfile), logname));

    // Create the log reader.
    LogReporter reporter;
@@ -421,17 +422,19 @@ class Repairer {
      if (range_del_iter != nullptr) {
        range_del_iters.emplace_back(range_del_iter);
      }
+
+      LegacyFileSystemWrapper fs(env_);
      status = BuildTable(
-          dbname_, env_, *cfd->ioptions(), *cfd->GetLatestMutableCFOptions(),
-          env_options_, table_cache_, iter.get(), std::move(range_del_iters),
-          &meta, cfd->internal_comparator(),
-          cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
-          {}, kMaxSequenceNumber, snapshot_checker, kNoCompression,
-          0 /* sample_for_compression */, CompressionOptions(), false,
-          nullptr /* internal_stats */, TableFileCreationReason::kRecovery,
-          nullptr /* event_logger */, 0 /* job_id */, Env::IO_HIGH,
-          nullptr /* table_properties */, -1 /* level */, current_time,
-          write_hint);
+          dbname_, env_, &fs, *cfd->ioptions(),
+          *cfd->GetLatestMutableCFOptions(), env_options_, table_cache_,
+          iter.get(), std::move(range_del_iters), &meta,
+          cfd->internal_comparator(), cfd->int_tbl_prop_collector_factories(),
+          cfd->GetID(), cfd->GetName(), {}, kMaxSequenceNumber,
+          snapshot_checker, kNoCompression, 0 /* sample_for_compression */,
+          CompressionOptions(), false, nullptr /* internal_stats */,
+          TableFileCreationReason::kRecovery, nullptr /* event_logger */,
+          0 /* job_id */, Env::IO_HIGH, nullptr /* table_properties */,
+          -1 /* level */, current_time, write_hint);
      ROCKS_LOG_INFO(db_options_.info_log,
                     "Log #%" PRIu64 ": %d ops saved to Table #%" PRIu64 " %s",
                     log, counter, meta.fd.GetNumber(),
@@ -667,8 +670,14 @@ Status RepairDB(const std::string& dbname, const DBOptions& db_options,
 }

 Status RepairDB(const std::string& dbname, const Options& options) {
-  DBOptions db_options(options);
-  ColumnFamilyOptions cf_options(options);
+  Options opts(options);
+  if (opts.file_system == nullptr) {
+    opts.file_system.reset(new LegacyFileSystemWrapper(opts.env));
+    ;
+  }
+
+  DBOptions db_options(opts);
+  ColumnFamilyOptions cf_options(opts);
  Repairer repairer(dbname, db_options,
                    {}, cf_options /* default_cf_opts */,
                    cf_options /* unknown_cf_opts */,
@@ -74,7 +74,9 @@ TEST_F(RepairTest, CorruptManifest) {

  Close();
  ASSERT_OK(env_->FileExists(manifest_path));
-  CreateFile(env_, manifest_path, "blah", false /* use_fsync */);
+
+  LegacyFileSystemWrapper fs(env_);
+  CreateFile(&fs, manifest_path, "blah", false /* use_fsync */);
  ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
  Reopen(CurrentOptions());

@@ -153,7 +155,9 @@ TEST_F(RepairTest, CorruptSst) {
  Flush();
  auto sst_path = GetFirstSstPath();
  ASSERT_FALSE(sst_path.empty());
-  CreateFile(env_, sst_path, "blah", false /* use_fsync */);
+
+  LegacyFileSystemWrapper fs(env_);
+  CreateFile(&fs, sst_path, "blah", false /* use_fsync */);

  Close();
  ASSERT_OK(RepairDB(dbname_, CurrentOptions()));
@@ -148,6 +148,14 @@ class SnapshotList {
    }
  }

+  int64_t GetOldestSnapshotSequence() const {
+    if (empty()) {
+      return 0;
+    } else {
+      return oldest()->GetSequenceNumber();
+    }
+  }
+
  uint64_t count() const { return count_; }

 private:
@@ -63,10 +63,10 @@ void AppendVarint64(IterKey* key, uint64_t v) {
 }  // namespace

 TableCache::TableCache(const ImmutableCFOptions& ioptions,
-                       const EnvOptions& env_options, Cache* const cache,
+                       const FileOptions& file_options, Cache* const cache,
                       BlockCacheTracer* const block_cache_tracer)
    : ioptions_(ioptions),
-      env_options_(env_options),
+      file_options_(file_options),
      cache_(cache),
      immortal_tables_(false),
      block_cache_tracer_(block_cache_tracer) {
@@ -89,7 +89,7 @@ void TableCache::ReleaseHandle(Cache::Handle* handle) {
 }

 Status TableCache::GetTableReader(
-    const EnvOptions& env_options,
+    const FileOptions& file_options,
    const InternalKeyComparator& internal_comparator, const FileDescriptor& fd,
    bool sequential_mode, bool record_read_stats, HistogramImpl* file_read_hist,
    std::unique_ptr<TableReader>* table_reader,
@@ -97,13 +97,14 @@ Status TableCache::GetTableReader(
    bool prefetch_index_and_filter_in_cache) {
  std::string fname =
      TableFileName(ioptions_.cf_paths, fd.GetNumber(), fd.GetPathId());
-  std::unique_ptr<RandomAccessFile> file;
-  Status s = ioptions_.env->NewRandomAccessFile(fname, &file, env_options);
+  std::unique_ptr<FSRandomAccessFile> file;
+  Status s = ioptions_.fs->NewRandomAccessFile(fname, file_options, &file,
+                                               nullptr);

  RecordTick(ioptions_.statistics, NO_FILE_OPENS);
  if (s.ok()) {
    if (!sequential_mode && ioptions_.advise_random_on_open) {
-      file->Hint(RandomAccessFile::RANDOM);
+      file->Hint(FSRandomAccessFile::kRandom);
    }
    StopWatch sw(ioptions_.env, ioptions_.statistics, TABLE_OPEN_IO_MICROS);
    std::unique_ptr<RandomAccessFileReader> file_reader(
@@ -112,7 +113,7 @@ Status TableCache::GetTableReader(
            record_read_stats ? ioptions_.statistics : nullptr, SST_READ_MICROS,
            file_read_hist, ioptions_.rate_limiter, ioptions_.listeners));
    s = ioptions_.table_factory->NewTableReader(
-        TableReaderOptions(ioptions_, prefix_extractor, env_options,
+        TableReaderOptions(ioptions_, prefix_extractor, file_options,
                           internal_comparator, skip_filters, immortal_tables_,
                           level, fd.largest_seqno, block_cache_tracer_),
        std::move(file_reader), fd.GetFileSize(), table_reader,
@@ -129,7 +130,7 @@ void TableCache::EraseHandle(const FileDescriptor& fd, Cache::Handle* handle) {
  cache_->Erase(key);
 }

-Status TableCache::FindTable(const EnvOptions& env_options,
+Status TableCache::FindTable(const FileOptions& file_options,
                             const InternalKeyComparator& internal_comparator,
                             const FileDescriptor& fd, Cache::Handle** handle,
                             const SliceTransform* prefix_extractor,
@@ -150,7 +151,7 @@ Status TableCache::FindTable(const EnvOptions& env_options,
      return Status::Incomplete("Table not found in table_cache, no_io is set");
    }
    std::unique_ptr<TableReader> table_reader;
-    s = GetTableReader(env_options, internal_comparator, fd,
+    s = GetTableReader(file_options, internal_comparator, fd,
                       false /* sequential mode */, record_read_stats,
                       file_read_hist, &table_reader, prefix_extractor,
                       skip_filters, level, prefetch_index_and_filter_in_cache);
@@ -172,7 +173,7 @@ Status TableCache::FindTable(const EnvOptions& env_options,
 }

 InternalIterator* TableCache::NewIterator(
-    const ReadOptions& options, const EnvOptions& env_options,
+    const ReadOptions& options, const FileOptions& file_options,
    const InternalKeyComparator& icomparator, const FileMetaData& file_meta,
    RangeDelAggregator* range_del_agg, const SliceTransform* prefix_extractor,
    TableReader** table_reader_ptr, HistogramImpl* file_read_hist,
@@ -191,7 +192,7 @@ InternalIterator* TableCache::NewIterator(
  auto& fd = file_meta.fd;
  table_reader = fd.table_reader;
  if (table_reader == nullptr) {
-    s = FindTable(env_options, icomparator, fd, &handle, prefix_extractor,
+    s = FindTable(file_options, icomparator, fd, &handle, prefix_extractor,
                  options.read_tier == kBlockCacheTier /* no_io */,
                  !for_compaction /* record_read_stats */, file_read_hist,
                  skip_filters, level);
@@ -206,8 +207,8 @@ InternalIterator* TableCache::NewIterator(
      result = NewEmptyInternalIterator<Slice>(arena);
    } else {
      result = table_reader->NewIterator(options, prefix_extractor, arena,
-                                         skip_filters, caller,
-                                         env_options.compaction_readahead_size);
+                                   skip_filters, caller,
+                                   file_options.compaction_readahead_size);
    }
    if (handle != nullptr) {
      result->RegisterCleanup(&UnrefEntry, cache_, handle);
@@ -264,7 +265,7 @@ Status TableCache::GetRangeTombstoneIterator(
  TableReader* t = fd.table_reader;
  Cache::Handle* handle = nullptr;
  if (t == nullptr) {
-    s = FindTable(env_options_, internal_comparator, fd, &handle);
+    s = FindTable(file_options_, internal_comparator, fd, &handle);
    if (s.ok()) {
      t = GetTableReaderFromHandle(handle);
    }
@@ -381,7 +382,7 @@ Status TableCache::Get(const ReadOptions& options,
  if (!done && s.ok()) {
    if (t == nullptr) {
      s = FindTable(
-          env_options_, internal_comparator, fd, &handle, prefix_extractor,
+          file_options_, internal_comparator, fd, &handle, prefix_extractor,
          options.read_tier == kBlockCacheTier /* no_io */,
          true /* record_read_stats */, file_read_hist, skip_filters, level);
      if (s.ok()) {
@@ -481,7 +482,7 @@ Status TableCache::MultiGet(const ReadOptions& options,
  if (s.ok() && !table_range.empty()) {
    if (t == nullptr) {
      s = FindTable(
-          env_options_, internal_comparator, fd, &handle, prefix_extractor,
+          file_options_, internal_comparator, fd, &handle, prefix_extractor,
          options.read_tier == kBlockCacheTier /* no_io */,
          true /* record_read_stats */, file_read_hist, skip_filters, level);
      if (s.ok()) {
@@ -551,7 +552,7 @@ Status TableCache::MultiGet(const ReadOptions& options,
 }

 Status TableCache::GetTableProperties(
-    const EnvOptions& env_options,
+    const FileOptions& file_options,
    const InternalKeyComparator& internal_comparator, const FileDescriptor& fd,
    std::shared_ptr<const TableProperties>* properties,
    const SliceTransform* prefix_extractor, bool no_io) {
@@ -565,7 +566,7 @@ Status TableCache::GetTableProperties(
  }

  Cache::Handle* table_handle = nullptr;
-  s = FindTable(env_options, internal_comparator, fd, &table_handle,
+  s = FindTable(file_options, internal_comparator, fd, &table_handle,
                prefix_extractor, no_io);
  if (!s.ok()) {
    return s;
@@ -578,7 +579,7 @@ Status TableCache::GetTableProperties(
 }

 size_t TableCache::GetMemoryUsageByTableReader(
-    const EnvOptions& env_options,
+    const FileOptions& file_options,
    const InternalKeyComparator& internal_comparator, const FileDescriptor& fd,
    const SliceTransform* prefix_extractor) {
  Status s;
@@ -589,7 +590,7 @@ size_t TableCache::GetMemoryUsageByTableReader(
  }

  Cache::Handle* table_handle = nullptr;
-  s = FindTable(env_options, internal_comparator, fd, &table_handle,
+  s = FindTable(file_options, internal_comparator, fd, &table_handle,
                prefix_extractor, true);
  if (!s.ok()) {
    return 0;
@@ -614,7 +615,7 @@ uint64_t TableCache::ApproximateOffsetOf(
  Cache::Handle* table_handle = nullptr;
  if (table_reader == nullptr) {
    const bool for_compaction = (caller == TableReaderCaller::kCompaction);
-    Status s = FindTable(env_options_, internal_comparator, fd, &table_handle,
+    Status s = FindTable(file_options_, internal_comparator, fd, &table_handle,
                         prefix_extractor, false /* no_io */,
                         !for_compaction /* record_read_stats */);
    if (s.ok()) {
@@ -641,7 +642,7 @@ uint64_t TableCache::ApproximateSize(
  Cache::Handle* table_handle = nullptr;
  if (table_reader == nullptr) {
    const bool for_compaction = (caller == TableReaderCaller::kCompaction);
-    Status s = FindTable(env_options_, internal_comparator, fd, &table_handle,
+    Status s = FindTable(file_options_, internal_comparator, fd, &table_handle,
                         prefix_extractor, false /* no_io */,
                         !for_compaction /* record_read_stats */);
    if (s.ok()) {
@@ -49,7 +49,7 @@ class HistogramImpl;
 class TableCache {
 public:
  TableCache(const ImmutableCFOptions& ioptions,
-             const EnvOptions& storage_options, Cache* cache,
+             const FileOptions& storage_options, Cache* cache,
             BlockCacheTracer* const block_cache_tracer);
  ~TableCache();

@@ -67,7 +67,7 @@ class TableCache {
  // @param skip_filters Disables loading/accessing the filter block
  // @param level The level this table is at, -1 for "not set / don't know"
  InternalIterator* NewIterator(
-      const ReadOptions& options, const EnvOptions& toptions,
+      const ReadOptions& options, const FileOptions& toptions,
      const InternalKeyComparator& internal_comparator,
      const FileMetaData& file_meta, RangeDelAggregator* range_del_agg,
      const SliceTransform* prefix_extractor, TableReader** table_reader_ptr,
@@ -128,7 +128,7 @@ class TableCache {
  // Find table reader
  // @param skip_filters Disables loading/accessing the filter block
  // @param level == -1 means not specified
-  Status FindTable(const EnvOptions& toptions,
+  Status FindTable(const FileOptions& toptions,
                   const InternalKeyComparator& internal_comparator,
                   const FileDescriptor& file_fd, Cache::Handle**,
                   const SliceTransform* prefix_extractor = nullptr,
@@ -146,7 +146,7 @@ class TableCache {
  // @returns: `properties` will be reset on success. Please note that we will
  //            return Status::Incomplete() if table is not present in cache and
  //            we set `no_io` to be true.
-  Status GetTableProperties(const EnvOptions& toptions,
+  Status GetTableProperties(const FileOptions& toptions,
                            const InternalKeyComparator& internal_comparator,
                            const FileDescriptor& file_meta,
                            std::shared_ptr<const TableProperties>* properties,
@@ -156,7 +156,7 @@ class TableCache {
  // Return total memory usage of the table reader of the file.
  // 0 if table reader of the file is not loaded.
  size_t GetMemoryUsageByTableReader(
-      const EnvOptions& toptions,
+      const FileOptions& toptions,
      const InternalKeyComparator& internal_comparator,
      const FileDescriptor& fd,
      const SliceTransform* prefix_extractor = nullptr);
@@ -193,7 +193,7 @@ class TableCache {

 private:
  // Build a table reader
-  Status GetTableReader(const EnvOptions& env_options,
+  Status GetTableReader(const FileOptions& file_options,
                        const InternalKeyComparator& internal_comparator,
                        const FileDescriptor& fd, bool sequential_mode,
                        bool record_read_stats, HistogramImpl* file_read_hist,
@@ -216,7 +216,7 @@ class TableCache {
                       size_t prefix_size, GetContext* get_context);

  const ImmutableCFOptions& ioptions_;
-  const EnvOptions& env_options_;
+  const FileOptions& file_options_;
  Cache* const cache_;
  std::string row_cache_id_;
  bool immortal_tables_;
@@ -12,6 +12,7 @@
 #include "db/db_impl/db_impl.h"
 #include "db/dbformat.h"
 #include "db/table_properties_collector.h"
+#include "env/composite_env_wrapper.h"
 #include "file/sequence_file_reader.h"
 #include "file/writable_file_writer.h"
 #include "options/cf_options.h"
@@ -48,7 +49,8 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
                 std::unique_ptr<TableBuilder>* builder) {
  std::unique_ptr<WritableFile> wf(new test::StringSink);
  writable->reset(
-      new WritableFileWriter(std::move(wf), "" /* don't care */, EnvOptions()));
+      new WritableFileWriter(NewLegacyWritableFileWrapper(std::move(wf)),
+                             "" /* don't care */, EnvOptions()));
  int unknown_level = -1;
  builder->reset(NewTableBuilder(
      ioptions, moptions, internal_comparator, int_tbl_prop_collector_factories,
@@ -282,8 +284,9 @@ void TestCustomizedTablePropertiesCollector(
  writer->Flush();

  // -- Step 2: Read properties
-  test::StringSink* fwf =
-      static_cast<test::StringSink*>(writer->writable_file());
+  LegacyWritableFileWrapper* file =
+      static_cast<LegacyWritableFileWrapper*>(writer->writable_file());
+  test::StringSink* fwf = static_cast<test::StringSink*>(file->target());
  std::unique_ptr<RandomAccessFileReader> fake_file_reader(
      test::GetRandomAccessFileReader(
          new test::StringSource(fwf->contents())));
@@ -422,8 +425,9 @@ void TestInternalKeyPropertiesCollector(
    ASSERT_OK(builder->Finish());
    writable->Flush();

-    test::StringSink* fwf =
-        static_cast<test::StringSink*>(writable->writable_file());
+    LegacyWritableFileWrapper* file =
+        static_cast<LegacyWritableFileWrapper*>(writable->writable_file());
+    test::StringSink* fwf = static_cast<test::StringSink*>(file->target());
    std::unique_ptr<RandomAccessFileReader> reader(
        test::GetRandomAccessFileReader(
            new test::StringSource(fwf->contents())));
@@ -42,22 +42,23 @@ TransactionLogIteratorImpl::TransactionLogIteratorImpl(
 Status TransactionLogIteratorImpl::OpenLogFile(
    const LogFile* log_file,
    std::unique_ptr<SequentialFileReader>* file_reader) {
-  Env* env = options_->env;
-  std::unique_ptr<SequentialFile> file;
+  FileSystem* fs = options_->fs.get();
+  std::unique_ptr<FSSequentialFile> file;
  std::string fname;
  Status s;
-  EnvOptions optimized_env_options = env->OptimizeForLogRead(soptions_);
+  EnvOptions optimized_env_options = fs->OptimizeForLogRead(soptions_);
  if (log_file->Type() == kArchivedLogFile) {
    fname = ArchivedLogFileName(dir_, log_file->LogNumber());
-    s = env->NewSequentialFile(fname, &file, optimized_env_options);
+    s = fs->NewSequentialFile(fname, optimized_env_options, &file, nullptr);
  } else {
    fname = LogFileName(dir_, log_file->LogNumber());
-    s = env->NewSequentialFile(fname, &file, optimized_env_options);
+    s = fs->NewSequentialFile(fname, optimized_env_options, &file, nullptr);
    if (!s.ok()) {
      //  If cannot open file in DB directory.
      //  Try the archive dir, as it could have moved in the meanwhile.
      fname = ArchivedLogFileName(dir_, log_file->LogNumber());
-      s = env->NewSequentialFile(fname, &file, optimized_env_options);
+      s = fs->NewSequentialFile(fname, optimized_env_options,
+                                &file, nullptr);
    }
  }
  if (s.ok()) {
@@ -34,10 +34,7 @@ ColumnFamilyData* TrimHistoryScheduler::TakeNextColumnFamily() {
      // success
      return cfd;
    }
-    if (cfd->Unref()) {
-      // no longer relevant, retry
-      delete cfd;
-    }
+    cfd->UnrefAndTryDelete();
  }
 }

@@ -49,9 +46,7 @@ bool TrimHistoryScheduler::Empty() {
 void TrimHistoryScheduler::Clear() {
  ColumnFamilyData* cfd;
  while ((cfd = TakeNextColumnFamily()) != nullptr) {
-    if (cfd->Unref()) {
-      delete cfd;
-    }
+    cfd->UnrefAndTryDelete();
  }
  assert(Empty());
 }
@@ -83,7 +83,7 @@ class VersionBuilder::Rep {
    std::unordered_map<uint64_t, FileMetaData*> added_files;
  };

-  const EnvOptions& env_options_;
+  const FileOptions& file_options_;
  Logger* info_log_;
  TableCache* table_cache_;
  VersionStorageInfo* base_vstorage_;
@@ -101,9 +101,10 @@ class VersionBuilder::Rep {
  FileComparator level_nonzero_cmp_;

 public:
-  Rep(const EnvOptions& env_options, Logger* info_log, TableCache* table_cache,
+  Rep(const FileOptions& file_options, Logger* info_log,
+      TableCache* table_cache,
      VersionStorageInfo* base_vstorage)
-      : env_options_(env_options),
+      : file_options_(file_options),
        info_log_(info_log),
        table_cache_(table_cache),
        base_vstorage_(base_vstorage),
@@ -461,7 +462,7 @@ class VersionBuilder::Rep {
        auto* file_meta = files_meta[file_idx].first;
        int level = files_meta[file_idx].second;
        statuses[file_idx] = table_cache_->FindTable(
-            env_options_, *(base_vstorage_->InternalComparator()),
+            file_options_, *(base_vstorage_->InternalComparator()),
            file_meta->fd, &file_meta->table_reader_handle, prefix_extractor,
            false /*no_io */, true /* record_read_stats */,
            internal_stats->GetFileReadHist(level), false, level,
@@ -500,11 +501,11 @@ class VersionBuilder::Rep {
  }
 };

-VersionBuilder::VersionBuilder(const EnvOptions& env_options,
+VersionBuilder::VersionBuilder(const FileOptions& file_options,
                               TableCache* table_cache,
                               VersionStorageInfo* base_vstorage,
                               Logger* info_log)
-    : rep_(new Rep(env_options, info_log, table_cache, base_vstorage)) {}
+    : rep_(new Rep(file_options, info_log, table_cache, base_vstorage)) {}

 VersionBuilder::~VersionBuilder() { delete rep_; }

@@ -8,7 +8,7 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 //
 #pragma once
-#include "rocksdb/env.h"
+#include "rocksdb/file_system.h"
 #include "rocksdb/slice_transform.h"

 namespace rocksdb {
@@ -24,7 +24,7 @@ class InternalStats;
 // Versions that contain full copies of the intermediate state.
 class VersionBuilder {
 public:
-  VersionBuilder(const EnvOptions& env_options, TableCache* table_cache,
+  VersionBuilder(const FileOptions& file_options, TableCache* table_cache,
                 VersionStorageInfo* base_vstorage, Logger* info_log = nullptr);
  ~VersionBuilder();
  Status CheckConsistency(VersionStorageInfo* vstorage);
@@ -853,7 +853,7 @@ namespace {
 class LevelIterator final : public InternalIterator {
 public:
  LevelIterator(TableCache* table_cache, const ReadOptions& read_options,
-                const EnvOptions& env_options,
+                const FileOptions& file_options,
                const InternalKeyComparator& icomparator,
                const LevelFilesBrief* flevel,
                const SliceTransform* prefix_extractor, bool should_sample,
@@ -863,7 +863,7 @@ class LevelIterator final : public InternalIterator {
                    compaction_boundaries = nullptr)
      : table_cache_(table_cache),
        read_options_(read_options),
-        env_options_(env_options),
+        file_options_(file_options),
        icomparator_(icomparator),
        user_comparator_(icomparator.user_comparator()),
        flevel_(flevel),
@@ -975,7 +975,7 @@ class LevelIterator final : public InternalIterator {
    }
    CheckMayBeOutOfLowerBound();
    return table_cache_->NewIterator(
-        read_options_, env_options_, icomparator_, *file_meta.file_metadata,
+        read_options_, file_options_, icomparator_, *file_meta.file_metadata,
        range_del_agg_, prefix_extractor_,
        nullptr /* don't need reference to table */, file_read_hist_, caller_,
        /*arena=*/nullptr, skip_filters_, level_, smallest_compaction_key,
@@ -998,7 +998,7 @@ class LevelIterator final : public InternalIterator {

  TableCache* table_cache_;
  const ReadOptions read_options_;
-  const EnvOptions& env_options_;
+  const FileOptions& file_options_;
  const InternalKeyComparator& icomparator_;
  const UserComparatorWrapper user_comparator_;
  const LevelFilesBrief* flevel_;
@@ -1206,7 +1206,7 @@ class BaseReferencedVersionBuilder {
 public:
  explicit BaseReferencedVersionBuilder(ColumnFamilyData* cfd)
      : version_builder_(new VersionBuilder(
-            cfd->current()->version_set()->env_options(), cfd->table_cache(),
+            cfd->current()->version_set()->file_options(), cfd->table_cache(),
            cfd->current()->storage_info(), cfd->ioptions()->info_log)),
        version_(cfd->current()) {
    version_->Ref();
@@ -1227,7 +1227,7 @@ Status Version::GetTableProperties(std::shared_ptr<const TableProperties>* tp,
  auto table_cache = cfd_->table_cache();
  auto ioptions = cfd_->ioptions();
  Status s = table_cache->GetTableProperties(
-      env_options_, cfd_->internal_comparator(), file_meta->fd, tp,
+      file_options_, cfd_->internal_comparator(), file_meta->fd, tp,
      mutable_cf_options_.prefix_extractor.get(), true /* no io */);
  if (s.ok()) {
    return s;
@@ -1241,7 +1241,7 @@ Status Version::GetTableProperties(std::shared_ptr<const TableProperties>* tp,

  // 2. Table is not present in table cache, we'll read the table properties
  // directly from the properties block in the file.
-  std::unique_ptr<RandomAccessFile> file;
+  std::unique_ptr<FSRandomAccessFile> file;
  std::string file_name;
  if (fname != nullptr) {
    file_name = *fname;
@@ -1250,7 +1250,8 @@ Status Version::GetTableProperties(std::shared_ptr<const TableProperties>* tp,
      TableFileName(ioptions->cf_paths, file_meta->fd.GetNumber(),
                    file_meta->fd.GetPathId());
  }
-  s = ioptions->env->NewRandomAccessFile(file_name, &file, env_options_);
+  s = ioptions->fs->NewRandomAccessFile(file_name, file_options_, &file,
+                                        nullptr);
  if (!s.ok()) {
    return s;
  }
@@ -1420,7 +1421,7 @@ size_t Version::GetMemoryUsageByTableReaders() {
  for (auto& file_level : storage_info_.level_files_brief_) {
    for (size_t i = 0; i < file_level.num_files; i++) {
      total_usage += cfd_->table_cache()->GetMemoryUsageByTableReader(
-          env_options_, cfd_->internal_comparator(), file_level.files[i].fd,
+          file_options_, cfd_->internal_comparator(), file_level.files[i].fd,
          mutable_cf_options_.prefix_extractor.get());
    }
  }
@@ -1547,7 +1548,7 @@ double VersionStorageInfo::GetEstimatedCompressionRatioAtLevel(
 }

 void Version::AddIterators(const ReadOptions& read_options,
-                           const EnvOptions& soptions,
+                           const FileOptions& soptions,
                           MergeIteratorBuilder* merge_iter_builder,
                           RangeDelAggregator* range_del_agg) {
  assert(storage_info_.finalized_);
@@ -1559,7 +1560,7 @@ void Version::AddIterators(const ReadOptions& read_options,
 }

 void Version::AddIteratorsForLevel(const ReadOptions& read_options,
-                                   const EnvOptions& soptions,
+                                   const FileOptions& soptions,
                                   MergeIteratorBuilder* merge_iter_builder,
                                   int level,
                                   RangeDelAggregator* range_del_agg) {
@@ -1614,7 +1615,7 @@ void Version::AddIteratorsForLevel(const ReadOptions& read_options,
 }

 Status Version::OverlapWithLevelIterator(const ReadOptions& read_options,
-                                         const EnvOptions& env_options,
+                                         const FileOptions& file_options,
                                         const Slice& smallest_user_key,
                                         const Slice& largest_user_key,
                                         int level, bool* overlap) {
@@ -1638,7 +1639,7 @@ Status Version::OverlapWithLevelIterator(const ReadOptions& read_options,
        continue;
      }
      ScopedArenaIterator iter(cfd_->table_cache()->NewIterator(
-          read_options, env_options, cfd_->internal_comparator(),
+          read_options, file_options, cfd_->internal_comparator(),
          *file->file_metadata, &range_del_agg,
          mutable_cf_options_.prefix_extractor.get(), nullptr,
          cfd_->internal_stats()->GetFileReadHist(0),
@@ -1655,7 +1656,7 @@ Status Version::OverlapWithLevelIterator(const ReadOptions& read_options,
  } else if (storage_info_.LevelFilesBrief(level).num_files > 0) {
    auto mem = arena.AllocateAligned(sizeof(LevelIterator));
    ScopedArenaIterator iter(new (mem) LevelIterator(
-        cfd_->table_cache(), read_options, env_options,
+        cfd_->table_cache(), read_options, file_options,
        cfd_->internal_comparator(), &storage_info_.LevelFilesBrief(level),
        mutable_cf_options_.prefix_extractor.get(), should_sample_file_read(),
        cfd_->internal_stats()->GetFileReadHist(level),
@@ -1719,7 +1720,7 @@ VersionStorageInfo::VersionStorageInfo(
 }

 Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset,
-                 const EnvOptions& env_opt,
+                 const FileOptions& file_opt,
                 const MutableCFOptions mutable_cf_options,
                 uint64_t version_number)
    : env_(vset->env_),
@@ -1744,7 +1745,7 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset,
      next_(this),
      prev_(this),
      refs_(0),
-      env_options_(env_opt),
+      file_options_(file_opt),
      mutable_cf_options_(mutable_cf_options),
      version_number_(version_number) {}

@@ -3509,7 +3510,7 @@ void AtomicGroupReadBuffer::Clear() {

 VersionSet::VersionSet(const std::string& dbname,
                       const ImmutableDBOptions* _db_options,
-                       const EnvOptions& storage_options, Cache* table_cache,
+                       const FileOptions& storage_options, Cache* table_cache,
                       WriteBufferManager* write_buffer_manager,
                       WriteController* write_controller,
                       BlockCacheTracer* const block_cache_tracer)
@@ -3517,6 +3518,7 @@ VersionSet::VersionSet(const std::string& dbname,
          dbname, _db_options, storage_options, table_cache,
          write_buffer_manager, write_controller, block_cache_tracer)),
      env_(_db_options->env),
+      fs_(_db_options->fs.get()),
      dbname_(dbname),
      db_options_(_db_options),
      next_file_number_(2),
@@ -3529,7 +3531,7 @@ VersionSet::VersionSet(const std::string& dbname,
      prev_log_number_(0),
      current_version_number_(0),
      manifest_file_size_(0),
-      env_options_(storage_options),
+      file_options_(storage_options),
      block_cache_tracer_(block_cache_tracer) {}

 VersionSet::~VersionSet() {
@@ -3655,7 +3657,7 @@ Status VersionSet::ProcessManifestWrites(
        }
      }
      if (version == nullptr) {
-        version = new Version(last_writer->cfd, this, env_options_,
+        version = new Version(last_writer->cfd, this, file_options_,
                              last_writer->mutable_cf_options,
                              current_version_number_++);
        versions.push_back(version);
@@ -3751,6 +3753,10 @@ Status VersionSet::ProcessManifestWrites(
    pending_manifest_file_number_ = manifest_file_number_;
  }

+  // Local cached copy of state variable(s). WriteCurrentStateToManifest()
+  // reads its content after releasing db mutex to avoid race with
+  // SwitchMemtable().
+  std::unordered_map<uint32_t, MutableCFState> curr_state;
  if (new_descriptor_log) {
    // if we are writing out new snapshot make sure to persist max column
    // family.
@@ -3758,10 +3764,14 @@ Status VersionSet::ProcessManifestWrites(
      first_writer.edit_list.front()->SetMaxColumnFamily(
          column_family_set_->GetMaxColumnFamily());
    }
+    for (const auto* cfd : *column_family_set_) {
+      assert(curr_state.find(cfd->GetID()) == curr_state.end());
+      curr_state[cfd->GetID()] = {cfd->GetLogNumber()};
+    }
  }

  {
-    EnvOptions opt_env_opts = env_->OptimizeForManifestWrite(env_options_);
+    FileOptions opt_file_opts = fs_->OptimizeForManifestWrite(file_options_);
    mu->Unlock();

    TEST_SYNC_POINT("VersionSet::LogAndApply:WriteManifest");
@@ -3788,19 +3798,19 @@ Status VersionSet::ProcessManifestWrites(
                     pending_manifest_file_number_);
      std::string descriptor_fname =
          DescriptorFileName(dbname_, pending_manifest_file_number_);
-      std::unique_ptr<WritableFile> descriptor_file;
-      s = NewWritableFile(env_, descriptor_fname, &descriptor_file,
-                          opt_env_opts);
+      std::unique_ptr<FSWritableFile> descriptor_file;
+      s = NewWritableFile(fs_, descriptor_fname, &descriptor_file,
+                          opt_file_opts);
      if (s.ok()) {
        descriptor_file->SetPreallocationBlockSize(
            db_options_->manifest_preallocation_size);

        std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
-            std::move(descriptor_file), descriptor_fname, opt_env_opts, env_,
+            std::move(descriptor_file), descriptor_fname, opt_file_opts, env_,
            nullptr, db_options_->listeners));
        descriptor_log_.reset(
            new log::Writer(std::move(file_writer), 0, false));
-        s = WriteCurrentStateToManifest(descriptor_log_.get());
+        s = WriteCurrentStateToManifest(curr_state, descriptor_log_.get());
      }
    }

@@ -3888,9 +3898,7 @@ Status VersionSet::ProcessManifestWrites(
    } else if (first_writer.edit_list.front()->is_column_family_drop_) {
      assert(batch_edits.size() == 1);
      first_writer.cfd->SetDropped();
-      if (first_writer.cfd->Unref()) {
-        delete first_writer.cfd;
-      }
+      first_writer.cfd->UnrefAndTryDelete();
    } else {
      // Each version in versions corresponds to a column family.
      // For each column family, update its log number indicating that logs
@@ -4168,8 +4176,7 @@ Status VersionSet::ApplyOneVersionEditToBuilder(
      builders.erase(builder);
      cfd = column_family_set_->GetColumnFamily(edit.column_family_);
      assert(cfd != nullptr);
-      if (cfd->Unref()) {
-        delete cfd;
+      if (cfd->UnrefAndTryDelete()) {
        cfd = nullptr;
      } else {
        // who else can have reference to cfd!?
@@ -4254,15 +4261,16 @@ Status VersionSet::ExtractInfoFromVersionEdit(
  return Status::OK();
 }

-Status VersionSet::GetCurrentManifestPath(const std::string& dbname, Env* env,
+Status VersionSet::GetCurrentManifestPath(const std::string& dbname,
+                                          FileSystem* fs,
                                          std::string* manifest_path,
                                          uint64_t* manifest_file_number) {
-  assert(env != nullptr);
+  assert(fs != nullptr);
  assert(manifest_path != nullptr);
  assert(manifest_file_number != nullptr);

  std::string fname;
-  Status s = ReadFileToString(env, CurrentFileName(dbname), &fname);
+  Status s = ReadFileToString(fs, CurrentFileName(dbname), &fname);
  if (!s.ok()) {
    return s;
  }
@@ -4364,7 +4372,7 @@ Status VersionSet::Recover(

  // Read "CURRENT" file, which contains a pointer to the current manifest file
  std::string manifest_path;
-  Status s = GetCurrentManifestPath(dbname_, env_, &manifest_path,
+  Status s = GetCurrentManifestPath(dbname_, fs_, &manifest_path,
                                    &manifest_file_number_);
  if (!s.ok()) {
    return s;
@@ -4375,9 +4383,10 @@ Status VersionSet::Recover(

  std::unique_ptr<SequentialFileReader> manifest_file_reader;
  {
-    std::unique_ptr<SequentialFile> manifest_file;
-    s = env_->NewSequentialFile(manifest_path, &manifest_file,
-                                env_->OptimizeForManifestRead(env_options_));
+    std::unique_ptr<FSSequentialFile> manifest_file;
+    s = fs_->NewSequentialFile(manifest_path,
+                               fs_->OptimizeForManifestRead(file_options_),
+                               &manifest_file, nullptr);
    if (!s.ok()) {
      return s;
    }
@@ -4386,7 +4395,8 @@ Status VersionSet::Recover(
                                 db_options_->log_readahead_size));
  }
  uint64_t current_manifest_file_size;
-  s = env_->GetFileSize(manifest_path, &current_manifest_file_size);
+  s = fs_->GetFileSize(manifest_path, IOOptions(), &current_manifest_file_size,
+                       nullptr);
  if (!s.ok()) {
    return s;
  }
@@ -4493,7 +4503,7 @@ Status VersionSet::Recover(
          true /* is_initial_load */,
          cfd->GetLatestMutableCFOptions()->prefix_extractor.get());

-      Version* v = new Version(cfd, this, env_options_,
+      Version* v = new Version(cfd, this, file_options_,
                               *cfd->GetLatestMutableCFOptions(),
                               current_version_number_++);
      builder->SaveTo(v->storage_info());
@@ -4538,23 +4548,24 @@ Status VersionSet::Recover(
 }

 Status VersionSet::ListColumnFamilies(std::vector<std::string>* column_families,
-                                      const std::string& dbname, Env* env) {
+                                      const std::string& dbname,
+                                      FileSystem* fs) {
  // these are just for performance reasons, not correcntes,
  // so we're fine using the defaults
-  EnvOptions soptions;
+  FileOptions soptions;
  // Read "CURRENT" file, which contains a pointer to the current manifest file
  std::string manifest_path;
  uint64_t manifest_file_number;
-  Status s = GetCurrentManifestPath(dbname, env, &manifest_path,
-                                    &manifest_file_number);
+  Status s =
+      GetCurrentManifestPath(dbname, fs, &manifest_path, &manifest_file_number);
  if (!s.ok()) {
    return s;
  }

  std::unique_ptr<SequentialFileReader> file_reader;
  {
-    std::unique_ptr<SequentialFile> file;
-    s = env->NewSequentialFile(manifest_path, &file, soptions);
+    std::unique_ptr<FSSequentialFile> file;
+    s = fs->NewSequentialFile(manifest_path, soptions, &file, nullptr);
    if (!s.ok()) {
      return s;
  }
@@ -4608,7 +4619,7 @@ Status VersionSet::ListColumnFamilies(std::vector<std::string>* column_families,
 #ifndef ROCKSDB_LITE
 Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
                                        const Options* options,
-                                        const EnvOptions& env_options,
+                                        const FileOptions& file_options,
                                        int new_levels) {
  if (new_levels <= 1) {
    return Status::InvalidArgument(
@@ -4621,7 +4632,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
                                        options->table_cache_numshardbits));
  WriteController wc(options->delayed_write_rate);
  WriteBufferManager wb(options->db_write_buffer_size);
-  VersionSet versions(dbname, &db_options, env_options, tc.get(), &wb, &wc,
+  VersionSet versions(dbname, &db_options, file_options, tc.get(), &wb, &wc,
                      /*block_cache_tracer=*/nullptr);
  Status status;

@@ -4697,9 +4708,11 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
  std::unique_ptr<SequentialFileReader> file_reader;
  Status s;
  {
-    std::unique_ptr<SequentialFile> file;
-    s = options.env->NewSequentialFile(
-        dscname, &file, env_->OptimizeForManifestRead(env_options_));
+    std::unique_ptr<FSSequentialFile> file;
+    s = options.file_system->NewSequentialFile(
+        dscname,
+        options.file_system->OptimizeForManifestRead(file_options_), &file,
+        nullptr);
    if (!s.ok()) {
      return s;
    }
@@ -4781,8 +4794,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
        comparators.erase(edit.column_family_);
        cfd = column_family_set_->GetColumnFamily(edit.column_family_);
        assert(cfd != nullptr);
-        cfd->Unref();
-        delete cfd;
+        cfd->UnrefAndTryDelete();
        cfd = nullptr;
      } else {
        if (!cf_in_builders) {
@@ -4860,7 +4872,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
      assert(builders_iter != builders.end());
      auto builder = builders_iter->second->version_builder();

-      Version* v = new Version(cfd, this, env_options_,
+      Version* v = new Version(cfd, this, file_options_,
                               *cfd->GetLatestMutableCFOptions(),
                               current_version_number_++);
      builder->SaveTo(v->storage_info());
@@ -4914,7 +4926,9 @@ void VersionSet::MarkMinLogNumberToKeep2PC(uint64_t number) {
  }
 }

-Status VersionSet::WriteCurrentStateToManifest(log::Writer* log) {
+Status VersionSet::WriteCurrentStateToManifest(
+    const std::unordered_map<uint32_t, MutableCFState>& curr_state,
+    log::Writer* log) {
  // TODO: Break up into multiple records to reduce memory usage on recovery?

  // WARNING: This method doesn't hold a mutex!!
@@ -4980,7 +4994,10 @@ Status VersionSet::WriteCurrentStateToManifest(log::Writer* log) {
                       f->oldest_ancester_time, f->file_creation_time);
        }
      }
-      edit.SetLogNumber(cfd->GetLogNumber());
+      const auto iter = curr_state.find(cfd->GetID());
+      assert(iter != curr_state.end());
+      uint64_t log_number = iter->second.log_number;
+      edit.SetLogNumber(log_number);
      std::string record;
      if (!edit.EncodeTo(&record)) {
        return Status::Corruption(
@@ -5237,7 +5254,7 @@ void VersionSet::AddLiveFiles(std::vector<FileDescriptor>* live_list) {

 InternalIterator* VersionSet::MakeInputIterator(
    const Compaction* c, RangeDelAggregator* range_del_agg,
-    const EnvOptions& env_options_compactions) {
+    const FileOptions& file_options_compactions) {
  auto cfd = c->column_family_data();
  ReadOptions read_options;
  read_options.verify_checksums = true;
@@ -5262,7 +5279,8 @@ InternalIterator* VersionSet::MakeInputIterator(
        const LevelFilesBrief* flevel = c->input_levels(which);
        for (size_t i = 0; i < flevel->num_files; i++) {
          list[num++] = cfd->table_cache()->NewIterator(
-              read_options, env_options_compactions, cfd->internal_comparator(),
+              read_options, file_options_compactions,
+              cfd->internal_comparator(),
              *flevel->files[i].file_metadata, range_del_agg,
              c->mutable_cf_options()->prefix_extractor.get(),
              /*table_reader_ptr=*/nullptr,
@@ -5275,7 +5293,7 @@ InternalIterator* VersionSet::MakeInputIterator(
      } else {
        // Create concatenating iterator for the files from this level
        list[num++] = new LevelIterator(
-            cfd->table_cache(), read_options, env_options_compactions,
+            cfd->table_cache(), read_options, file_options_compactions,
            cfd->internal_comparator(), c->input_levels(which),
            c->mutable_cf_options()->prefix_extractor.get(),
            /*should_sample=*/false,
@@ -5428,7 +5446,7 @@ ColumnFamilyData* VersionSet::CreateColumnFamily(

  MutableCFOptions dummy_cf_options;
  Version* dummy_versions =
-      new Version(nullptr, this, env_options_, dummy_cf_options);
+      new Version(nullptr, this, file_options_, dummy_cf_options);
  // Ref() dummy version once so that later we can call Unref() to delete it
  // by avoiding calling "delete" explicitly (~Version is private)
  dummy_versions->Ref();
@@ -5436,7 +5454,7 @@ ColumnFamilyData* VersionSet::CreateColumnFamily(
      edit->column_family_name_, edit->column_family_, dummy_versions,
      cf_options);

-  Version* v = new Version(new_cfd, this, env_options_,
+  Version* v = new Version(new_cfd, this, file_options_,
                           *new_cfd->GetLatestMutableCFOptions(),
                           current_version_number_++);

@@ -5480,11 +5498,11 @@ uint64_t VersionSet::GetTotalSstFilesSize(Version* dummy_versions) {

 ReactiveVersionSet::ReactiveVersionSet(const std::string& dbname,
                                       const ImmutableDBOptions* _db_options,
-                                       const EnvOptions& _env_options,
+                                       const FileOptions& _file_options,
                                       Cache* table_cache,
                                       WriteBufferManager* write_buffer_manager,
                                       WriteController* write_controller)
-    : VersionSet(dbname, _db_options, _env_options, table_cache,
+    : VersionSet(dbname, _db_options, _file_options, table_cache,
                 write_buffer_manager, write_controller,
                 /*block_cache_tracer=*/nullptr),
      number_of_edits_to_skip_(0) {}
@@ -5615,7 +5633,7 @@ Status ReactiveVersionSet::Recover(
      assert(builders_iter != builders.end());
      auto* builder = builders_iter->second->version_builder();

-      Version* v = new Version(cfd, this, env_options_,
+      Version* v = new Version(cfd, this, file_options_,
                               *cfd->GetLatestMutableCFOptions(),
                               current_version_number_++);
      builder->SaveTo(v->storage_info());
@@ -5808,8 +5826,7 @@ Status ReactiveVersionSet::ApplyOneVersionEditToBuilder(
    // secondary instance. (Is it possible that the ref count for cfd is 0 but
    // the ref count for its versions is higher than 0?)
    cfd->SetDropped();
-    if (cfd->Unref()) {
-      delete cfd;
+    if (cfd->UnrefAndTryDelete()) {
      cfd = nullptr;
    }
    active_version_builders_.erase(builder_iter);
@@ -5836,7 +5853,7 @@ Status ReactiveVersionSet::ApplyOneVersionEditToBuilder(
        &s);

    if (s.ok()) {
-      auto version = new Version(cfd, this, env_options_,
+      auto version = new Version(cfd, this, file_options_,
                                 *cfd->GetLatestMutableCFOptions(),
                                 current_version_number_++);
      builder->SaveTo(version->storage_info());
@@ -5879,9 +5896,9 @@ Status ReactiveVersionSet::MaybeSwitchManifest(
  Status s;
  do {
    std::string manifest_path;
-    s = GetCurrentManifestPath(dbname_, env_, &manifest_path,
+    s = GetCurrentManifestPath(dbname_, fs_, &manifest_path,
                               &manifest_file_number_);
-    std::unique_ptr<SequentialFile> manifest_file;
+    std::unique_ptr<FSSequentialFile> manifest_file;
    if (s.ok()) {
      if (nullptr == manifest_reader->get() ||
          manifest_reader->get()->file()->file_name() != manifest_path) {
@@ -5891,9 +5908,9 @@ Status ReactiveVersionSet::MaybeSwitchManifest(
        TEST_SYNC_POINT(
            "ReactiveVersionSet::MaybeSwitchManifest:"
            "AfterGetCurrentManifestPath:1");
-        s = env_->NewSequentialFile(
-            manifest_path, &manifest_file,
-            env_->OptimizeForManifestRead(env_options_));
+        s = fs_->NewSequentialFile(manifest_path,
+                                   env_->OptimizeForManifestRead(file_options_),
+                                   &manifest_file, nullptr);
      } else {
        // No need to switch manifest.
        break;
@@ -559,15 +559,15 @@ class Version {
  // Append to *iters a sequence of iterators that will
  // yield the contents of this Version when merged together.
  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  void AddIterators(const ReadOptions&, const EnvOptions& soptions,
+  void AddIterators(const ReadOptions&, const FileOptions& soptions,
                    MergeIteratorBuilder* merger_iter_builder,
                    RangeDelAggregator* range_del_agg);

-  void AddIteratorsForLevel(const ReadOptions&, const EnvOptions& soptions,
+  void AddIteratorsForLevel(const ReadOptions&, const FileOptions& soptions,
                            MergeIteratorBuilder* merger_iter_builder,
                            int level, RangeDelAggregator* range_del_agg);

-  Status OverlapWithLevelIterator(const ReadOptions&, const EnvOptions&,
+  Status OverlapWithLevelIterator(const ReadOptions&, const FileOptions&,
                                  const Slice& smallest_user_key,
                                  const Slice& largest_user_key,
                                  int level, bool* overlap);
@@ -684,6 +684,7 @@ class Version {

 private:
  Env* env_;
+  FileSystem* fs_;
  friend class ReactiveVersionSet;
  friend class VersionSet;

@@ -729,14 +730,14 @@ class Version {
  Version* next_;               // Next version in linked list
  Version* prev_;               // Previous version in linked list
  int refs_;                    // Number of live refs to this version
-  const EnvOptions env_options_;
+  const FileOptions file_options_;
  const MutableCFOptions mutable_cf_options_;

  // A version number that uniquely represents this version. This is
  // used for debugging and logging purposes only.
  uint64_t version_number_;

-  Version(ColumnFamilyData* cfd, VersionSet* vset, const EnvOptions& env_opt,
+  Version(ColumnFamilyData* cfd, VersionSet* vset, const FileOptions& file_opt,
          MutableCFOptions mutable_cf_options, uint64_t version_number = 0);

  ~Version();
@@ -801,7 +802,7 @@ class AtomicGroupReadBuffer {
 class VersionSet {
 public:
  VersionSet(const std::string& dbname, const ImmutableDBOptions* db_options,
-             const EnvOptions& env_options, Cache* table_cache,
+             const FileOptions& file_options, Cache* table_cache,
             WriteBufferManager* write_buffer_manager,
             WriteController* write_controller,
             BlockCacheTracer* const block_cache_tracer);
@@ -863,7 +864,8 @@ class VersionSet {
      bool new_descriptor_log = false,
      const ColumnFamilyOptions* new_cf_options = nullptr);

-  static Status GetCurrentManifestPath(const std::string& dbname, Env* env,
+  static Status GetCurrentManifestPath(const std::string& dbname,
+                                       FileSystem* fs,
                                       std::string* manifest_filename,
                                       uint64_t* manifest_file_number);

@@ -876,7 +878,7 @@ class VersionSet {
  // Reads a manifest file and returns a list of column families in
  // column_families.
  static Status ListColumnFamilies(std::vector<std::string>* column_families,
-                                   const std::string& dbname, Env* env);
+                                   const std::string& dbname, FileSystem* fs);

 #ifndef ROCKSDB_LITE
  // Try to reduce the number of levels. This call is valid when
@@ -890,7 +892,7 @@ class VersionSet {
  // among [4-6] contains files.
  static Status ReduceNumberOfLevels(const std::string& dbname,
                                     const Options* options,
-                                     const EnvOptions& env_options,
+                                     const FileOptions& file_options,
                                     int new_levels);

  // printf contents (for debugging)
@@ -1004,7 +1006,7 @@ class VersionSet {
  // The caller should delete the iterator when no longer needed.
  InternalIterator* MakeInputIterator(
      const Compaction* c, RangeDelAggregator* range_del_agg,
-      const EnvOptions& env_options_compactions);
+      const FileOptions& file_options_compactions);

  // Add all files listed in any live version to *live.
  void AddLiveFiles(std::vector<FileDescriptor>* live_list);
@@ -1037,9 +1039,9 @@ class VersionSet {
                        uint64_t min_pending_output);

  ColumnFamilySet* GetColumnFamilySet() { return column_family_set_.get(); }
-  const EnvOptions& env_options() { return env_options_; }
-  void ChangeEnvOptions(const MutableDBOptions& new_options) {
-    env_options_.writable_file_max_buffer_size =
+  const FileOptions& file_options() { return file_options_; }
+  void ChangeFileOptions(const MutableDBOptions& new_options) {
+    file_options_.writable_file_max_buffer_size =
        new_options.writable_file_max_buffer_size;
  }

@@ -1073,8 +1075,14 @@ class VersionSet {
                           const Slice& start, const Slice& end,
                           TableReaderCaller caller);

+  struct MutableCFState {
+    uint64_t log_number;
+  };
+
  // Save current contents to *log
-  Status WriteCurrentStateToManifest(log::Writer* log);
+  Status WriteCurrentStateToManifest(
+      const std::unordered_map<uint32_t, MutableCFState>& curr_state,
+      log::Writer* log);

  void AppendVersion(ColumnFamilyData* column_family_data, Version* v);

@@ -1106,6 +1114,7 @@ class VersionSet {
  std::unique_ptr<ColumnFamilySet> column_family_set_;

  Env* const env_;
+  FileSystem* const fs_;
  const std::string dbname_;
  std::string db_id_;
  const ImmutableDBOptions* const db_options_;
@@ -1150,7 +1159,7 @@ class VersionSet {
  std::vector<std::string> obsolete_manifests_;

  // env options for all reads and writes except compactions
-  EnvOptions env_options_;
+  FileOptions file_options_;

  BlockCacheTracer* const block_cache_tracer_;

@@ -1174,7 +1183,7 @@ class ReactiveVersionSet : public VersionSet {
 public:
  ReactiveVersionSet(const std::string& dbname,
                     const ImmutableDBOptions* _db_options,
-                     const EnvOptions& _env_options, Cache* table_cache,
+                     const FileOptions& _file_options, Cache* table_cache,
                     WriteBufferManager* write_buffer_manager,
                     WriteController* write_controller);

@@ -610,21 +610,25 @@ class VersionSetTestBase {

  VersionSetTestBase()
      : env_(Env::Default()),
+        fs_(std::make_shared<LegacyFileSystemWrapper>(env_)),
        dbname_(test::PerThreadDBPath("version_set_test")),
        db_options_(),
        mutable_cf_options_(cf_options_),
        table_cache_(NewLRUCache(50000, 16)),
        write_buffer_manager_(db_options_.db_write_buffer_size),
-        versions_(new VersionSet(dbname_, &db_options_, env_options_,
-                                 table_cache_.get(), &write_buffer_manager_,
-                                 &write_controller_,
-                                 /*block_cache_tracer=*/nullptr)),
-        reactive_versions_(std::make_shared<ReactiveVersionSet>(
-            dbname_, &db_options_, env_options_, table_cache_.get(),
-            &write_buffer_manager_, &write_controller_)),
        shutting_down_(false),
        mock_table_factory_(std::make_shared<mock::MockTableFactory>()) {
    EXPECT_OK(env_->CreateDirIfMissing(dbname_));
+
+    db_options_.env = env_;
+    db_options_.fs = fs_;
+    versions_.reset(new VersionSet(dbname_, &db_options_, env_options_,
+                                   table_cache_.get(), &write_buffer_manager_,
+                                   &write_controller_,
+                                   /*block_cache_tracer=*/nullptr)),
+        reactive_versions_ = std::make_shared<ReactiveVersionSet>(
+            dbname_, &db_options_, env_options_, table_cache_.get(),
+            &write_buffer_manager_, &write_controller_);
    db_options_.db_paths.emplace_back(dbname_,
                                      std::numeric_limits<uint64_t>::max());
  }
@@ -669,8 +673,8 @@ class VersionSetTestBase {
    Status s = env_->NewWritableFile(
        manifest, &file, env_->OptimizeForManifestWrite(env_options_));
    ASSERT_OK(s);
-    std::unique_ptr<WritableFileWriter> file_writer(
-        new WritableFileWriter(std::move(file), manifest, env_options_));
+    std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
+        NewLegacyWritableFileWrapper(std::move(file)), manifest, env_options_));
    {
      log_writer->reset(new log::Writer(std::move(file_writer), 0, false));
      std::string record;
@@ -709,6 +713,7 @@ class VersionSetTestBase {
  }

  Env* env_;
+  std::shared_ptr<FileSystem> fs_;
  const std::string dbname_;
  EnvOptions env_options_;
  ImmutableDBOptions db_options_;
@@ -120,7 +120,7 @@ Status WalManager::GetUpdatesSince(
    return s;
  }
  iter->reset(new TransactionLogIteratorImpl(
-      db_options_.wal_dir, &db_options_, read_options, env_options_, seq,
+      db_options_.wal_dir, &db_options_, read_options, file_options_, seq,
      std::move(wal_files), version_set, seq_per_batch_));
  return (*iter)->status();
 }
@@ -464,9 +464,10 @@ Status WalManager::ReadFirstLine(const std::string& fname,
    }
  };

-  std::unique_ptr<SequentialFile> file;
-  Status status = env_->NewSequentialFile(
-      fname, &file, env_->OptimizeForLogRead(env_options_));
+  std::unique_ptr<FSSequentialFile> file;
+  Status status = fs_->NewSequentialFile(fname,
+                                         fs_->OptimizeForLogRead(file_options_),
+                                         &file, nullptr);
  std::unique_ptr<SequentialFileReader> file_reader(
      new SequentialFileReader(std::move(file), fname));

@@ -36,10 +36,11 @@ namespace rocksdb {
 class WalManager {
 public:
  WalManager(const ImmutableDBOptions& db_options,
-             const EnvOptions& env_options, const bool seq_per_batch = false)
+             const FileOptions& file_options, const bool seq_per_batch = false)
      : db_options_(db_options),
-        env_options_(env_options),
+        file_options_(file_options),
        env_(db_options.env),
+        fs_(db_options.fs.get()),
        purge_wal_files_last_run_(0),
        seq_per_batch_(seq_per_batch),
        wal_in_db_path_(IsWalDirSameAsDBPath(&db_options)) {}
@@ -88,8 +89,9 @@ class WalManager {

  // ------- state from DBImpl ------
  const ImmutableDBOptions& db_options_;
-  const EnvOptions& env_options_;
+  const FileOptions file_options_;
  Env* env_;
+  FileSystem* fs_;

  // ------- WalManager state -------
  // cache for ReadFirstRecord() calls
@@ -47,6 +47,8 @@ class WalManagerTest : public testing::Test {
                                      std::numeric_limits<uint64_t>::max());
    db_options_.wal_dir = dbname_;
    db_options_.env = env_.get();
+    fs_.reset(new LegacyFileSystemWrapper(env_.get()));
+    db_options_.fs = fs_;

    versions_.reset(new VersionSet(dbname_, &db_options_, env_options_,
                                   table_cache_.get(), &write_buffer_manager_,
@@ -79,8 +81,8 @@ class WalManagerTest : public testing::Test {
    std::string fname = ArchivedLogFileName(dbname_, current_log_number_);
    std::unique_ptr<WritableFile> file;
    ASSERT_OK(env_->NewWritableFile(fname, &file, env_options_));
-    std::unique_ptr<WritableFileWriter> file_writer(
-        new WritableFileWriter(std::move(file), fname, env_options_));
+    std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
+        NewLegacyWritableFileWrapper(std::move(file)), fname, env_options_));
    current_log_writer_.reset(new log::Writer(std::move(file_writer), 0, false));
  }

@@ -111,6 +113,7 @@ class WalManagerTest : public testing::Test {
  WriteBufferManager write_buffer_manager_;
  std::unique_ptr<VersionSet> versions_;
  std::unique_ptr<WalManager> wal_manager_;
+  std::shared_ptr<LegacyFileSystemWrapper> fs_;

  std::unique_ptr<log::Writer> current_log_writer_;
  uint64_t current_log_number_;
@@ -130,8 +133,8 @@ TEST_F(WalManagerTest, ReadFirstRecordCache) {
      wal_manager_->TEST_ReadFirstRecord(kAliveLogFile, 1 /* number */, &s));
  ASSERT_EQ(s, 0U);

-  std::unique_ptr<WritableFileWriter> file_writer(
-      new WritableFileWriter(std::move(file), path, EnvOptions()));
+  std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
+      NewLegacyWritableFileWrapper(std::move(file)), path, EnvOptions()));
  log::Writer writer(std::move(file_writer), 1,
                     db_options_.recycle_log_file_num > 0);
  WriteBatch batch;
@@ -1784,14 +1784,28 @@ class MemTableInserter : public WriteBatch::Handler {
    // check if memtable_list size exceeds max_write_buffer_size_to_maintain
    if (trim_history_scheduler_ != nullptr) {
      auto* cfd = cf_mems_->current();
-      assert(cfd != nullptr);
-      if (cfd->ioptions()->max_write_buffer_size_to_maintain > 0 &&
-          cfd->mem()->ApproximateMemoryUsageFast() +
-                  cfd->imm()->ApproximateMemoryUsageExcludingLast() >=
-              static_cast<size_t>(
-                  cfd->ioptions()->max_write_buffer_size_to_maintain) &&
-          cfd->imm()->MarkTrimHistoryNeeded()) {
-        trim_history_scheduler_->ScheduleWork(cfd);
+
+      assert(cfd);
+      assert(cfd->ioptions());
+
+      const size_t size_to_maintain = static_cast<size_t>(
+          cfd->ioptions()->max_write_buffer_size_to_maintain);
+
+      if (size_to_maintain > 0) {
+        MemTableList* const imm = cfd->imm();
+        assert(imm);
+
+        if (imm->HasHistory()) {
+          const MemTable* const mem = cfd->mem();
+          assert(mem);
+
+          if (mem->ApproximateMemoryUsageFast() +
+                      imm->ApproximateMemoryUsageExcludingLast() >=
+                  size_to_maintain &&
+              imm->MarkTrimHistoryNeeded()) {
+            trim_history_scheduler_->ScheduleWork(cfd);
+          }
+        }
      }
    }
  }
@@ -175,8 +175,8 @@ class BatchedOpsStressTest : public StressTest {
      const std::vector<int64_t>& rand_keys) override {
    size_t num_keys = rand_keys.size();
    std::vector<Status> ret_status(num_keys);
-    std::array<std::string, 10> keys = {"0", "1", "2", "3", "4",
-                                        "5", "6", "7", "8", "9"};
+    std::array<std::string, 10> keys = {{"0", "1", "2", "3", "4",
+                                         "5", "6", "7", "8", "9"}};
    size_t num_prefixes = keys.size();
    for (size_t rand_key = 0; rand_key < num_keys; ++rand_key) {
      std::vector<Slice> key_slices;
@@ -325,6 +325,7 @@ class BatchedOpsStressTest : public StressTest {
    if (s.ok()) {
      thread->stats.AddPrefixes(1, count);
    } else {
+      fprintf(stderr, "TestPrefixScan error: %s\n", s.ToString().c_str());
      thread->stats.AddErrors(1);
    }

@@ -15,7 +15,7 @@ class CfConsistencyStressTest : public StressTest {
 public:
  CfConsistencyStressTest() : batch_id_(0) {}

-  virtual ~CfConsistencyStressTest() {}
+  ~CfConsistencyStressTest() override {}

  Status TestPut(ThreadState* thread, WriteOptions& write_opts,
                 const ReadOptions& /* read_opts */,
@@ -184,6 +184,7 @@ class CfConsistencyStressTest : public StressTest {
      db_->ReleaseSnapshot(snapshot);
    }
    if (!is_consistent) {
+      fprintf(stderr, "TestGet error: is_consistent is false\n");
      thread->stats.AddErrors(1);
      // Fail fast to preserve the DB state.
      thread->shared->SetVerificationFailure();
@@ -192,6 +193,7 @@ class CfConsistencyStressTest : public StressTest {
    } else if (s.IsNotFound()) {
      thread->stats.AddGets(1, 0);
    } else {
+      fprintf(stderr, "TestGet error: %s\n", s.ToString().c_str());
      thread->stats.AddErrors(1);
    }
    return s;
@@ -225,6 +227,7 @@ class CfConsistencyStressTest : public StressTest {
        thread->stats.AddGets(1, 0);
      } else {
        // errors case
+        fprintf(stderr, "MultiGet error: %s\n", s.ToString().c_str());
        thread->stats.AddErrors(1);
      }
    }
@@ -263,6 +266,7 @@ class CfConsistencyStressTest : public StressTest {
    if (s.ok()) {
      thread->stats.AddPrefixes(1, count);
    } else {
+      fprintf(stderr, "TestPrefixScan error: %s\n", s.ToString().c_str());
      thread->stats.AddErrors(1);
    }
    delete iter;
@@ -292,7 +296,12 @@ class CfConsistencyStressTest : public StressTest {
                        const std::vector<int64_t>& /* rand_keys */) override {
    std::string checkpoint_dir =
        FLAGS_db + "/.checkpoint" + ToString(thread->tid);
-    DestroyDB(checkpoint_dir, options_);
+
+    // We need to clear DB including manifest files, so make a copy
+    Options opt_copy = options_;
+    opt_copy.env = db_stress_env->target();
+    DestroyDB(checkpoint_dir, opt_copy);
+
    Checkpoint* checkpoint = nullptr;
    Status s = Checkpoint::Create(db_, &checkpoint);
    if (s.ok()) {
@@ -326,7 +335,7 @@ class CfConsistencyStressTest : public StressTest {
      delete checkpoint_db;
      checkpoint_db = nullptr;
    }
-    DestroyDB(checkpoint_dir, options_);
+    DestroyDB(checkpoint_dir, opt_copy);
    if (!s.ok()) {
      fprintf(stderr, "A checkpoint operation failed with: %s\n",
              s.ToString().c_str());
@@ -345,6 +354,12 @@ class CfConsistencyStressTest : public StressTest {
    // iterate the memtable using this iterator any more, although the memtable
    // contains the most up-to-date key-values.
    options.total_order_seek = true;
+    const auto ss_deleter = [this](const Snapshot* ss) {
+      db_->ReleaseSnapshot(ss);
+    };
+    std::unique_ptr<const Snapshot, decltype(ss_deleter)> snapshot_guard(
+        db_->GetSnapshot(), ss_deleter);
+    options.snapshot = snapshot_guard.get();
    assert(thread != nullptr);
    auto shared = thread->shared;
    std::vector<std::unique_ptr<Iterator>> iters(column_families_.size());
@@ -383,9 +398,6 @@ class CfConsistencyStressTest : public StressTest {
            shared->SetVerificationFailure();
          }
        }
-        if (status.ok()) {
-          fprintf(stdout, "Finished scanning all column families.\n");
-        }
        break;
      } else if (valid_cnt != iters.size()) {
        shared->SetVerificationFailure();
@@ -486,6 +498,67 @@ class CfConsistencyStressTest : public StressTest {
    } while (true);
  }

+#ifndef ROCKSDB_LITE
+  void ContinuouslyVerifyDb(ThreadState* thread) const override {
+    assert(thread);
+    Status status;
+
+    DB* db_ptr = cmp_db_ ? cmp_db_ : db_;
+    const auto& cfhs = cmp_db_ ? cmp_cfhs_ : column_families_;
+    const auto ss_deleter = [&](const Snapshot* ss) {
+      db_ptr->ReleaseSnapshot(ss);
+    };
+    std::unique_ptr<const Snapshot, decltype(ss_deleter)> snapshot_guard(
+        db_ptr->GetSnapshot(), ss_deleter);
+    if (cmp_db_) {
+      status = cmp_db_->TryCatchUpWithPrimary();
+    }
+    SharedState* shared = thread->shared;
+    assert(shared);
+    if (!status.ok()) {
+      shared->SetShouldStopTest();
+      return;
+    }
+    assert(cmp_db_ || snapshot_guard.get());
+    const auto checksum_column_family = [](Iterator* iter,
+                                           uint32_t* checksum) -> Status {
+      assert(nullptr != checksum);
+      uint32_t ret = 0;
+      for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+        ret = crc32c::Extend(ret, iter->key().data(), iter->key().size());
+        ret = crc32c::Extend(ret, iter->value().data(), iter->value().size());
+      }
+      *checksum = ret;
+      return iter->status();
+    };
+    ReadOptions ropts;
+    ropts.total_order_seek = true;
+    ropts.snapshot = snapshot_guard.get();
+    uint32_t crc = 0;
+    {
+      // Compute crc for all key-values of default column family.
+      std::unique_ptr<Iterator> it(db_ptr->NewIterator(ropts));
+      status = checksum_column_family(it.get(), &crc);
+    }
+    uint32_t tmp_crc = 0;
+    if (status.ok()) {
+      for (ColumnFamilyHandle* cfh : cfhs) {
+        if (cfh == db_ptr->DefaultColumnFamily()) {
+          continue;
+        }
+        std::unique_ptr<Iterator> it(db_ptr->NewIterator(ropts, cfh));
+        status = checksum_column_family(it.get(), &tmp_crc);
+        if (!status.ok() || tmp_crc != crc) {
+          break;
+        }
+      }
+    }
+    if (!status.ok() || tmp_crc != crc) {
+      shared->SetShouldStopTest();
+    }
+  }
+#endif  // !ROCKSDB_LITE
+
  std::vector<int> GenerateColumnFamilies(
      const int /* num_column_families */,
      int /* rand_column_family */) const override {
@@ -10,14 +10,74 @@

 #ifdef GFLAGS
 #include "db_stress_tool/db_stress_common.h"
+#include <cmath>

-rocksdb::Env* FLAGS_env = rocksdb::Env::Default();
-enum rocksdb::CompressionType FLAGS_compression_type_e =
+rocksdb::DbStressEnvWrapper* db_stress_env = nullptr;
+enum rocksdb::CompressionType compression_type_e = rocksdb::kSnappyCompression;
+enum rocksdb::CompressionType bottommost_compression_type_e =
    rocksdb::kSnappyCompression;
-enum rocksdb::ChecksumType FLAGS_checksum_type_e = rocksdb::kCRC32c;
+enum rocksdb::ChecksumType checksum_type_e = rocksdb::kCRC32c;
 enum RepFactory FLAGS_rep_factory = kSkipList;
+std::vector<double> sum_probs(100001);
+int64_t zipf_sum_size = 100000;

 namespace rocksdb {
+
+// Zipfian distribution is generated based on a pre-calculated array.
+// It should be used before start the stress test.
+// First, the probability distribution function (PDF) of this Zipfian follows
+// power low. P(x) = 1/(x^alpha).
+// So we calculate the PDF when x is from 0 to zipf_sum_size in first for loop
+// and add the PDF value togetger as c. So we get the total probability in c.
+// Next, we calculate inverse CDF of Zipfian and store the value of each in
+// an array (sum_probs). The rank is from 0 to zipf_sum_size. For example, for
+// integer k, its Zipfian CDF value is sum_probs[k].
+// Third, when we need to get an integer whose probability follows Zipfian
+// distribution, we use a rand_seed [0,1] which follows uniform distribution
+// as a seed and search it in the sum_probs via binary search. When we find
+// the closest sum_probs[i] of rand_seed, i is the integer that in
+// [0, zipf_sum_size] following Zipfian distribution with parameter alpha.
+// Finally, we can scale i to [0, max_key] scale.
+// In order to avoid that hot keys are close to each other and skew towards 0,
+// we use Rando64 to shuffle it.
+void InitializeHotKeyGenerator(double alpha) {
+  double c = 0;
+  for (int64_t i = 1; i <= zipf_sum_size; i++) {
+    c = c + (1.0 / std::pow(static_cast<double>(i), alpha));
+  }
+  c = 1.0 / c;
+
+  sum_probs[0] = 0;
+  for (int64_t i = 1; i <= zipf_sum_size; i++) {
+    sum_probs[i] =
+        sum_probs[i - 1] + c / std::pow(static_cast<double>(i), alpha);
+  }
+}
+
+// Generate one key that follows the Zipfian distribution. The skewness
+// is decided by the parameter alpha. Input is the rand_seed [0,1] and
+// the max of the key to be generated. If we directly return tmp_zipf_seed,
+// the closer to 0, the higher probability will be. To randomly distribute
+// the hot keys in [0, max_key], we use Random64 to shuffle it.
+int64_t GetOneHotKeyID(double rand_seed, int64_t max_key) {
+  int64_t low = 1, mid, high = zipf_sum_size, zipf = 0;
+  while (low <= high) {
+    mid = std::floor((low + high) / 2);
+    if (sum_probs[mid] >= rand_seed && sum_probs[mid - 1] < rand_seed) {
+      zipf = mid;
+      break;
+    } else if (sum_probs[mid] >= rand_seed) {
+      high = mid - 1;
+    } else {
+      low = mid + 1;
+    }
+  }
+  int64_t tmp_zipf_seed = static_cast<int64_t>(
+      std::floor(zipf * max_key / (static_cast<double>(zipf_sum_size))));
+  Random64 rand_local(tmp_zipf_seed);
+  return rand_local.Next() % max_key;
+}
+
 void PoolSizeChangeThread(void* v) {
  assert(FLAGS_compaction_thread_pool_adjust_interval > 0);
  ThreadState* thread = reinterpret_cast<ThreadState*>(v);
@@ -26,9 +86,11 @@ void PoolSizeChangeThread(void* v) {
  while (true) {
    {
      MutexLock l(shared->GetMutex());
-      if (shared->ShoudStopBgThread()) {
-        shared->SetBgThreadFinish();
-        shared->GetCondVar()->SignalAll();
+      if (shared->ShouldStopBgThread()) {
+        shared->IncBgThreadsFinished();
+        if (shared->BgThreadsFinished()) {
+          shared->GetCondVar()->SignalAll();
+        }
        return;
      }
    }
@@ -41,15 +103,42 @@ void PoolSizeChangeThread(void* v) {
    if (new_thread_pool_size < 1) {
      new_thread_pool_size = 1;
    }
-    FLAGS_env->SetBackgroundThreads(new_thread_pool_size);
+    db_stress_env->SetBackgroundThreads(new_thread_pool_size,
+                                        rocksdb::Env::Priority::LOW);
    // Sleep up to 3 seconds
-    FLAGS_env->SleepForMicroseconds(
+    db_stress_env->SleepForMicroseconds(
        thread->rand.Next() % FLAGS_compaction_thread_pool_adjust_interval *
            1000 +
        1);
  }
 }

+void DbVerificationThread(void* v) {
+  assert(FLAGS_continuous_verification_interval > 0);
+  auto* thread = reinterpret_cast<ThreadState*>(v);
+  SharedState* shared = thread->shared;
+  StressTest* stress_test = shared->GetStressTest();
+  assert(stress_test != nullptr);
+  while (true) {
+    {
+      MutexLock l(shared->GetMutex());
+      if (shared->ShouldStopBgThread()) {
+        shared->IncBgThreadsFinished();
+        if (shared->BgThreadsFinished()) {
+          shared->GetCondVar()->SignalAll();
+        }
+        return;
+      }
+    }
+    if (!shared->HasVerificationFailedYet()) {
+      stress_test->ContinuouslyVerifyDb(thread);
+    }
+    db_stress_env->SleepForMicroseconds(
+        thread->rand.Next() % FLAGS_continuous_verification_interval * 1000 +
+        1);
+  }
+}
+
 void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz) {
  if (!FLAGS_verbose) {
    return;
@@ -65,14 +154,33 @@ void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz) {
          key, sz, tmp.c_str());
 }

+// Note that if hot_key_alpha != 0, it generates the key based on Zipfian
+// distribution. Keys are randomly scattered to [0, FLAGS_max_key]. It does
+// not ensure the order of the keys being generated and the keys does not have
+// the active range which is related to FLAGS_active_width.
 int64_t GenerateOneKey(ThreadState* thread, uint64_t iteration) {
  const double completed_ratio =
      static_cast<double>(iteration) / FLAGS_ops_per_thread;
  const int64_t base_key = static_cast<int64_t>(
      completed_ratio * (FLAGS_max_key - FLAGS_active_width));
-  return base_key + thread->rand.Next() % FLAGS_active_width;
+  int64_t rand_seed = base_key + thread->rand.Next() % FLAGS_active_width;
+  int64_t cur_key = rand_seed;
+  if (FLAGS_hot_key_alpha != 0) {
+    // If set the Zipfian distribution Alpha to non 0, use Zipfian
+    double float_rand =
+        (static_cast<double>(thread->rand.Next() % FLAGS_max_key)) /
+        FLAGS_max_key;
+    cur_key = GetOneHotKeyID(float_rand, FLAGS_max_key);
+  }
+  return cur_key;
 }

+// Note that if hot_key_alpha != 0, it generates the key based on Zipfian
+// distribution. Keys being generated are in random order.
+// If user want to generate keys based on uniform distribution, user needs to
+// set hot_key_alpha == 0. It will generate the random keys in increasing
+// order in the key array (ensure key[i] >= key[i+1]) and constrained in a
+// range related to FLAGS_active_width.
 std::vector<int64_t> GenerateNKeys(ThreadState* thread, int num_keys,
                                   uint64_t iteration) {
  const double completed_ratio =
@@ -84,9 +192,17 @@ std::vector<int64_t> GenerateNKeys(ThreadState* thread, int num_keys,
  int64_t next_key = base_key + thread->rand.Next() % FLAGS_active_width;
  keys.push_back(next_key);
  for (int i = 1; i < num_keys; ++i) {
-    // This may result in some duplicate keys
-    next_key = next_key + thread->rand.Next() %
-                              (FLAGS_active_width - (next_key - base_key));
+    // Generate the key follows zipfian distribution
+    if (FLAGS_hot_key_alpha != 0) {
+      double float_rand =
+          (static_cast<double>(thread->rand.Next() % FLAGS_max_key)) /
+          FLAGS_max_key;
+      next_key = GetOneHotKeyID(float_rand, FLAGS_max_key);
+    } else {
+      // This may result in some duplicate keys
+      next_key = next_key + thread->rand.Next() %
+                                (FLAGS_active_width - (next_key - base_key));
+    }
    keys.push_back(next_key);
  }
  return keys;
@@ -36,6 +36,7 @@

 #include "db/db_impl/db_impl.h"
 #include "db/version_set.h"
+#include "db_stress_tool/db_stress_env_wrapper.h"
 #include "db_stress_tool/db_stress_listener.h"
 #include "db_stress_tool/db_stress_shared_state.h"
 #include "db_stress_tool/db_stress_test_base.h"
@@ -64,6 +65,7 @@
 #include "util/mutexlock.h"
 #include "util/random.h"
 #include "util/string_util.h"
+#include "utilities/blob_db/blob_db.h"
 // SyncPoint is not supported in Released Windows Mode.
 #if !(defined NDEBUG) || !defined(OS_WIN)
 #include "test_util/sync_point.h"
@@ -79,6 +81,7 @@ using GFLAGS_NAMESPACE::SetUsageMessage;
 DECLARE_uint64(seed);
 DECLARE_bool(read_only);
 DECLARE_int64(max_key);
+DECLARE_double(hot_key_alpha);
 DECLARE_int32(column_families);
 DECLARE_string(options_file);
 DECLARE_int64(active_width);
@@ -122,6 +125,7 @@ DECLARE_int32(universal_min_merge_width);
 DECLARE_int32(universal_max_merge_width);
 DECLARE_int32(universal_max_size_amplification_percent);
 DECLARE_int32(clear_column_family_one_in);
+DECLARE_int32(get_live_files_and_wal_files_one_in);
 DECLARE_int32(set_options_one_in);
 DECLARE_int32(set_in_place_one_in);
 DECLARE_int64(cache_size);
@@ -139,7 +143,7 @@ DECLARE_bool(partition_filters);
 DECLARE_int32(index_type);
 DECLARE_string(db);
 DECLARE_string(secondaries_base);
-DECLARE_bool(enable_secondary);
+DECLARE_bool(test_secondary);
 DECLARE_string(expected_values_path);
 DECLARE_bool(verify_checksum);
 DECLARE_bool(mmap_read);
@@ -161,6 +165,8 @@ DECLARE_int32(range_deletion_width);
 DECLARE_uint64(rate_limiter_bytes_per_sec);
 DECLARE_bool(rate_limit_bg_reads);
 DECLARE_bool(use_txn);
+DECLARE_uint64(txn_write_policy);
+DECLARE_bool(unordered_write);
 DECLARE_int32(backup_one_in);
 DECLARE_int32(checkpoint_one_in);
 DECLARE_int32(ingest_external_file_one_in);
@@ -173,6 +179,7 @@ DECLARE_int32(compact_range_width);
 DECLARE_int32(acquire_snapshot_one_in);
 DECLARE_bool(compare_full_db_state_snapshot);
 DECLARE_uint64(snapshot_hold_ops);
+DECLARE_bool(long_running_snapshots);
 DECLARE_bool(use_multiget);
 DECLARE_int32(readpercent);
 DECLARE_int32(prefixpercent);
@@ -183,6 +190,7 @@ DECLARE_int32(nooverwritepercent);
 DECLARE_int32(iterpercent);
 DECLARE_uint64(num_iterations);
 DECLARE_string(compression_type);
+DECLARE_string(bottommost_compression_type);
 DECLARE_int32(compression_max_dict_bytes);
 DECLARE_int32(compression_zstd_max_train_bytes);
 DECLARE_string(checksum_type);
@@ -198,16 +206,34 @@ DECLARE_int32(prefix_size);
 DECLARE_bool(use_merge);
 DECLARE_bool(use_full_merge_v1);
 DECLARE_int32(sync_wal_one_in);
+DECLARE_bool(avoid_unnecessary_blocking_io);
+DECLARE_bool(write_dbid_to_manifest);
+DECLARE_uint64(max_write_batch_group_size_bytes);
+DECLARE_bool(level_compaction_dynamic_level_bytes);
+DECLARE_int32(verify_checksum_one_in);
+DECLARE_int32(verify_db_one_in);
+DECLARE_int32(continuous_verification_interval);
+
+#ifndef ROCKSDB_LITE
+DECLARE_bool(use_blob_db);
+DECLARE_uint64(blob_db_min_blob_size);
+DECLARE_uint64(blob_db_bytes_per_sync);
+DECLARE_uint64(blob_db_file_size);
+DECLARE_bool(blob_db_enable_gc);
+DECLARE_double(blob_db_gc_cutoff);
+#endif  // !ROCKSDB_LITE
+DECLARE_int32(approximate_size_one_in);

 const long KB = 1024;
 const int kRandomValueMaxFactor = 3;
 const int kValueMaxLen = 100;

-// posix or hdfs environment
-extern rocksdb::Env* FLAGS_env;
+// wrapped posix or hdfs environment
+extern rocksdb::DbStressEnvWrapper* db_stress_env;

-extern enum rocksdb::CompressionType FLAGS_compression_type_e;
-extern enum rocksdb::ChecksumType FLAGS_checksum_type_e;
+extern enum rocksdb::CompressionType compression_type_e;
+extern enum rocksdb::CompressionType bottommost_compression_type_e;
+extern enum rocksdb::ChecksumType checksum_type_e;

 enum RepFactory { kSkipList, kHashSkipList, kVectorRep };

@@ -232,25 +258,37 @@ inline enum rocksdb::CompressionType StringToCompressionType(
    const char* ctype) {
  assert(ctype);

-  if (!strcasecmp(ctype, "none"))
-    return rocksdb::kNoCompression;
-  else if (!strcasecmp(ctype, "snappy"))
-    return rocksdb::kSnappyCompression;
-  else if (!strcasecmp(ctype, "zlib"))
-    return rocksdb::kZlibCompression;
-  else if (!strcasecmp(ctype, "bzip2"))
-    return rocksdb::kBZip2Compression;
-  else if (!strcasecmp(ctype, "lz4"))
-    return rocksdb::kLZ4Compression;
-  else if (!strcasecmp(ctype, "lz4hc"))
-    return rocksdb::kLZ4HCCompression;
-  else if (!strcasecmp(ctype, "xpress"))
-    return rocksdb::kXpressCompression;
-  else if (!strcasecmp(ctype, "zstd"))
-    return rocksdb::kZSTD;
+  rocksdb::CompressionType ret_compression_type;

-  fprintf(stderr, "Cannot parse compression type '%s'\n", ctype);
-  return rocksdb::kSnappyCompression;  // default value
+  if (!strcasecmp(ctype, "disable")) {
+    ret_compression_type = rocksdb::kDisableCompressionOption;
+  } else if (!strcasecmp(ctype, "none")) {
+    ret_compression_type = rocksdb::kNoCompression;
+  } else if (!strcasecmp(ctype, "snappy")) {
+    ret_compression_type = rocksdb::kSnappyCompression;
+  } else if (!strcasecmp(ctype, "zlib")) {
+    ret_compression_type = rocksdb::kZlibCompression;
+  } else if (!strcasecmp(ctype, "bzip2")) {
+    ret_compression_type = rocksdb::kBZip2Compression;
+  } else if (!strcasecmp(ctype, "lz4")) {
+    ret_compression_type = rocksdb::kLZ4Compression;
+  } else if (!strcasecmp(ctype, "lz4hc")) {
+    ret_compression_type = rocksdb::kLZ4HCCompression;
+  } else if (!strcasecmp(ctype, "xpress")) {
+    ret_compression_type = rocksdb::kXpressCompression;
+  } else if (!strcasecmp(ctype, "zstd")) {
+    ret_compression_type = rocksdb::kZSTD;
+  } else {
+    fprintf(stderr, "Cannot parse compression type '%s'\n", ctype);
+    ret_compression_type = rocksdb::kSnappyCompression;  // default value
+  }
+  if (ret_compression_type != rocksdb::kDisableCompressionOption &&
+      !CompressionTypeSupported(ret_compression_type)) {
+    // Use no compression will be more portable but considering this is
+    // only a stress test and snappy is widely available. Use snappy here.
+    ret_compression_type = rocksdb::kSnappyCompression;
+  }
+  return ret_compression_type;
 }

 inline enum rocksdb::ChecksumType StringToChecksumType(const char* ctype) {
@@ -357,6 +395,8 @@ extern inline void SanitizeDoubleParam(double* param) {

 extern void PoolSizeChangeThread(void* v);

+extern void DbVerificationThread(void* v);
+
 extern void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz);

 extern int64_t GenerateOneKey(ThreadState* thread, uint64_t iteration);
@@ -369,5 +409,7 @@ extern size_t GenerateValue(uint32_t rand, char* v, size_t max_sz);
 extern StressTest* CreateCfConsistencyStressTest();
 extern StressTest* CreateBatchedOpsStressTest();
 extern StressTest* CreateNonBatchedOpsStressTest();
+extern void InitializeHotKeyGenerator(double alpha);
+extern int64_t GetOneHotKeyID(double rand_seed, int64_t max_key);
 }  // namespace rocksdb
 #endif  // GFLAGS
@@ -56,24 +56,29 @@ void ThreadBody(void* v) {
 bool RunStressTest(StressTest* stress) {
  stress->InitDb();

-  SharedState shared(FLAGS_env, stress);
+  SharedState shared(db_stress_env, stress);
  if (FLAGS_read_only) {
    stress->InitReadonlyDb(&shared);
  }

  uint32_t n = shared.GetNumThreads();

-  uint64_t now = FLAGS_env->NowMicros();
+  uint64_t now = db_stress_env->NowMicros();
  fprintf(stdout, "%s Initializing worker threads\n",
-          FLAGS_env->TimeToString(now / 1000000).c_str());
+          db_stress_env->TimeToString(now / 1000000).c_str());
  std::vector<ThreadState*> threads(n);
  for (uint32_t i = 0; i < n; i++) {
    threads[i] = new ThreadState(i, &shared);
-    FLAGS_env->StartThread(ThreadBody, threads[i]);
+    db_stress_env->StartThread(ThreadBody, threads[i]);
  }
  ThreadState bg_thread(0, &shared);
  if (FLAGS_compaction_thread_pool_adjust_interval > 0) {
-    FLAGS_env->StartThread(PoolSizeChangeThread, &bg_thread);
+    db_stress_env->StartThread(PoolSizeChangeThread, &bg_thread);
+  }
+  ThreadState continuous_verification_thread(0, &shared);
+  if (FLAGS_continuous_verification_interval > 0) {
+    db_stress_env->StartThread(DbVerificationThread,
+                               &continuous_verification_thread);
  }

  // Each thread goes through the following states:
@@ -87,15 +92,15 @@ bool RunStressTest(StressTest* stress) {
    }
    if (shared.ShouldVerifyAtBeginning()) {
      if (shared.HasVerificationFailedYet()) {
-        printf("Crash-recovery verification failed :(\n");
+        fprintf(stderr, "Crash-recovery verification failed :(\n");
      } else {
-        printf("Crash-recovery verification passed :)\n");
+        fprintf(stdout, "Crash-recovery verification passed :)\n");
      }
    }

-    now = FLAGS_env->NowMicros();
+    now = db_stress_env->NowMicros();
    fprintf(stdout, "%s Starting database operations\n",
-            FLAGS_env->TimeToString(now / 1000000).c_str());
+            db_stress_env->TimeToString(now / 1000000).c_str());

    shared.SetStart();
    shared.GetCondVar()->SignalAll();
@@ -103,13 +108,13 @@ bool RunStressTest(StressTest* stress) {
      shared.GetCondVar()->Wait();
    }

-    now = FLAGS_env->NowMicros();
+    now = db_stress_env->NowMicros();
    if (FLAGS_test_batches_snapshots) {
      fprintf(stdout, "%s Limited verification already done during gets\n",
-              FLAGS_env->TimeToString((uint64_t)now / 1000000).c_str());
+              db_stress_env->TimeToString((uint64_t)now / 1000000).c_str());
    } else {
      fprintf(stdout, "%s Starting verification\n",
-              FLAGS_env->TimeToString((uint64_t)now / 1000000).c_str());
+              db_stress_env->TimeToString((uint64_t)now / 1000000).c_str());
    }

    shared.SetStartVerify();
@@ -128,17 +133,18 @@ bool RunStressTest(StressTest* stress) {
    delete threads[i];
    threads[i] = nullptr;
  }
-  now = FLAGS_env->NowMicros();
+  now = db_stress_env->NowMicros();
  if (!FLAGS_test_batches_snapshots && !shared.HasVerificationFailedYet()) {
    fprintf(stdout, "%s Verification successful\n",
-            FLAGS_env->TimeToString(now / 1000000).c_str());
+            db_stress_env->TimeToString(now / 1000000).c_str());
  }
  stress->PrintStatistics();

-  if (FLAGS_compaction_thread_pool_adjust_interval > 0) {
+  if (FLAGS_compaction_thread_pool_adjust_interval > 0 ||
+      FLAGS_continuous_verification_interval > 0) {
    MutexLock l(shared.GetMutex());
    shared.SetShouldStopBgThread();
-    while (!shared.BgThreadFinished()) {
+    while (!shared.BgThreadsFinished()) {
      shared.GetCondVar()->Wait();
    }
  }
@@ -148,7 +154,7 @@ bool RunStressTest(StressTest* stress) {
  }

  if (shared.HasVerificationFailedYet()) {
-    printf("Verification failed :(\n");
+    fprintf(stderr, "Verification failed :(\n");
    return false;
  }
  return true;
@@ -0,0 +1,36 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifdef GFLAGS
+#pragma once
+#include "db_stress_tool/db_stress_common.h"
+
+namespace rocksdb {
+class DbStressEnvWrapper : public EnvWrapper {
+ public:
+  explicit DbStressEnvWrapper(Env* t) : EnvWrapper(t) {}
+
+  Status DeleteFile(const std::string& f) override {
+    // We determine whether it is a manifest file by searching a strong,
+    // so that there will be false positive if the directory path contains the
+    // keyword but it is unlikely.
+    // Checkpoint directory needs to be exempted.
+    if (!if_preserve_all_manifests ||
+        f.find("MANIFEST-") == std::string::npos ||
+        f.find("checkpoint") != std::string::npos) {
+      return target()->DeleteFile(f);
+    }
+    return Status::OK();
+  }
+
+  // If true, all manifest files will not be delted in DeleteFile().
+  bool if_preserve_all_manifests = true;
+};
+}  // namespace rocksdb
+#endif  // GFLAGS
@@ -30,6 +30,16 @@ DEFINE_int64(max_key, 1 * KB * KB,

 DEFINE_int32(column_families, 10, "Number of column families");

+DEFINE_double(
+    hot_key_alpha, 0,
+    "Use Zipfian distribution to generate the key "
+    "distribution. If it is not specified, write path will use random "
+    "distribution to generate the keys. The parameter is [0, double_max]). "
+    "However, the larger alpha is, the more shewed will be. If alpha is "
+    "larger than 2, it is likely that only 1 key will be accessed. The "
+    "Recommended value is [0.8-1.5]. The distribution is also related to "
+    "max_key and total iterations of generating the hot key. ");
+
 DEFINE_string(
    options_file, "",
    "The path to a RocksDB options file.  If specified, then db_stress will "
@@ -229,6 +239,11 @@ DEFINE_int32(clear_column_family_one_in, 1000000,
             "it again. If N == 0, never drop/create column families. "
             "When test_batches_snapshots is true, this flag has no effect");

+DEFINE_int32(get_live_files_and_wal_files_one_in, 1000000,
+             "With a chance of 1/N, call GetLiveFiles, GetSortedWalFiles "
+             "and GetCurrentWalFile to verify if it returns correctly. If "
+             "N == 0, never call the three interfaces.");
+
 DEFINE_int32(set_options_one_in, 0,
             "With a chance of 1/N, change some random options");

@@ -260,6 +275,32 @@ DEFINE_bool(allow_concurrent_memtable_write, false,
 DEFINE_bool(enable_write_thread_adaptive_yield, true,
            "Use a yielding spin loop for brief writer thread waits.");

+#ifndef ROCKSDB_LITE
+// BlobDB Options
+DEFINE_bool(use_blob_db, false, "Use BlobDB.");
+
+DEFINE_uint64(blob_db_min_blob_size,
+              rocksdb::blob_db::BlobDBOptions().min_blob_size,
+              "Smallest blob to store in a file. Blobs smaller than this "
+              "will be inlined with the key in the LSM tree.");
+
+DEFINE_uint64(blob_db_bytes_per_sync,
+              rocksdb::blob_db::BlobDBOptions().bytes_per_sync,
+              "Sync blob files once per every N bytes written.");
+
+DEFINE_uint64(blob_db_file_size,
+              rocksdb::blob_db::BlobDBOptions().blob_file_size,
+              "Target size of each blob file.");
+
+DEFINE_bool(blob_db_enable_gc,
+            rocksdb::blob_db::BlobDBOptions().enable_garbage_collection,
+            "Enable BlobDB garbage collection.");
+
+DEFINE_double(blob_db_gc_cutoff,
+              rocksdb::blob_db::BlobDBOptions().garbage_collection_cutoff,
+              "Cutoff ratio for BlobDB garbage collection.");
+#endif  // !ROCKSDB_LITE
+
 static const bool FLAGS_subcompactions_dummy __attribute__((__unused__)) =
    RegisterFlagValidator(&FLAGS_subcompactions, &ValidateUint32Range);

@@ -297,7 +338,7 @@ DEFINE_string(db, "", "Use the db with the following name.");
 DEFINE_string(secondaries_base, "",
              "Use this path as the base path for secondary instances.");

-DEFINE_bool(enable_secondary, false, "Enable secondary instance.");
+DEFINE_bool(test_secondary, false, "Test secondary instance.");

 DEFINE_string(
    expected_values_path, "",
@@ -371,6 +412,16 @@ DEFINE_bool(use_txn, false,
            "Use TransactionDB. Currently the default write policy is "
            "TxnDBWritePolicy::WRITE_PREPARED");

+DEFINE_uint64(txn_write_policy, 0,
+              "The transaction write policy. Default is "
+              "TxnDBWritePolicy::WRITE_COMMITTED. Note that this should not be "
+              "changed accross crashes.");
+
+DEFINE_bool(unordered_write, false,
+            "Turn on the unordered_write feature. This options is currently "
+            "tested only in combination with use_txn=true and "
+            "txn_write_policy=TxnDBWritePolicy::WRITE_PREPARED.");
+
 DEFINE_int32(backup_one_in, 0,
             "If non-zero, then CreateNewBackup() will be called once for "
             "every N operations on average.  0 indicates CreateNewBackup() "
@@ -420,6 +471,9 @@ DEFINE_uint64(snapshot_hold_ops, 0,
              "If non-zero, then releases snapshots N operations after they're "
              "acquired.");

+DEFINE_bool(long_running_snapshots, false,
+            "If set, hold on some some snapshots for much longer time.");
+
 DEFINE_bool(use_multiget, false,
            "If set, use the batched MultiGet API for reads");

@@ -486,6 +540,10 @@ DEFINE_int32(compression_zstd_max_train_bytes, 0,
             "Maximum size of training data passed to zstd's dictionary "
             "trainer.");

+DEFINE_string(bottommost_compression_type, "disable",
+              "Algorithm to use to compress bottommost level of the database. "
+              "\"disable\" means disabling the feature");
+
 DEFINE_string(checksum_type, "kCRC32c", "Algorithm to use to checksum blocks");

 DEFINE_string(hdfs, "", "Name of hdfs environment");
@@ -537,4 +595,38 @@ DEFINE_bool(use_full_merge_v1, false,
 DEFINE_int32(sync_wal_one_in, 0,
             "If non-zero, then SyncWAL() will be called once for every N ops "
             "on average. 0 indicates that calls to SyncWAL() are disabled.");
+
+DEFINE_bool(avoid_unnecessary_blocking_io,
+            rocksdb::Options().avoid_unnecessary_blocking_io,
+            "If true, some expensive cleaning up operations will be moved from "
+            "user reads to high-pri background threads.");
+
+DEFINE_bool(write_dbid_to_manifest, rocksdb::Options().write_dbid_to_manifest,
+            "Write DB_ID to manifest");
+
+DEFINE_uint64(max_write_batch_group_size_bytes,
+              rocksdb::Options().max_write_batch_group_size_bytes,
+              "Max write batch group size");
+
+DEFINE_bool(level_compaction_dynamic_level_bytes,
+            rocksdb::Options().level_compaction_dynamic_level_bytes,
+            "Use dynamic level");
+
+DEFINE_int32(verify_checksum_one_in, 0,
+             "If non-zero, then DB::VerifyChecksum() will be called to do"
+             " checksum verification of all the files in the database once for"
+             " every N ops on average. 0 indicates that calls to"
+             " VerifyChecksum() are disabled.");
+DEFINE_int32(verify_db_one_in, 0,
+             "If non-zero, call VerifyDb() once for every N ops. 0 indicates "
+             "that VerifyDb() will not be called in OperateDb(). Note that "
+             "enabling this can slow down tests.");
+
+DEFINE_int32(continuous_verification_interval, 1000,
+             "While test is running, verify db every N milliseconds. 0 "
+             "disables continuous verification.");
+
+DEFINE_int32(approximate_size_one_in, 64,
+             "If non-zero, DB::GetApproximateSizes() will be called against"
+             " random key ranges.");
 #endif  // GFLAGS
@@ -21,18 +21,29 @@ class DbStressListener : public EventListener {
        db_paths_(db_paths),
        column_families_(column_families),
        num_pending_file_creations_(0) {}
-  virtual ~DbStressListener() { assert(num_pending_file_creations_ == 0); }
 #ifndef ROCKSDB_LITE
-  virtual void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
+  ~DbStressListener() override { assert(num_pending_file_creations_ == 0); }
+  void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
    assert(IsValidColumnFamilyName(info.cf_name));
    VerifyFilePath(info.file_path);
    // pretending doing some work here
-    std::this_thread::sleep_for(
-        std::chrono::microseconds(Random::GetTLSInstance()->Uniform(5000)));
+    RandomSleep();
  }

-  virtual void OnCompactionCompleted(DB* /*db*/,
-                                     const CompactionJobInfo& ci) override {
+  void OnFlushBegin(DB* /*db*/,
+                    const FlushJobInfo& /*flush_job_info*/) override {
+    RandomSleep();
+  }
+
+  void OnTableFileDeleted(const TableFileDeletionInfo& /*info*/) override {
+    RandomSleep();
+  }
+
+  void OnCompactionBegin(DB* /*db*/, const CompactionJobInfo& /*ci*/) override {
+    RandomSleep();
+  }
+
+  void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override {
    assert(IsValidColumnFamilyName(ci.cf_name));
    assert(ci.input_files.size() + ci.output_files.size() > 0U);
    for (const auto& file_path : ci.input_files) {
@@ -42,15 +53,15 @@ class DbStressListener : public EventListener {
      VerifyFilePath(file_path);
    }
    // pretending doing some work here
-    std::this_thread::sleep_for(
-        std::chrono::microseconds(Random::GetTLSInstance()->Uniform(5000)));
+    RandomSleep();
  }

-  virtual void OnTableFileCreationStarted(
+  void OnTableFileCreationStarted(
      const TableFileCreationBriefInfo& /*info*/) override {
    ++num_pending_file_creations_;
  }
-  virtual void OnTableFileCreated(const TableFileCreationInfo& info) override {
+
+  void OnTableFileCreated(const TableFileCreationInfo& info) override {
    assert(info.db_name == db_name_);
    assert(IsValidColumnFamilyName(info.cf_name));
    if (info.file_size) {
@@ -66,6 +77,70 @@ class DbStressListener : public EventListener {
    --num_pending_file_creations_;
  }

+  void OnMemTableSealed(const MemTableInfo& /*info*/) override {
+    RandomSleep();
+  }
+
+  void OnColumnFamilyHandleDeletionStarted(
+      ColumnFamilyHandle* /*handle*/) override {
+    RandomSleep();
+  }
+
+  void OnExternalFileIngested(
+      DB* /*db*/, const ExternalFileIngestionInfo& /*info*/) override {
+    RandomSleep();
+  }
+
+  void OnBackgroundError(BackgroundErrorReason /* reason */,
+                         Status* /* bg_error */) override {
+    RandomSleep();
+  }
+
+  void OnStallConditionsChanged(const WriteStallInfo& /*info*/) override {
+    RandomSleep();
+  }
+
+  void OnFileReadFinish(const FileOperationInfo& info) override {
+    // Even empty callback is valuable because sometimes some locks are
+    // released in order to make the callback.
+
+    // Sleep carefully here as it is a frequent operation and we don't want
+    // to slow down the tests. We always sleep when the read is large.
+    // When read is small, sleep in a small chance.
+    size_t length_read = info.length;
+    if (length_read >= 1000000 || Random::GetTLSInstance()->OneIn(1000)) {
+      RandomSleep();
+    }
+  }
+
+  void OnFileWriteFinish(const FileOperationInfo& info) override {
+    // Even empty callback is valuable because sometimes some locks are
+    // released in order to make the callback.
+
+    // Sleep carefully here as it is a frequent operation and we don't want
+    // to slow down the tests. When the write is large, always sleep.
+    // Otherwise, sleep in a relatively small chance.
+    size_t length_write = info.length;
+    if (length_write >= 1000000 || Random::GetTLSInstance()->OneIn(64)) {
+      RandomSleep();
+    }
+  }
+
+  bool ShouldBeNotifiedOnFileIO() override {
+    RandomSleep();
+    return static_cast<bool>(Random::GetTLSInstance()->OneIn(1));
+  }
+
+  void OnErrorRecoveryBegin(BackgroundErrorReason /* reason */,
+                            Status /* bg_error */,
+                            bool* /* auto_recovery */) override {
+    RandomSleep();
+  }
+
+  void OnErrorRecoveryCompleted(Status /* old_bg_error */) override {
+    RandomSleep();
+  }
+
 protected:
  bool IsValidColumnFamilyName(const std::string& cf_name) const {
    if (cf_name == kDefaultColumnFamilyName) {
@@ -130,6 +205,11 @@ class DbStressListener : public EventListener {
    (void)file_path;
 #endif  // !NDEBUG
  }
+
+  void RandomSleep() {
+    std::this_thread::sleep_for(
+        std::chrono::microseconds(Random::GetTLSInstance()->Uniform(5000)));
+  }
 #endif  // !ROCKSDB_LITE

 private:
@@ -13,7 +13,6 @@

 namespace rocksdb {
 const uint32_t SharedState::UNKNOWN_SENTINEL = 0xfffffffe;
-// indicates a key should definitely be deleted
 const uint32_t SharedState::DELETION_SENTINEL = 0xffffffff;
 }  // namespace rocksdb
 #endif  // GFLAGS
@@ -22,6 +22,8 @@ DECLARE_int32(nooverwritepercent);
 DECLARE_string(expected_values_path);
 DECLARE_int32(clear_column_family_one_in);
 DECLARE_bool(test_batches_snapshots);
+DECLARE_int32(compaction_thread_pool_adjust_interval);
+DECLARE_int32(continuous_verification_interval);

 namespace rocksdb {
 class StressTest;
@@ -47,17 +49,19 @@ class SharedState {
        num_done_(0),
        start_(false),
        start_verify_(false),
+        num_bg_threads_(0),
        should_stop_bg_thread_(false),
-        bg_thread_finished_(false),
+        bg_thread_finished_(0),
        stress_test_(stress_test),
        verification_failure_(false),
+        should_stop_test_(false),
        no_overwrite_ids_(FLAGS_column_families),
        values_(nullptr),
        printing_verification_results_(false) {
    // Pick random keys in each column family that will not experience
    // overwrite

-    printf("Choosing random keys with no overwrite\n");
+    fprintf(stdout, "Choosing random keys with no overwrite\n");
    Random64 rnd(seed_);
    // Start with the identity permutation. Subsequent iterations of
    // for loop below will start with perm of previous for loop
@@ -159,6 +163,14 @@ class SharedState {
        ptr.reset(new port::Mutex);
      }
    }
+    if (FLAGS_compaction_thread_pool_adjust_interval > 0) {
+      ++num_bg_threads_;
+      fprintf(stdout, "Starting compaction_thread_pool_adjust_thread\n");
+    }
+    if (FLAGS_continuous_verification_interval > 0) {
+      ++num_bg_threads_;
+      fprintf(stdout, "Starting continuous_verification_thread\n");
+    }
  }

  ~SharedState() {}
@@ -199,7 +211,11 @@ class SharedState {

  void SetVerificationFailure() { verification_failure_.store(true); }

-  bool HasVerificationFailedYet() { return verification_failure_.load(); }
+  bool HasVerificationFailedYet() const { return verification_failure_.load(); }
+
+  void SetShouldStopTest() { should_stop_test_.store(true); }
+
+  bool ShouldStopTest() const { return should_stop_test_.load(); }

  port::Mutex* GetMutexForKey(int cf, int64_t key) {
    return key_locks_[cf][key >> log2_keys_per_lock_].get();
@@ -290,11 +306,13 @@ class SharedState {

  void SetShouldStopBgThread() { should_stop_bg_thread_ = true; }

-  bool ShoudStopBgThread() { return should_stop_bg_thread_; }
+  bool ShouldStopBgThread() { return should_stop_bg_thread_; }

-  void SetBgThreadFinish() { bg_thread_finished_ = true; }
+  void IncBgThreadsFinished() { ++bg_thread_finished_; }

-  bool BgThreadFinished() const { return bg_thread_finished_; }
+  bool BgThreadsFinished() const {
+    return bg_thread_finished_ == num_bg_threads_;
+  }

  bool ShouldVerifyAtBeginning() const {
    return expected_mmap_buffer_.get() != nullptr;
@@ -323,10 +341,12 @@ class SharedState {
  long num_done_;
  bool start_;
  bool start_verify_;
+  int num_bg_threads_;
  bool should_stop_bg_thread_;
-  bool bg_thread_finished_;
+  int bg_thread_finished_;
  StressTest* stress_test_;
  std::atomic<bool> verification_failure_;
+  std::atomic<bool> should_stop_test_;

  // Keys that should not be overwritten
  std::unordered_set<size_t> no_overwrite_ids_;
@@ -34,6 +34,7 @@ class StressTest {

  void OperateDb(ThreadState* thread);
  virtual void VerifyDb(ThreadState* thread) const = 0;
+  virtual void ContinuouslyVerifyDb(ThreadState* /*thread*/) const {}

  void PrintStatistics();

@@ -51,6 +52,8 @@ class StressTest {
  Status NewTxn(WriteOptions& write_opts, Transaction** txn);

  Status CommitTxn(Transaction* txn);
+
+  Status RollbackTxn(Transaction* txn);
 #endif

  virtual void MaybeClearOneColumnFamily(ThreadState* /* thread */) {}
@@ -142,7 +145,13 @@ class StressTest {
                             const std::vector<int64_t>& rand_keys);

  // Enum used by VerifyIterator() to identify the mode to validate.
-  enum LastIterateOp { kLastOpSeek, kLastOpSeekForPrev, kLastOpNextOrPrev };
+  enum LastIterateOp {
+    kLastOpSeek,
+    kLastOpSeekForPrev,
+    kLastOpNextOrPrev,
+    kLastOpSeekToFirst,
+    kLastOpSeekToLast
+  };

  // Compare the two iterator, iter and cmp_iter are in the same position,
  // unless iter might be made invalidate or undefined because of
@@ -150,9 +159,11 @@ class StressTest {
  // Will flag failure if the verification fails.
  // diverged = true if the two iterator is already diverged.
  // True if verification passed, false if not.
+  // op_logs is the information to print when validation fails.
  void VerifyIterator(ThreadState* thread, ColumnFamilyHandle* cmp_cfh,
                      const ReadOptions& ro, Iterator* iter, Iterator* cmp_iter,
-                      LastIterateOp op, const Slice& seek_key, bool* diverged);
+                      LastIterateOp op, const Slice& seek_key,
+                      const std::string& op_logs, bool* diverged);

  virtual Status TestBackupRestore(ThreadState* thread,
                                   const std::vector<int>& rand_column_families,
@@ -162,6 +173,24 @@ class StressTest {
                                const std::vector<int>& rand_column_families,
                                const std::vector<int64_t>& rand_keys);

+  void TestCompactFiles(ThreadState* thread, ColumnFamilyHandle* column_family);
+
+  Status TestFlush(const std::vector<int>& rand_column_families);
+
+  Status TestPauseBackground(ThreadState* thread);
+
+  void TestAcquireSnapshot(ThreadState* thread, int rand_column_family,
+                           const std::string& keystr, uint64_t i);
+
+  Status MaybeReleaseSnapshots(ThreadState* thread, uint64_t i);
+#ifndef ROCKSDB_LITE
+  Status VerifyGetLiveAndWalFiles(ThreadState* thread);
+  virtual Status TestApproximateSize(
+      ThreadState* thread, uint64_t iteration,
+      const std::vector<int>& rand_column_families,
+      const std::vector<int64_t>& rand_keys);
+#endif  // !ROCKSDB_LITE
+
  void VerificationAbort(SharedState* shared, std::string msg, Status s) const;

  void VerificationAbort(SharedState* shared, std::string msg, int cf,
@@ -192,6 +221,10 @@ class StressTest {
  // Fields used for stress-testing secondary instance in the same process
  std::vector<DB*> secondaries_;
  std::vector<std::vector<ColumnFamilyHandle*>> secondary_cfh_lists_;
+
+  // Fields used for continuous verification from another thread
+  DB* cmp_db_;
+  std::vector<ColumnFamilyHandle*> cmp_cfhs_;
 };

 }  // namespace rocksdb
@@ -27,6 +27,7 @@
 namespace rocksdb {
 namespace {
 static std::shared_ptr<rocksdb::Env> env_guard;
+static std::shared_ptr<rocksdb::DbStressEnvWrapper> env_wrapper_guard;
 }  // namespace

 int db_stress_tool(int argc, char** argv) {
@@ -40,33 +41,43 @@ int db_stress_tool(int argc, char** argv) {

  if (FLAGS_statistics) {
    dbstats = rocksdb::CreateDBStatistics();
-    if (FLAGS_enable_secondary) {
+    if (FLAGS_test_secondary) {
      dbstats_secondaries = rocksdb::CreateDBStatistics();
    }
  }
-  FLAGS_compression_type_e =
-      StringToCompressionType(FLAGS_compression_type.c_str());
-  FLAGS_checksum_type_e = StringToChecksumType(FLAGS_checksum_type.c_str());
+  compression_type_e = StringToCompressionType(FLAGS_compression_type.c_str());
+  bottommost_compression_type_e =
+      StringToCompressionType(FLAGS_bottommost_compression_type.c_str());
+  checksum_type_e = StringToChecksumType(FLAGS_checksum_type.c_str());
+
+  Env* raw_env;
+
  if (!FLAGS_hdfs.empty()) {
    if (!FLAGS_env_uri.empty()) {
      fprintf(stderr, "Cannot specify both --hdfs and --env_uri.\n");
      exit(1);
    }
-    FLAGS_env = new rocksdb::HdfsEnv(FLAGS_hdfs);
+    raw_env = new rocksdb::HdfsEnv(FLAGS_hdfs);
  } else if (!FLAGS_env_uri.empty()) {
-    Status s = Env::LoadEnv(FLAGS_env_uri, &FLAGS_env, &env_guard);
-    if (FLAGS_env == nullptr) {
+    Status s = Env::LoadEnv(FLAGS_env_uri, &raw_env, &env_guard);
+    if (raw_env == nullptr) {
      fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str());
      exit(1);
    }
+  } else {
+    raw_env = Env::Default();
  }
+  env_wrapper_guard = std::make_shared<DbStressEnvWrapper>(raw_env);
+  db_stress_env = env_wrapper_guard.get();
+
  FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str());

  // The number of background threads should be at least as much the
  // max number of concurrent compactions.
-  FLAGS_env->SetBackgroundThreads(FLAGS_max_background_compactions);
-  FLAGS_env->SetBackgroundThreads(FLAGS_num_bottom_pri_threads,
-                                  rocksdb::Env::Priority::BOTTOM);
+  db_stress_env->SetBackgroundThreads(FLAGS_max_background_compactions,
+                                      rocksdb::Env::Priority::LOW);
+  db_stress_env->SetBackgroundThreads(FLAGS_num_bottom_pri_threads,
+                                      rocksdb::Env::Priority::BOTTOM);
  if (FLAGS_prefixpercent > 0 && FLAGS_prefix_size < 0) {
    fprintf(stderr,
            "Error: prefixpercent is non-zero while prefix_size is "
@@ -156,16 +167,18 @@ int db_stress_tool(int argc, char** argv) {
  // Choose a location for the test database if none given with --db=<path>
  if (FLAGS_db.empty()) {
    std::string default_db_path;
-    FLAGS_env->GetTestDirectory(&default_db_path);
+    db_stress_env->GetTestDirectory(&default_db_path);
    default_db_path += "/dbstress";
    FLAGS_db = default_db_path;
  }

-  if (FLAGS_enable_secondary && FLAGS_secondaries_base.empty()) {
+  if ((FLAGS_test_secondary || FLAGS_continuous_verification_interval > 0) &&
+      FLAGS_secondaries_base.empty()) {
    std::string default_secondaries_path;
-    FLAGS_env->GetTestDirectory(&default_secondaries_path);
+    db_stress_env->GetTestDirectory(&default_secondaries_path);
    default_secondaries_path += "/dbstress_secondaries";
-    rocksdb::Status s = FLAGS_env->CreateDirIfMissing(default_secondaries_path);
+    rocksdb::Status s =
+        db_stress_env->CreateDirIfMissing(default_secondaries_path);
    if (!s.ok()) {
      fprintf(stderr, "Failed to create directory %s: %s\n",
              default_secondaries_path.c_str(), s.ToString().c_str());
@@ -174,8 +187,10 @@ int db_stress_tool(int argc, char** argv) {
    FLAGS_secondaries_base = default_secondaries_path;
  }

-  if (!FLAGS_enable_secondary && FLAGS_secondary_catch_up_one_in > 0) {
-    fprintf(stderr, "Secondary instance is disabled.\n");
+  if (!FLAGS_test_secondary && FLAGS_secondary_catch_up_one_in > 0) {
+    fprintf(
+        stderr,
+        "Must set -test_secondary=true if secondary_catch_up_one_in > 0.\n");
    exit(1);
  }

@@ -190,6 +205,8 @@ int db_stress_tool(int argc, char** argv) {
  } else {
    stress.reset(CreateNonBatchedOpsStressTest());
  }
+  // Initialize the Zipfian pre-calculated array
+  InitializeHotKeyGenerator(FLAGS_hot_key_alpha);
  if (RunStressTest(stress.get())) {
    return 0;
  } else {
@@ -42,10 +42,9 @@ class NonBatchedOpsStressTest : public StressTest {
          if (thread->shared->HasVerificationFailedYet()) {
            break;
          }
-          // TODO(ljin): update "long" to uint64_t
          // Reseek when the prefix changes
          if (prefix_to_use > 0 &&
-              i % (static_cast<int64_t>(1) << 8 * (8 - prefix_to_use)) == 0) {
+              i % (static_cast<uint64_t>(1) << 8 * (8 - prefix_to_use)) == 0) {
            iter->Seek(Key(i));
          }
          std::string from_db;
@@ -65,7 +64,7 @@ class NonBatchedOpsStressTest : public StressTest {
          } else {
            // The iterator found no value for the key in question, so do not
            // move to the next item in the iterator
-            s = Status::NotFound(Slice());
+            s = Status::NotFound();
          }
          VerifyValue(static_cast<int>(cf), i, options, shared, from_db, s,
                      true);
@@ -96,8 +95,8 @@ class NonBatchedOpsStressTest : public StressTest {
  }

  void MaybeClearOneColumnFamily(ThreadState* thread) override {
-    if (FLAGS_clear_column_family_one_in != 0 && FLAGS_column_families > 1) {
-      if (thread->rand.OneIn(FLAGS_clear_column_family_one_in)) {
+    if (FLAGS_column_families > 1) {
+      if (thread->rand.OneInOpt(FLAGS_clear_column_family_one_in)) {
        // drop column family and then create it again (can't drop default)
        int cf = thread->rand.Next() % (FLAGS_column_families - 1) + 1;
        std::string new_name = ToString(new_column_family_name_.fetch_add(1));
@@ -148,6 +147,7 @@ class NonBatchedOpsStressTest : public StressTest {
      thread->stats.AddGets(1, 0);
    } else {
      // errors case
+      fprintf(stderr, "TestGet error: %s\n", s.ToString().c_str());
      thread->stats.AddErrors(1);
    }
    return s;
@@ -166,12 +166,75 @@ class NonBatchedOpsStressTest : public StressTest {
    std::vector<Status> statuses(num_keys);
    ColumnFamilyHandle* cfh = column_families_[rand_column_families[0]];

+    // To appease clang analyzer
+    const bool use_txn = FLAGS_use_txn;
+
+    // Create a transaction in order to write some data. The purpose is to
+    // exercise WriteBatchWithIndex::MultiGetFromBatchAndDB. The transaction
+    // will be rolled back once MultiGet returns.
+#ifndef ROCKSDB_LITE
+    Transaction* txn = nullptr;
+    if (use_txn) {
+      WriteOptions wo;
+      Status s = NewTxn(wo, &txn);
+      if (!s.ok()) {
+        fprintf(stderr, "NewTxn: %s\n", s.ToString().c_str());
+        std::terminate();
+      }
+    }
+#endif
    for (size_t i = 0; i < num_keys; ++i) {
      key_str.emplace_back(Key(rand_keys[i]));
      keys.emplace_back(key_str.back());
+#ifndef ROCKSDB_LITE
+      if (use_txn) {
+        // With a 1 in 10 probability, insert the just added key in the batch
+        // into the transaction. This will create an overlap with the MultiGet
+        // keys and exercise some corner cases in the code
+        if (thread->rand.OneIn(10)) {
+          int op = thread->rand.Uniform(2);
+          Status s;
+          switch (op) {
+            case 0:
+            case 1: {
+              uint32_t value_base =
+                  thread->rand.Next() % thread->shared->UNKNOWN_SENTINEL;
+              char value[100];
+              size_t sz = GenerateValue(value_base, value, sizeof(value));
+              Slice v(value, sz);
+              if (op == 0) {
+                s = txn->Put(cfh, keys.back(), v);
+              } else {
+                s = txn->Merge(cfh, keys.back(), v);
+              }
+              break;
+            }
+            case 2:
+              s = txn->Delete(cfh, keys.back());
+              break;
+            default:
+              assert(false);
+          }
+          if (!s.ok()) {
+            fprintf(stderr, "Transaction put: %s\n", s.ToString().c_str());
+            std::terminate();
+          }
+        }
+      }
+#endif
    }
-    db_->MultiGet(read_opts, cfh, num_keys, keys.data(), values.data(),
-                  statuses.data());
+
+    if (!use_txn) {
+      db_->MultiGet(read_opts, cfh, num_keys, keys.data(), values.data(),
+                    statuses.data());
+    } else {
+#ifndef ROCKSDB_LITE
+      txn->MultiGet(read_opts, cfh, num_keys, keys.data(), values.data(),
+                    statuses.data());
+      RollbackTxn(txn);
+#endif
+    }
+
    for (const auto& s : statuses) {
      if (s.ok()) {
        // found case
@@ -179,8 +242,12 @@ class NonBatchedOpsStressTest : public StressTest {
      } else if (s.IsNotFound()) {
        // not found case
        thread->stats.AddGets(1, 0);
+      } else if (s.IsMergeInProgress() && use_txn) {
+        // With txn this is sometimes expected.
+        thread->stats.AddGets(1, 1);
      } else {
        // errors case
+        fprintf(stderr, "MultiGet error: %s\n", s.ToString().c_str());
        thread->stats.AddErrors(1);
      }
    }
@@ -215,6 +282,7 @@ class NonBatchedOpsStressTest : public StressTest {
    if (iter->status().ok()) {
      thread->stats.AddPrefixes(1, count);
    } else {
+      fprintf(stderr, "TestPrefixScan error: %s\n", s.ToString().c_str());
      thread->stats.AddErrors(1);
    }
    delete iter;
@@ -447,10 +515,10 @@ class NonBatchedOpsStressTest : public StressTest {
    const std::string sst_filename =
        FLAGS_db + "/." + ToString(thread->tid) + ".sst";
    Status s;
-    if (FLAGS_env->FileExists(sst_filename).ok()) {
+    if (db_stress_env->FileExists(sst_filename).ok()) {
      // Maybe we terminated abnormally before, so cleanup to give this file
      // ingestion a clean slate
-      s = FLAGS_env->DeleteFile(sst_filename);
+      s = db_stress_env->DeleteFile(sst_filename);
    }

    SstFileWriter sst_file_writer(EnvOptions(options_), options_);
@@ -506,7 +574,7 @@ class NonBatchedOpsStressTest : public StressTest {

  bool VerifyValue(int cf, int64_t key, const ReadOptions& /*opts*/,
                   SharedState* shared, const std::string& value_from_db,
-                   Status s, bool strict = false) const {
+                   const Status& s, bool strict = false) const {
    if (shared->HasVerificationFailedYet()) {
      return false;
    }
@@ -10,6 +10,7 @@
 #include "rocksdb/env.h"

 #include <thread>
+#include "env/composite_env_wrapper.h"
 #include "logging/env_logger.h"
 #include "memory/arena.h"
 #include "options/db_options.h"
@@ -485,8 +486,9 @@ Status NewEnvLogger(const std::string& fname, Env* env,
    return status;
  }

-  *result = std::make_shared<EnvLogger>(std::move(writable_file), fname,
-                                        options, env);
+  *result = std::make_shared<EnvLogger>(
+      NewLegacyWritableFileWrapper(std::move(writable_file)), fname, options,
+      env);
  return Status::OK();
 }

@@ -53,6 +53,7 @@
 #include <set>
 #include <vector>

+#include "env/composite_env_wrapper.h"
 #include "env/io_posix.h"
 #include "logging/logging.h"
 #include "logging/posix_logger.h"
@@ -98,45 +99,6 @@ ThreadStatusUpdater* CreateThreadStatusUpdater() {
  return new ThreadStatusUpdater();
 }

-inline mode_t GetDBFileMode(bool allow_non_owner_access) {
-  return allow_non_owner_access ? 0644 : 0600;
-}
-
-// list of pathnames that are locked
-static std::set<std::string> lockedFiles;
-static port::Mutex mutex_lockedFiles;
-
-static int LockOrUnlock(int fd, bool lock) {
-  errno = 0;
-  struct flock f;
-  memset(&f, 0, sizeof(f));
-  f.l_type = (lock ? F_WRLCK : F_UNLCK);
-  f.l_whence = SEEK_SET;
-  f.l_start = 0;
-  f.l_len = 0;        // Lock/unlock entire file
-  int value = fcntl(fd, F_SETLK, &f);
-
-  return value;
-}
-
-class PosixFileLock : public FileLock {
- public:
-  int fd_;
-  std::string filename;
-};
-
-int cloexec_flags(int flags, const EnvOptions* options) {
-  // If the system supports opening the file with cloexec enabled,
-  // do so, as this avoids a race condition if a db is opened around
-  // the same time that a child process is forked
-#ifdef O_CLOEXEC
-  if (options == nullptr || options->set_fd_cloexec) {
-    flags |= O_CLOEXEC;
-  }
-#endif
-  return flags;
-}
-
 #ifndef ROCKSDB_NO_DYNAMIC_EXTENSION
 class PosixDynamicLibrary : public DynamicLibrary {
 public:
@@ -164,7 +126,7 @@ class PosixDynamicLibrary : public DynamicLibrary {
 };
 #endif  // !ROCKSDB_NO_DYNAMIC_EXTENSION

-class PosixEnv : public Env {
+class PosixEnv : public CompositeEnvWrapper {
 public:
  PosixEnv();

@@ -190,599 +152,6 @@ class PosixEnv : public Env {
    }
  }

-  Status NewSequentialFile(const std::string& fname,
-                           std::unique_ptr<SequentialFile>* result,
-                           const EnvOptions& options) override {
-    result->reset();
-    int fd = -1;
-    int flags = cloexec_flags(O_RDONLY, &options);
-    FILE* file = nullptr;
-
-    if (options.use_direct_reads && !options.use_mmap_reads) {
-#ifdef ROCKSDB_LITE
-      return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
-#endif  // !ROCKSDB_LITE
-#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
-      flags |= O_DIRECT;
-#endif
-    }
-
-    do {
-      IOSTATS_TIMER_GUARD(open_nanos);
-      fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_));
-    } while (fd < 0 && errno == EINTR);
-    if (fd < 0) {
-      return IOError("While opening a file for sequentially reading", fname,
-                     errno);
-    }
-
-    SetFD_CLOEXEC(fd, &options);
-
-    if (options.use_direct_reads && !options.use_mmap_reads) {
-#ifdef OS_MACOSX
-      if (fcntl(fd, F_NOCACHE, 1) == -1) {
-        close(fd);
-        return IOError("While fcntl NoCache", fname, errno);
-      }
-#endif
-    } else {
-      do {
-        IOSTATS_TIMER_GUARD(open_nanos);
-        file = fdopen(fd, "r");
-      } while (file == nullptr && errno == EINTR);
-      if (file == nullptr) {
-        close(fd);
-        return IOError("While opening file for sequentially read", fname,
-                       errno);
-      }
-    }
-    result->reset(new PosixSequentialFile(fname, file, fd, options));
-    return Status::OK();
-  }
-
-  Status NewRandomAccessFile(const std::string& fname,
-                             std::unique_ptr<RandomAccessFile>* result,
-                             const EnvOptions& options) override {
-    result->reset();
-    Status s;
-    int fd;
-    int flags = cloexec_flags(O_RDONLY, &options);
-
-    if (options.use_direct_reads && !options.use_mmap_reads) {
-#ifdef ROCKSDB_LITE
-      return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
-#endif  // !ROCKSDB_LITE
-#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
-      flags |= O_DIRECT;
-      TEST_SYNC_POINT_CALLBACK("NewRandomAccessFile:O_DIRECT", &flags);
-#endif
-    }
-
-    do {
-      IOSTATS_TIMER_GUARD(open_nanos);
-      fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_));
-    } while (fd < 0 && errno == EINTR);
-    if (fd < 0) {
-      return IOError("While open a file for random read", fname, errno);
-    }
-    SetFD_CLOEXEC(fd, &options);
-
-    if (options.use_mmap_reads && sizeof(void*) >= 8) {
-      // Use of mmap for random reads has been removed because it
-      // kills performance when storage is fast.
-      // Use mmap when virtual address-space is plentiful.
-      uint64_t size;
-      s = GetFileSize(fname, &size);
-      if (s.ok()) {
-        void* base = mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0);
-        if (base != MAP_FAILED) {
-          result->reset(new PosixMmapReadableFile(fd, fname, base,
-                                                  size, options));
-        } else {
-          s = IOError("while mmap file for read", fname, errno);
-          close(fd);
-        }
-      }
-    } else {
-      if (options.use_direct_reads && !options.use_mmap_reads) {
-#ifdef OS_MACOSX
-        if (fcntl(fd, F_NOCACHE, 1) == -1) {
-          close(fd);
-          return IOError("while fcntl NoCache", fname, errno);
-        }
-#endif
-      }
-      result->reset(new PosixRandomAccessFile(fname, fd, options
-#if defined(ROCKSDB_IOURING_PRESENT)
-                                              ,
-                                              thread_local_io_urings_.get()
-#endif
-                                                  ));
-    }
-    return s;
-  }
-
-  virtual Status OpenWritableFile(const std::string& fname,
-                                  std::unique_ptr<WritableFile>* result,
-                                  const EnvOptions& options,
-                                  bool reopen = false) {
-    result->reset();
-    Status s;
-    int fd = -1;
-    int flags = (reopen) ? (O_CREAT | O_APPEND) : (O_CREAT | O_TRUNC);
-    // Direct IO mode with O_DIRECT flag or F_NOCAHCE (MAC OSX)
-    if (options.use_direct_writes && !options.use_mmap_writes) {
-      // Note: we should avoid O_APPEND here due to ta the following bug:
-      // POSIX requires that opening a file with the O_APPEND flag should
-      // have no affect on the location at which pwrite() writes data.
-      // However, on Linux, if a file is opened with O_APPEND, pwrite()
-      // appends data to the end of the file, regardless of the value of
-      // offset.
-      // More info here: https://linux.die.net/man/2/pwrite
-#ifdef ROCKSDB_LITE
-      return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
-#endif  // ROCKSDB_LITE
-      flags |= O_WRONLY;
-#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
-      flags |= O_DIRECT;
-#endif
-      TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags);
-    } else if (options.use_mmap_writes) {
-      // non-direct I/O
-      flags |= O_RDWR;
-    } else {
-      flags |= O_WRONLY;
-    }
-
-    flags = cloexec_flags(flags, &options);
-
-    do {
-      IOSTATS_TIMER_GUARD(open_nanos);
-      fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_));
-    } while (fd < 0 && errno == EINTR);
-
-    if (fd < 0) {
-      s = IOError("While open a file for appending", fname, errno);
-      return s;
-    }
-    SetFD_CLOEXEC(fd, &options);
-
-    if (options.use_mmap_writes) {
-      if (!checkedDiskForMmap_) {
-        // this will be executed once in the program's lifetime.
-        // do not use mmapWrite on non ext-3/xfs/tmpfs systems.
-        if (!SupportsFastAllocate(fname)) {
-          forceMmapOff_ = true;
-        }
-        checkedDiskForMmap_ = true;
-      }
-    }
-    if (options.use_mmap_writes && !forceMmapOff_) {
-      result->reset(new PosixMmapFile(fname, fd, page_size_, options));
-    } else if (options.use_direct_writes && !options.use_mmap_writes) {
-#ifdef OS_MACOSX
-      if (fcntl(fd, F_NOCACHE, 1) == -1) {
-        close(fd);
-        s = IOError("While fcntl NoCache an opened file for appending", fname,
-                    errno);
-        return s;
-      }
-#elif defined(OS_SOLARIS)
-      if (directio(fd, DIRECTIO_ON) == -1) {
-        if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON
-          close(fd);
-          s = IOError("While calling directio()", fname, errno);
-          return s;
-        }
-      }
-#endif
-      result->reset(new PosixWritableFile(fname, fd, options));
-    } else {
-      // disable mmap writes
-      EnvOptions no_mmap_writes_options = options;
-      no_mmap_writes_options.use_mmap_writes = false;
-      result->reset(new PosixWritableFile(fname, fd, no_mmap_writes_options));
-    }
-    return s;
-  }
-
-  Status NewWritableFile(const std::string& fname,
-                         std::unique_ptr<WritableFile>* result,
-                         const EnvOptions& options) override {
-    return OpenWritableFile(fname, result, options, false);
-  }
-
-  Status ReopenWritableFile(const std::string& fname,
-                            std::unique_ptr<WritableFile>* result,
-                            const EnvOptions& options) override {
-    return OpenWritableFile(fname, result, options, true);
-  }
-
-  Status ReuseWritableFile(const std::string& fname,
-                           const std::string& old_fname,
-                           std::unique_ptr<WritableFile>* result,
-                           const EnvOptions& options) override {
-    result->reset();
-    Status s;
-    int fd = -1;
-
-    int flags = 0;
-    // Direct IO mode with O_DIRECT flag or F_NOCAHCE (MAC OSX)
-    if (options.use_direct_writes && !options.use_mmap_writes) {
-#ifdef ROCKSDB_LITE
-      return Status::IOError(fname, "Direct I/O not supported in RocksDB lite");
-#endif  // !ROCKSDB_LITE
-      flags |= O_WRONLY;
-#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
-      flags |= O_DIRECT;
-#endif
-      TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags);
-    } else if (options.use_mmap_writes) {
-      // mmap needs O_RDWR mode
-      flags |= O_RDWR;
-    } else {
-      flags |= O_WRONLY;
-    }
-
-    flags = cloexec_flags(flags, &options);
-
-    do {
-      IOSTATS_TIMER_GUARD(open_nanos);
-      fd = open(old_fname.c_str(), flags,
-                GetDBFileMode(allow_non_owner_access_));
-    } while (fd < 0 && errno == EINTR);
-    if (fd < 0) {
-      s = IOError("while reopen file for write", fname, errno);
-      return s;
-    }
-
-    SetFD_CLOEXEC(fd, &options);
-    // rename into place
-    if (rename(old_fname.c_str(), fname.c_str()) != 0) {
-      s = IOError("while rename file to " + fname, old_fname, errno);
-      close(fd);
-      return s;
-    }
-
-    if (options.use_mmap_writes) {
-      if (!checkedDiskForMmap_) {
-        // this will be executed once in the program's lifetime.
-        // do not use mmapWrite on non ext-3/xfs/tmpfs systems.
-        if (!SupportsFastAllocate(fname)) {
-          forceMmapOff_ = true;
-        }
-        checkedDiskForMmap_ = true;
-      }
-    }
-    if (options.use_mmap_writes && !forceMmapOff_) {
-      result->reset(new PosixMmapFile(fname, fd, page_size_, options));
-    } else if (options.use_direct_writes && !options.use_mmap_writes) {
-#ifdef OS_MACOSX
-      if (fcntl(fd, F_NOCACHE, 1) == -1) {
-        close(fd);
-        s = IOError("while fcntl NoCache for reopened file for append", fname,
-                    errno);
-        return s;
-      }
-#elif defined(OS_SOLARIS)
-      if (directio(fd, DIRECTIO_ON) == -1) {
-        if (errno != ENOTTY) { // ZFS filesystems don't support DIRECTIO_ON
-          close(fd);
-          s = IOError("while calling directio()", fname, errno);
-          return s;
-        }
-      }
-#endif
-      result->reset(new PosixWritableFile(fname, fd, options));
-    } else {
-      // disable mmap writes
-      EnvOptions no_mmap_writes_options = options;
-      no_mmap_writes_options.use_mmap_writes = false;
-      result->reset(new PosixWritableFile(fname, fd, no_mmap_writes_options));
-    }
-    return s;
-  }
-
-  Status NewRandomRWFile(const std::string& fname,
-                         std::unique_ptr<RandomRWFile>* result,
-                         const EnvOptions& options) override {
-    int fd = -1;
-    int flags = cloexec_flags(O_RDWR, &options);
-
-    while (fd < 0) {
-      IOSTATS_TIMER_GUARD(open_nanos);
-
-      fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_));
-      if (fd < 0) {
-        // Error while opening the file
-        if (errno == EINTR) {
-          continue;
-        }
-        return IOError("While open file for random read/write", fname, errno);
-      }
-    }
-
-    SetFD_CLOEXEC(fd, &options);
-    result->reset(new PosixRandomRWFile(fname, fd, options));
-    return Status::OK();
-  }
-
-  Status NewMemoryMappedFileBuffer(
-      const std::string& fname,
-      std::unique_ptr<MemoryMappedFileBuffer>* result) override {
-    int fd = -1;
-    Status status;
-    int flags = cloexec_flags(O_RDWR, nullptr);
-
-    while (fd < 0) {
-      IOSTATS_TIMER_GUARD(open_nanos);
-      fd = open(fname.c_str(), flags, 0644);
-      if (fd < 0) {
-        // Error while opening the file
-        if (errno == EINTR) {
-          continue;
-        }
-        status =
-            IOError("While open file for raw mmap buffer access", fname, errno);
-        break;
-      }
-    }
-    uint64_t size;
-    if (status.ok()) {
-      status = GetFileSize(fname, &size);
-    }
-    void* base = nullptr;
-    if (status.ok()) {
-      base = mmap(nullptr, static_cast<size_t>(size), PROT_READ | PROT_WRITE,
-                  MAP_SHARED, fd, 0);
-      if (base == MAP_FAILED) {
-        status = IOError("while mmap file for read", fname, errno);
-      }
-    }
-    if (status.ok()) {
-      result->reset(
-          new PosixMemoryMappedFileBuffer(base, static_cast<size_t>(size)));
-    }
-    if (fd >= 0) {
-      // don't need to keep it open after mmap has been called
-      close(fd);
-    }
-    return status;
-  }
-
-  Status NewDirectory(const std::string& name,
-                      std::unique_ptr<Directory>* result) override {
-    result->reset();
-    int fd;
-    int flags = cloexec_flags(0, nullptr);
-    {
-      IOSTATS_TIMER_GUARD(open_nanos);
-      fd = open(name.c_str(), flags);
-    }
-    if (fd < 0) {
-      return IOError("While open directory", name, errno);
-    } else {
-      result->reset(new PosixDirectory(fd));
-    }
-    return Status::OK();
-  }
-
-  Status FileExists(const std::string& fname) override {
-    int result = access(fname.c_str(), F_OK);
-
-    if (result == 0) {
-      return Status::OK();
-    }
-
-    int err = errno;
-    switch (err) {
-      case EACCES:
-      case ELOOP:
-      case ENAMETOOLONG:
-      case ENOENT:
-      case ENOTDIR:
-        return Status::NotFound();
-      default:
-        assert(err == EIO || err == ENOMEM);
-        return Status::IOError("Unexpected error(" + ToString(err) +
-                               ") accessing file `" + fname + "' ");
-    }
-  }
-
-  Status GetChildren(const std::string& dir,
-                     std::vector<std::string>* result) override {
-    result->clear();
-    DIR* d = opendir(dir.c_str());
-    if (d == nullptr) {
-      switch (errno) {
-        case EACCES:
-        case ENOENT:
-        case ENOTDIR:
-          return Status::NotFound();
-        default:
-          return IOError("While opendir", dir, errno);
-      }
-    }
-    struct dirent* entry;
-    while ((entry = readdir(d)) != nullptr) {
-      result->push_back(entry->d_name);
-    }
-    closedir(d);
-    return Status::OK();
-  }
-
-  Status DeleteFile(const std::string& fname) override {
-    Status result;
-    if (unlink(fname.c_str()) != 0) {
-      result = IOError("while unlink() file", fname, errno);
-    }
-    return result;
-  };
-
-  Status CreateDir(const std::string& name) override {
-    Status result;
-    if (mkdir(name.c_str(), 0755) != 0) {
-      result = IOError("While mkdir", name, errno);
-    }
-    return result;
-  };
-
-  Status CreateDirIfMissing(const std::string& name) override {
-    Status result;
-    if (mkdir(name.c_str(), 0755) != 0) {
-      if (errno != EEXIST) {
-        result = IOError("While mkdir if missing", name, errno);
-      } else if (!DirExists(name)) { // Check that name is actually a
-                                     // directory.
-        // Message is taken from mkdir
-        result = Status::IOError("`"+name+"' exists but is not a directory");
-      }
-    }
-    return result;
-  };
-
-  Status DeleteDir(const std::string& name) override {
-    Status result;
-    if (rmdir(name.c_str()) != 0) {
-      result = IOError("file rmdir", name, errno);
-    }
-    return result;
-  };
-
-  Status GetFileSize(const std::string& fname, uint64_t* size) override {
-    Status s;
-    struct stat sbuf;
-    if (stat(fname.c_str(), &sbuf) != 0) {
-      *size = 0;
-      s = IOError("while stat a file for size", fname, errno);
-    } else {
-      *size = sbuf.st_size;
-    }
-    return s;
-  }
-
-  Status GetFileModificationTime(const std::string& fname,
-                                 uint64_t* file_mtime) override {
-    struct stat s;
-    if (stat(fname.c_str(), &s) !=0) {
-      return IOError("while stat a file for modification time", fname, errno);
-    }
-    *file_mtime = static_cast<uint64_t>(s.st_mtime);
-    return Status::OK();
-  }
-  Status RenameFile(const std::string& src,
-                    const std::string& target) override {
-    Status result;
-    if (rename(src.c_str(), target.c_str()) != 0) {
-      result = IOError("While renaming a file to " + target, src, errno);
-    }
-    return result;
-  }
-
-  Status LinkFile(const std::string& src, const std::string& target) override {
-    Status result;
-    if (link(src.c_str(), target.c_str()) != 0) {
-      if (errno == EXDEV) {
-        return Status::NotSupported("No cross FS links allowed");
-      }
-      result = IOError("while link file to " + target, src, errno);
-    }
-    return result;
-  }
-
-  Status NumFileLinks(const std::string& fname, uint64_t* count) override {
-    struct stat s;
-    if (stat(fname.c_str(), &s) != 0) {
-      return IOError("while stat a file for num file links", fname, errno);
-    }
-    *count = static_cast<uint64_t>(s.st_nlink);
-    return Status::OK();
-  }
-
-  Status AreFilesSame(const std::string& first, const std::string& second,
-                      bool* res) override {
-    struct stat statbuf[2];
-    if (stat(first.c_str(), &statbuf[0]) != 0) {
-      return IOError("stat file", first, errno);
-    }
-    if (stat(second.c_str(), &statbuf[1]) != 0) {
-      return IOError("stat file", second, errno);
-    }
-
-    if (major(statbuf[0].st_dev) != major(statbuf[1].st_dev) ||
-        minor(statbuf[0].st_dev) != minor(statbuf[1].st_dev) ||
-        statbuf[0].st_ino != statbuf[1].st_ino) {
-      *res = false;
-    } else {
-      *res = true;
-    }
-    return Status::OK();
-  }
-
-  Status LockFile(const std::string& fname, FileLock** lock) override {
-    *lock = nullptr;
-    Status result;
-
-    mutex_lockedFiles.Lock();
-    // If it already exists in the lockedFiles set, then it is already locked,
-    // and fail this lock attempt. Otherwise, insert it into lockedFiles.
-    // This check is needed because fcntl() does not detect lock conflict
-    // if the fcntl is issued by the same thread that earlier acquired
-    // this lock.
-    // We must do this check *before* opening the file:
-    // Otherwise, we will open a new file descriptor. Locks are associated with
-    // a process, not a file descriptor and when *any* file descriptor is closed,
-    // all locks the process holds for that *file* are released
-    if (lockedFiles.insert(fname).second == false) {
-      mutex_lockedFiles.Unlock();
-      errno = ENOLCK;
-      return IOError("lock ", fname, errno);
-    }
-
-    int fd;
-    int flags = cloexec_flags(O_RDWR | O_CREAT, nullptr);
-
-    {
-      IOSTATS_TIMER_GUARD(open_nanos);
-      fd = open(fname.c_str(), flags, 0644);
-    }
-    if (fd < 0) {
-      result = IOError("while open a file for lock", fname, errno);
-    } else if (LockOrUnlock(fd, true) == -1) {
-      // if there is an error in locking, then remove the pathname from lockedfiles
-      lockedFiles.erase(fname);
-      result = IOError("While lock file", fname, errno);
-      close(fd);
-    } else {
-      SetFD_CLOEXEC(fd, nullptr);
-      PosixFileLock* my_lock = new PosixFileLock;
-      my_lock->fd_ = fd;
-      my_lock->filename = fname;
-      *lock = my_lock;
-    }
-
-    mutex_lockedFiles.Unlock();
-    return result;
-  }
-
-  Status UnlockFile(FileLock* lock) override {
-    PosixFileLock* my_lock = reinterpret_cast<PosixFileLock*>(lock);
-    Status result;
-    mutex_lockedFiles.Lock();
-    // If we are unlocking, then verify that we had locked it earlier,
-    // it should already exist in lockedFiles. Remove it from lockedFiles.
-    if (lockedFiles.erase(my_lock->filename) != 1) {
-      errno = ENOLCK;
-      result = IOError("unlock", my_lock->filename, errno);
-    } else if (LockOrUnlock(my_lock->fd_, false) == -1) {
-      result = IOError("unlock", my_lock->filename, errno);
-    }
-    close(my_lock->fd_);
-    delete my_lock;
-    mutex_lockedFiles.Unlock();
-    return result;
-  }
-
 #ifndef ROCKSDB_NO_DYNAMIC_EXTENSION
  // Loads the named library into the result.
  // If the input name is empty, the current executable is loaded
@@ -883,17 +252,6 @@ class PosixEnv : public Env {

  uint64_t GetThreadID() const override { return gettid(pthread_self()); }

-  Status GetFreeSpace(const std::string& fname, uint64_t* free_space) override {
-    struct statvfs sbuf;
-
-    if (statvfs(fname.c_str(), &sbuf) < 0) {
-      return IOError("While doing statvfs", fname, errno);
-    }
-
-    *free_space = ((uint64_t)sbuf.f_bsize * sbuf.f_bfree);
-    return Status::OK();
-  }
-
  Status NewLogger(const std::string& fname,
                   std::shared_ptr<Logger>* result) override {
    FILE* f;
@@ -963,10 +321,11 @@ class PosixEnv : public Env {
  Status GetHostName(char* name, uint64_t len) override {
    int ret = gethostname(name, static_cast<size_t>(len));
    if (ret < 0) {
-      if (errno == EFAULT || errno == EINVAL)
+      if (errno == EFAULT || errno == EINVAL) {
        return Status::InvalidArgument(strerror(errno));
-      else
+      } else {
        return IOError("GetHostName", name, errno);
+      }
    }
    return Status::OK();
  }
@@ -980,23 +339,12 @@ class PosixEnv : public Env {
    return Status::OK();
  }

-  Status GetAbsolutePath(const std::string& db_path,
-                         std::string* output_path) override {
-    if (!db_path.empty() && db_path[0] == '/') {
-      *output_path = db_path;
-      return Status::OK();
-    }
-
-    char the_path[256];
-    char* ret = getcwd(the_path, 256);
-    if (ret == nullptr) {
-      return Status::IOError(strerror(errno));
-    }
-
-    *output_path = ret;
-    return Status::OK();
+  ThreadStatusUpdater* GetThreadStatusUpdater() const override {
+    return Env::GetThreadStatusUpdater();
  }

+  std::string GenerateUniqueId() override { return Env::GenerateUniqueId(); }
+
  // Allow increasing the number of worker threads.
  void SetBackgroundThreads(int num, Priority pri) override {
    assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH);
@@ -1057,72 +405,7 @@ class PosixEnv : public Env {
    return dummy;
  }

-  EnvOptions OptimizeForLogWrite(const EnvOptions& env_options,
-                                 const DBOptions& db_options) const override {
-    EnvOptions optimized = env_options;
-    optimized.use_mmap_writes = false;
-    optimized.use_direct_writes = false;
-    optimized.bytes_per_sync = db_options.wal_bytes_per_sync;
-    // TODO(icanadi) it's faster if fallocate_with_keep_size is false, but it
-    // breaks TransactionLogIteratorStallAtLastRecord unit test. Fix the unit
-    // test and make this false
-    optimized.fallocate_with_keep_size = true;
-    optimized.writable_file_max_buffer_size =
-        db_options.writable_file_max_buffer_size;
-    return optimized;
-  }
-
-  EnvOptions OptimizeForManifestWrite(
-      const EnvOptions& env_options) const override {
-    EnvOptions optimized = env_options;
-    optimized.use_mmap_writes = false;
-    optimized.use_direct_writes = false;
-    optimized.fallocate_with_keep_size = true;
-    return optimized;
-  }
-
 private:
-  bool checkedDiskForMmap_;
-  bool forceMmapOff_;  // do we override Env options?
-
-  // Returns true iff the named directory exists and is a directory.
-  virtual bool DirExists(const std::string& dname) {
-    struct stat statbuf;
-    if (stat(dname.c_str(), &statbuf) == 0) {
-      return S_ISDIR(statbuf.st_mode);
-    }
-    return false; // stat() failed return false
-  }
-
-  bool SupportsFastAllocate(const std::string& path) {
-#ifdef ROCKSDB_FALLOCATE_PRESENT
-    struct statfs s;
-    if (statfs(path.c_str(), &s)){
-      return false;
-    }
-    switch (s.f_type) {
-      case EXT4_SUPER_MAGIC:
-        return true;
-      case XFS_SUPER_MAGIC:
-        return true;
-      case TMPFS_MAGIC:
-        return true;
-      default:
-        return false;
-    }
-#else
-    (void)path;
-    return false;
-#endif
-  }
-
-#if defined(ROCKSDB_IOURING_PRESENT)
-  // io_uring instance
-  std::unique_ptr<ThreadLocalPtr> thread_local_io_urings_;
-#endif
-
-  size_t page_size_;
-
  std::vector<ThreadPoolImpl> thread_pools_;
  pthread_mutex_t mu_;
  std::vector<pthread_t> threads_to_join_;
@@ -1132,9 +415,7 @@ class PosixEnv : public Env {
 };

 PosixEnv::PosixEnv()
-    : checkedDiskForMmap_(false),
-      forceMmapOff_(false),
-      page_size_(getpagesize()),
+    : CompositeEnvWrapper(this, FileSystem::Default().get()),
      thread_pools_(Priority::TOTAL),
      allow_non_owner_access_(true) {
  ThreadPoolImpl::PthreadCall("mutex_init", pthread_mutex_init(&mu_, nullptr));
@@ -1145,17 +426,6 @@ PosixEnv::PosixEnv()
    thread_pools_[pool_id].SetHostEnv(this);
  }
  thread_status_updater_ = CreateThreadStatusUpdater();
-
-#if defined(ROCKSDB_IOURING_PRESENT)
-  // Test whether IOUring is supported, and if it does, create a managing
-  // object for thread local point so that in the future thread-local
-  // io_uring can be created.
-  struct io_uring* new_io_uring = CreateIOUring();
-  if (new_io_uring != nullptr) {
-    thread_local_io_urings_.reset(new ThreadLocalPtr(DeleteIOUring));
-    delete new_io_uring;
-  }
-#endif
 }

 void PosixEnv::Schedule(void (*function)(void* arg1), void* arg, Priority pri,
@@ -1249,7 +519,9 @@ Env* Env::Default() {
  CompressionContextCache::InitSingleton();
  INIT_SYNC_POINT_SINGLETONS();
  static PosixEnv default_env;
-  return &default_env;
+  static CompositeEnvWrapper composite_env(&default_env,
+                                           FileSystem::Default().get());
+  return &composite_env;
 }

 }  // namespace rocksdb
@@ -0,0 +1,110 @@
+// Copyright (c) 2019-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+#include "env/composite_env_wrapper.h"
+#include "rocksdb/file_system.h"
+#include "options/db_options.h"
+#include "rocksdb/utilities/object_registry.h"
+
+namespace rocksdb {
+
+FileSystem::FileSystem() {}
+
+FileSystem::~FileSystem() {}
+
+Status FileSystem::Load(const std::string& value,
+                        std::shared_ptr<FileSystem>* result) {
+  Status s;
+#ifndef ROCKSDB_LITE
+  s = ObjectRegistry::NewInstance()->NewSharedObject<FileSystem>(value, result);
+#else
+  (void)result;
+  s = Status::NotSupported("Cannot load FileSystem in LITE mode: ", value);
+#endif
+  return s;
+}
+
+FileOptions FileSystem::OptimizeForLogRead(
+              const FileOptions& file_options) const {
+  FileOptions optimized_file_options(file_options);
+  optimized_file_options.use_direct_reads = false;
+  return optimized_file_options;
+}
+
+FileOptions FileSystem::OptimizeForManifestRead(
+    const FileOptions& file_options) const {
+  FileOptions optimized_file_options(file_options);
+  optimized_file_options.use_direct_reads = false;
+  return optimized_file_options;
+}
+
+FileOptions FileSystem::OptimizeForLogWrite(const FileOptions& file_options,
+                                           const DBOptions& db_options) const {
+  FileOptions optimized_file_options(file_options);
+  optimized_file_options.bytes_per_sync = db_options.wal_bytes_per_sync;
+  optimized_file_options.writable_file_max_buffer_size =
+      db_options.writable_file_max_buffer_size;
+  return optimized_file_options;
+}
+
+FileOptions FileSystem::OptimizeForManifestWrite(
+    const FileOptions& file_options) const {
+  return file_options;
+}
+
+FileOptions FileSystem::OptimizeForCompactionTableWrite(
+    const FileOptions& file_options,
+    const ImmutableDBOptions& db_options) const {
+  FileOptions optimized_file_options(file_options);
+  optimized_file_options.use_direct_writes =
+      db_options.use_direct_io_for_flush_and_compaction;
+  return optimized_file_options;
+}
+
+FileOptions FileSystem::OptimizeForCompactionTableRead(
+    const FileOptions& file_options,
+    const ImmutableDBOptions& db_options) const {
+  FileOptions optimized_file_options(file_options);
+  optimized_file_options.use_direct_reads = db_options.use_direct_reads;
+  return optimized_file_options;
+}
+
+Status ReadFileToString(FileSystem* fs, const std::string& fname,
+                        std::string* data) {
+  FileOptions soptions;
+  data->clear();
+  std::unique_ptr<FSSequentialFile> file;
+  Status s = fs->NewSequentialFile(fname, soptions, &file, nullptr);
+  if (!s.ok()) {
+    return s;
+  }
+  static const int kBufferSize = 8192;
+  char* space = new char[kBufferSize];
+  while (true) {
+    Slice fragment;
+    s = file->Read(kBufferSize, IOOptions(), &fragment, space,
+                   nullptr);
+    if (!s.ok()) {
+      break;
+    }
+    data->append(fragment.data(), fragment.size());
+    if (fragment.empty()) {
+      break;
+    }
+  }
+  delete[] space;
+  return s;
+}
+
+#ifdef OS_WIN
+std::shared_ptr<FileSystem> FileSystem::Default() {
+  static LegacyFileSystemWrapper default_fs(Env::Default());
+  static std::shared_ptr<LegacyFileSystemWrapper> default_fs_ptr(
+      &default_fs, [](LegacyFileSystemWrapper*) {});
+  return default_fs_ptr;
+}
+#endif
+
+}  // namespace rocksdb
@@ -0,0 +1,913 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors
+#include <dirent.h>
+#ifndef ROCKSDB_NO_DYNAMIC_EXTENSION
+#include <dlfcn.h>
+#endif
+#include <errno.h>
+#include <fcntl.h>
+
+#if defined(OS_LINUX)
+#include <linux/fs.h>
+#endif
+#include <pthread.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#if defined(OS_LINUX) || defined(OS_SOLARIS) || defined(OS_ANDROID)
+#include <sys/statfs.h>
+#include <sys/syscall.h>
+#include <sys/sysmacros.h>
+#endif
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <time.h>
+#include <algorithm>
+// Get nano time includes
+#if defined(OS_LINUX) || defined(OS_FREEBSD)
+#elif defined(__MACH__)
+#include <Availability.h>
+#include <mach/clock.h>
+#include <mach/mach.h>
+#else
+#include <chrono>
+#endif
+#include <deque>
+#include <set>
+#include <vector>
+
+#include "env/io_posix.h"
+#include "logging/logging.h"
+#include "logging/posix_logger.h"
+#include "monitoring/iostats_context_imp.h"
+#include "monitoring/thread_status_updater.h"
+#include "port/port.h"
+#include "rocksdb/options.h"
+#include "rocksdb/slice.h"
+#include "test_util/sync_point.h"
+#include "util/coding.h"
+#include "util/compression_context_cache.h"
+#include "util/random.h"
+#include "util/string_util.h"
+#include "util/thread_local.h"
+#include "util/threadpool_imp.h"
+
+#if !defined(TMPFS_MAGIC)
+#define TMPFS_MAGIC 0x01021994
+#endif
+#if !defined(XFS_SUPER_MAGIC)
+#define XFS_SUPER_MAGIC 0x58465342
+#endif
+#if !defined(EXT4_SUPER_MAGIC)
+#define EXT4_SUPER_MAGIC 0xEF53
+#endif
+
+namespace rocksdb {
+
+namespace {
+
+inline mode_t GetDBFileMode(bool allow_non_owner_access) {
+  return allow_non_owner_access ? 0644 : 0600;
+}
+
+// list of pathnames that are locked
+static std::set<std::string> lockedFiles;
+static port::Mutex mutex_lockedFiles;
+
+static int LockOrUnlock(int fd, bool lock) {
+  errno = 0;
+  struct flock f;
+  memset(&f, 0, sizeof(f));
+  f.l_type = (lock ? F_WRLCK : F_UNLCK);
+  f.l_whence = SEEK_SET;
+  f.l_start = 0;
+  f.l_len = 0;  // Lock/unlock entire file
+  int value = fcntl(fd, F_SETLK, &f);
+
+  return value;
+}
+
+class PosixFileLock : public FileLock {
+ public:
+  int fd_;
+  std::string filename;
+};
+
+int cloexec_flags(int flags, const EnvOptions* options) {
+  // If the system supports opening the file with cloexec enabled,
+  // do so, as this avoids a race condition if a db is opened around
+  // the same time that a child process is forked
+#ifdef O_CLOEXEC
+  if (options == nullptr || options->set_fd_cloexec) {
+    flags |= O_CLOEXEC;
+  }
+#endif
+  return flags;
+}
+
+class PosixFileSystem : public FileSystem {
+ public:
+  PosixFileSystem();
+
+  const char* Name() const override { return "Posix File System"; }
+
+  ~PosixFileSystem() override {}
+
+  void SetFD_CLOEXEC(int fd, const EnvOptions* options) {
+    if ((options == nullptr || options->set_fd_cloexec) && fd > 0) {
+      fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
+    }
+  }
+
+  IOStatus NewSequentialFile(const std::string& fname,
+                             const FileOptions& options,
+                             std::unique_ptr<FSSequentialFile>* result,
+                             IODebugContext* /*dbg*/) override {
+    result->reset();
+    int fd = -1;
+    int flags = cloexec_flags(O_RDONLY, &options);
+    FILE* file = nullptr;
+
+    if (options.use_direct_reads && !options.use_mmap_reads) {
+#ifdef ROCKSDB_LITE
+      return IOStatus::IOError(fname,
+                               "Direct I/O not supported in RocksDB lite");
+#endif  // !ROCKSDB_LITE
+#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
+      flags |= O_DIRECT;
+#endif
+    }
+
+    do {
+      IOSTATS_TIMER_GUARD(open_nanos);
+      fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_));
+    } while (fd < 0 && errno == EINTR);
+    if (fd < 0) {
+      return IOError("While opening a file for sequentially reading", fname,
+                     errno);
+    }
+
+    SetFD_CLOEXEC(fd, &options);
+
+    if (options.use_direct_reads && !options.use_mmap_reads) {
+#ifdef OS_MACOSX
+      if (fcntl(fd, F_NOCACHE, 1) == -1) {
+        close(fd);
+        return IOError("While fcntl NoCache", fname, errno);
+      }
+#endif
+    } else {
+      do {
+        IOSTATS_TIMER_GUARD(open_nanos);
+        file = fdopen(fd, "r");
+      } while (file == nullptr && errno == EINTR);
+      if (file == nullptr) {
+        close(fd);
+        return IOError("While opening file for sequentially read", fname,
+                       errno);
+      }
+    }
+    result->reset(new PosixSequentialFile(fname, file, fd, options));
+    return IOStatus::OK();
+  }
+
+  IOStatus NewRandomAccessFile(const std::string& fname,
+                               const FileOptions& options,
+                               std::unique_ptr<FSRandomAccessFile>* result,
+                               IODebugContext* /*dbg*/) override {
+    result->reset();
+    IOStatus s;
+    int fd;
+    int flags = cloexec_flags(O_RDONLY, &options);
+
+    if (options.use_direct_reads && !options.use_mmap_reads) {
+#ifdef ROCKSDB_LITE
+      return IOStatus::IOError(fname,
+                               "Direct I/O not supported in RocksDB lite");
+#endif  // !ROCKSDB_LITE
+#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
+      flags |= O_DIRECT;
+      TEST_SYNC_POINT_CALLBACK("NewRandomAccessFile:O_DIRECT", &flags);
+#endif
+    }
+
+    do {
+      IOSTATS_TIMER_GUARD(open_nanos);
+      fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_));
+    } while (fd < 0 && errno == EINTR);
+    if (fd < 0) {
+      return IOError("While open a file for random read", fname, errno);
+    }
+    SetFD_CLOEXEC(fd, &options);
+
+    if (options.use_mmap_reads && sizeof(void*) >= 8) {
+      // Use of mmap for random reads has been removed because it
+      // kills performance when storage is fast.
+      // Use mmap when virtual address-space is plentiful.
+      uint64_t size;
+      IOOptions opts;
+      s = GetFileSize(fname, opts, &size, nullptr);
+      if (s.ok()) {
+        void* base = mmap(nullptr, size, PROT_READ, MAP_SHARED, fd, 0);
+        if (base != MAP_FAILED) {
+          result->reset(
+              new PosixMmapReadableFile(fd, fname, base, size, options));
+        } else {
+          s = IOError("while mmap file for read", fname, errno);
+          close(fd);
+        }
+      }
+    } else {
+      if (options.use_direct_reads && !options.use_mmap_reads) {
+#ifdef OS_MACOSX
+        if (fcntl(fd, F_NOCACHE, 1) == -1) {
+          close(fd);
+          return IOError("while fcntl NoCache", fname, errno);
+        }
+#endif
+      }
+      result->reset(new PosixRandomAccessFile(fname, fd, options
+#if defined(ROCKSDB_IOURING_PRESENT)
+                                              ,
+                                              thread_local_io_urings_.get()
+#endif
+                                                  ));
+    }
+    return s;
+  }
+
+  virtual IOStatus OpenWritableFile(const std::string& fname,
+                                    const FileOptions& options,
+                                    bool reopen,
+                                    std::unique_ptr<FSWritableFile>* result,
+                                    IODebugContext* /*dbg*/) {
+    result->reset();
+    IOStatus s;
+    int fd = -1;
+    int flags = (reopen) ? (O_CREAT | O_APPEND) : (O_CREAT | O_TRUNC);
+    // Direct IO mode with O_DIRECT flag or F_NOCAHCE (MAC OSX)
+    if (options.use_direct_writes && !options.use_mmap_writes) {
+      // Note: we should avoid O_APPEND here due to ta the following bug:
+      // POSIX requires that opening a file with the O_APPEND flag should
+      // have no affect on the location at which pwrite() writes data.
+      // However, on Linux, if a file is opened with O_APPEND, pwrite()
+      // appends data to the end of the file, regardless of the value of
+      // offset.
+      // More info here: https://linux.die.net/man/2/pwrite
+#ifdef ROCKSDB_LITE
+      return IOStatus::IOError(fname,
+                               "Direct I/O not supported in RocksDB lite");
+#endif  // ROCKSDB_LITE
+      flags |= O_WRONLY;
+#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
+      flags |= O_DIRECT;
+#endif
+      TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags);
+    } else if (options.use_mmap_writes) {
+      // non-direct I/O
+      flags |= O_RDWR;
+    } else {
+      flags |= O_WRONLY;
+    }
+
+    flags = cloexec_flags(flags, &options);
+
+    do {
+      IOSTATS_TIMER_GUARD(open_nanos);
+      fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_));
+    } while (fd < 0 && errno == EINTR);
+
+    if (fd < 0) {
+      s = IOError("While open a file for appending", fname, errno);
+      return s;
+    }
+    SetFD_CLOEXEC(fd, &options);
+
+    if (options.use_mmap_writes) {
+      if (!checkedDiskForMmap_) {
+        // this will be executed once in the program's lifetime.
+        // do not use mmapWrite on non ext-3/xfs/tmpfs systems.
+        if (!SupportsFastAllocate(fname)) {
+          forceMmapOff_ = true;
+        }
+        checkedDiskForMmap_ = true;
+      }
+    }
+    if (options.use_mmap_writes && !forceMmapOff_) {
+      result->reset(new PosixMmapFile(fname, fd, page_size_, options));
+    } else if (options.use_direct_writes && !options.use_mmap_writes) {
+#ifdef OS_MACOSX
+      if (fcntl(fd, F_NOCACHE, 1) == -1) {
+        close(fd);
+        s = IOError("While fcntl NoCache an opened file for appending", fname,
+                    errno);
+        return s;
+      }
+#elif defined(OS_SOLARIS)
+      if (directio(fd, DIRECTIO_ON) == -1) {
+        if (errno != ENOTTY) {  // ZFS filesystems don't support DIRECTIO_ON
+          close(fd);
+          s = IOError("While calling directio()", fname, errno);
+          return s;
+        }
+      }
+#endif
+      result->reset(new PosixWritableFile(fname, fd, options));
+    } else {
+      // disable mmap writes
+      EnvOptions no_mmap_writes_options = options;
+      no_mmap_writes_options.use_mmap_writes = false;
+      result->reset(new PosixWritableFile(fname, fd, no_mmap_writes_options));
+    }
+    return s;
+  }
+
+  IOStatus NewWritableFile(const std::string& fname, const FileOptions& options,
+                           std::unique_ptr<FSWritableFile>* result,
+                           IODebugContext* dbg) override {
+    return OpenWritableFile(fname, options, false, result, dbg);
+  }
+
+  IOStatus ReopenWritableFile(const std::string& fname,
+                              const FileOptions& options,
+                              std::unique_ptr<FSWritableFile>* result,
+                              IODebugContext* dbg) override {
+    return OpenWritableFile(fname, options, true, result, dbg);
+  }
+
+  IOStatus ReuseWritableFile(const std::string& fname,
+                             const std::string& old_fname,
+                             const FileOptions& options,
+                             std::unique_ptr<FSWritableFile>* result,
+                             IODebugContext* /*dbg*/) override {
+    result->reset();
+    IOStatus s;
+    int fd = -1;
+
+    int flags = 0;
+    // Direct IO mode with O_DIRECT flag or F_NOCAHCE (MAC OSX)
+    if (options.use_direct_writes && !options.use_mmap_writes) {
+#ifdef ROCKSDB_LITE
+      return IOStatus::IOError(fname,
+                               "Direct I/O not supported in RocksDB lite");
+#endif  // !ROCKSDB_LITE
+      flags |= O_WRONLY;
+#if !defined(OS_MACOSX) && !defined(OS_OPENBSD) && !defined(OS_SOLARIS)
+      flags |= O_DIRECT;
+#endif
+      TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags);
+    } else if (options.use_mmap_writes) {
+      // mmap needs O_RDWR mode
+      flags |= O_RDWR;
+    } else {
+      flags |= O_WRONLY;
+    }
+
+    flags = cloexec_flags(flags, &options);
+
+    do {
+      IOSTATS_TIMER_GUARD(open_nanos);
+      fd = open(old_fname.c_str(), flags,
+                GetDBFileMode(allow_non_owner_access_));
+    } while (fd < 0 && errno == EINTR);
+    if (fd < 0) {
+      s = IOError("while reopen file for write", fname, errno);
+      return s;
+    }
+
+    SetFD_CLOEXEC(fd, &options);
+    // rename into place
+    if (rename(old_fname.c_str(), fname.c_str()) != 0) {
+      s = IOError("while rename file to " + fname, old_fname, errno);
+      close(fd);
+      return s;
+    }
+
+    if (options.use_mmap_writes) {
+      if (!checkedDiskForMmap_) {
+        // this will be executed once in the program's lifetime.
+        // do not use mmapWrite on non ext-3/xfs/tmpfs systems.
+        if (!SupportsFastAllocate(fname)) {
+          forceMmapOff_ = true;
+        }
+        checkedDiskForMmap_ = true;
+      }
+    }
+    if (options.use_mmap_writes && !forceMmapOff_) {
+      result->reset(new PosixMmapFile(fname, fd, page_size_, options));
+    } else if (options.use_direct_writes && !options.use_mmap_writes) {
+#ifdef OS_MACOSX
+      if (fcntl(fd, F_NOCACHE, 1) == -1) {
+        close(fd);
+        s = IOError("while fcntl NoCache for reopened file for append", fname,
+                    errno);
+        return s;
+      }
+#elif defined(OS_SOLARIS)
+      if (directio(fd, DIRECTIO_ON) == -1) {
+        if (errno != ENOTTY) {  // ZFS filesystems don't support DIRECTIO_ON
+          close(fd);
+          s = IOError("while calling directio()", fname, errno);
+          return s;
+        }
+      }
+#endif
+      result->reset(new PosixWritableFile(fname, fd, options));
+    } else {
+      // disable mmap writes
+      FileOptions no_mmap_writes_options = options;
+      no_mmap_writes_options.use_mmap_writes = false;
+      result->reset(new PosixWritableFile(fname, fd, no_mmap_writes_options));
+    }
+    return s;
+  }
+
+  IOStatus NewRandomRWFile(const std::string& fname, const FileOptions& options,
+                           std::unique_ptr<FSRandomRWFile>* result,
+                           IODebugContext* /*dbg*/) override {
+    int fd = -1;
+    int flags = cloexec_flags(O_RDWR, &options);
+
+    while (fd < 0) {
+      IOSTATS_TIMER_GUARD(open_nanos);
+
+      fd = open(fname.c_str(), flags, GetDBFileMode(allow_non_owner_access_));
+      if (fd < 0) {
+        // Error while opening the file
+        if (errno == EINTR) {
+          continue;
+        }
+        return IOError("While open file for random read/write", fname, errno);
+      }
+    }
+
+    SetFD_CLOEXEC(fd, &options);
+    result->reset(new PosixRandomRWFile(fname, fd, options));
+    return IOStatus::OK();
+  }
+
+  IOStatus NewMemoryMappedFileBuffer(
+      const std::string& fname,
+      std::unique_ptr<MemoryMappedFileBuffer>* result) override {
+    int fd = -1;
+    IOStatus status;
+    int flags = cloexec_flags(O_RDWR, nullptr);
+
+    while (fd < 0) {
+      IOSTATS_TIMER_GUARD(open_nanos);
+      fd = open(fname.c_str(), flags, 0644);
+      if (fd < 0) {
+        // Error while opening the file
+        if (errno == EINTR) {
+          continue;
+        }
+        status =
+            IOError("While open file for raw mmap buffer access", fname, errno);
+        break;
+      }
+    }
+    uint64_t size;
+    if (status.ok()) {
+      IOOptions opts;
+      status = GetFileSize(fname, opts, &size, nullptr);
+    }
+    void* base = nullptr;
+    if (status.ok()) {
+      base = mmap(nullptr, static_cast<size_t>(size), PROT_READ | PROT_WRITE,
+                  MAP_SHARED, fd, 0);
+      if (base == MAP_FAILED) {
+        status = IOError("while mmap file for read", fname, errno);
+      }
+    }
+    if (status.ok()) {
+      result->reset(
+          new PosixMemoryMappedFileBuffer(base, static_cast<size_t>(size)));
+    }
+    if (fd >= 0) {
+      // don't need to keep it open after mmap has been called
+      close(fd);
+    }
+    return status;
+  }
+
+  IOStatus NewDirectory(const std::string& name, const IOOptions& /*opts*/,
+                        std::unique_ptr<FSDirectory>* result,
+                        IODebugContext* /*dbg*/) override {
+    result->reset();
+    int fd;
+    int flags = cloexec_flags(0, nullptr);
+    {
+      IOSTATS_TIMER_GUARD(open_nanos);
+      fd = open(name.c_str(), flags);
+    }
+    if (fd < 0) {
+      return IOError("While open directory", name, errno);
+    } else {
+      result->reset(new PosixDirectory(fd));
+    }
+    return IOStatus::OK();
+  }
+
+  IOStatus NewLogger(const std::string& /*fname*/, const IOOptions& /*opts*/,
+                     std::shared_ptr<rocksdb::Logger>* /*ptr*/,
+                     IODebugContext* /*dbg*/) override {
+    return IOStatus::NotSupported();
+  }
+
+  IOStatus FileExists(const std::string& fname, const IOOptions& /*opts*/,
+                      IODebugContext* /*dbg*/) override {
+    int result = access(fname.c_str(), F_OK);
+
+    if (result == 0) {
+      return IOStatus::OK();
+    }
+
+    int err = errno;
+    switch (err) {
+      case EACCES:
+      case ELOOP:
+      case ENAMETOOLONG:
+      case ENOENT:
+      case ENOTDIR:
+        return IOStatus::NotFound();
+      default:
+        assert(err == EIO || err == ENOMEM);
+        return IOStatus::IOError("Unexpected error(" + ToString(err) +
+                                 ") accessing file `" + fname + "' ");
+    }
+  }
+
+  IOStatus GetChildren(const std::string& dir, const IOOptions& /*opts*/,
+                       std::vector<std::string>* result,
+                       IODebugContext* /*dbg*/) override {
+    result->clear();
+    DIR* d = opendir(dir.c_str());
+    if (d == nullptr) {
+      switch (errno) {
+        case EACCES:
+        case ENOENT:
+        case ENOTDIR:
+          return IOStatus::NotFound();
+        default:
+          return IOError("While opendir", dir, errno);
+      }
+    }
+    struct dirent* entry;
+    while ((entry = readdir(d)) != nullptr) {
+      result->push_back(entry->d_name);
+    }
+    closedir(d);
+    return IOStatus::OK();
+  }
+
+  IOStatus DeleteFile(const std::string& fname, const IOOptions& /*opts*/,
+                      IODebugContext* /*dbg*/) override {
+    IOStatus result;
+    if (unlink(fname.c_str()) != 0) {
+      result = IOError("while unlink() file", fname, errno);
+    }
+    return result;
+  }
+
+  IOStatus CreateDir(const std::string& name, const IOOptions& /*opts*/,
+                     IODebugContext* /*dbg*/) override {
+    IOStatus result;
+    if (mkdir(name.c_str(), 0755) != 0) {
+      result = IOError("While mkdir", name, errno);
+    }
+    return result;
+  }
+
+  IOStatus CreateDirIfMissing(const std::string& name,
+                              const IOOptions& /*opts*/,
+                              IODebugContext* /*dbg*/) override {
+    IOStatus result;
+    if (mkdir(name.c_str(), 0755) != 0) {
+      if (errno != EEXIST) {
+        result = IOError("While mkdir if missing", name, errno);
+      } else if (!DirExists(name)) {  // Check that name is actually a
+                                      // directory.
+        // Message is taken from mkdir
+        result =
+            IOStatus::IOError("`" + name + "' exists but is not a directory");
+      }
+    }
+    return result;
+  }
+
+  IOStatus DeleteDir(const std::string& name, const IOOptions& /*opts*/,
+                     IODebugContext* /*dbg*/) override {
+    IOStatus result;
+    if (rmdir(name.c_str()) != 0) {
+      result = IOError("file rmdir", name, errno);
+    }
+    return result;
+  }
+
+  IOStatus GetFileSize(const std::string& fname, const IOOptions& /*opts*/,
+                       uint64_t* size, IODebugContext* /*dbg*/) override {
+    IOStatus s;
+    struct stat sbuf;
+    if (stat(fname.c_str(), &sbuf) != 0) {
+      *size = 0;
+      s = IOError("while stat a file for size", fname, errno);
+    } else {
+      *size = sbuf.st_size;
+    }
+    return s;
+  }
+
+  IOStatus GetFileModificationTime(const std::string& fname,
+                                   const IOOptions& /*opts*/,
+                                   uint64_t* file_mtime,
+                                   IODebugContext* /*dbg*/) override {
+    struct stat s;
+    if (stat(fname.c_str(), &s) != 0) {
+      return IOError("while stat a file for modification time", fname, errno);
+    }
+    *file_mtime = static_cast<uint64_t>(s.st_mtime);
+    return IOStatus::OK();
+  }
+
+  IOStatus RenameFile(const std::string& src, const std::string& target,
+                      const IOOptions& /*opts*/,
+                      IODebugContext* /*dbg*/) override {
+    IOStatus result;
+    if (rename(src.c_str(), target.c_str()) != 0) {
+      result = IOError("While renaming a file to " + target, src, errno);
+    }
+    return result;
+  }
+
+  IOStatus LinkFile(const std::string& src, const std::string& target,
+                    const IOOptions& /*opts*/,
+                    IODebugContext* /*dbg*/) override {
+    IOStatus result;
+    if (link(src.c_str(), target.c_str()) != 0) {
+      if (errno == EXDEV) {
+        return IOStatus::NotSupported("No cross FS links allowed");
+      }
+      result = IOError("while link file to " + target, src, errno);
+    }
+    return result;
+  }
+
+  IOStatus NumFileLinks(const std::string& fname, const IOOptions& /*opts*/,
+                        uint64_t* count, IODebugContext* /*dbg*/) override {
+    struct stat s;
+    if (stat(fname.c_str(), &s) != 0) {
+      return IOError("while stat a file for num file links", fname, errno);
+    }
+    *count = static_cast<uint64_t>(s.st_nlink);
+    return IOStatus::OK();
+  }
+
+  IOStatus AreFilesSame(const std::string& first, const std::string& second,
+                        const IOOptions& /*opts*/, bool* res,
+                        IODebugContext* /*dbg*/) override {
+    struct stat statbuf[2];
+    if (stat(first.c_str(), &statbuf[0]) != 0) {
+      return IOError("stat file", first, errno);
+    }
+    if (stat(second.c_str(), &statbuf[1]) != 0) {
+      return IOError("stat file", second, errno);
+    }
+
+    if (major(statbuf[0].st_dev) != major(statbuf[1].st_dev) ||
+        minor(statbuf[0].st_dev) != minor(statbuf[1].st_dev) ||
+        statbuf[0].st_ino != statbuf[1].st_ino) {
+      *res = false;
+    } else {
+      *res = true;
+    }
+    return IOStatus::OK();
+  }
+
+  IOStatus LockFile(const std::string& fname, const IOOptions& /*opts*/,
+                    FileLock** lock, IODebugContext* /*dbg*/) override {
+    *lock = nullptr;
+    IOStatus result;
+
+    mutex_lockedFiles.Lock();
+    // If it already exists in the lockedFiles set, then it is already locked,
+    // and fail this lock attempt. Otherwise, insert it into lockedFiles.
+    // This check is needed because fcntl() does not detect lock conflict
+    // if the fcntl is issued by the same thread that earlier acquired
+    // this lock.
+    // We must do this check *before* opening the file:
+    // Otherwise, we will open a new file descriptor. Locks are associated with
+    // a process, not a file descriptor and when *any* file descriptor is
+    // closed, all locks the process holds for that *file* are released
+    if (lockedFiles.insert(fname).second == false) {
+      mutex_lockedFiles.Unlock();
+      errno = ENOLCK;
+      return IOError("lock ", fname, errno);
+    }
+
+    int fd;
+    int flags = cloexec_flags(O_RDWR | O_CREAT, nullptr);
+
+    {
+      IOSTATS_TIMER_GUARD(open_nanos);
+      fd = open(fname.c_str(), flags, 0644);
+    }
+    if (fd < 0) {
+      result = IOError("while open a file for lock", fname, errno);
+    } else if (LockOrUnlock(fd, true) == -1) {
+      // if there is an error in locking, then remove the pathname from
+      // lockedfiles
+      lockedFiles.erase(fname);
+      result = IOError("While lock file", fname, errno);
+      close(fd);
+    } else {
+      SetFD_CLOEXEC(fd, nullptr);
+      PosixFileLock* my_lock = new PosixFileLock;
+      my_lock->fd_ = fd;
+      my_lock->filename = fname;
+      *lock = my_lock;
+    }
+
+    mutex_lockedFiles.Unlock();
+    return result;
+  }
+
+  IOStatus UnlockFile(FileLock* lock, const IOOptions& /*opts*/,
+                      IODebugContext* /*dbg*/) override {
+    PosixFileLock* my_lock = reinterpret_cast<PosixFileLock*>(lock);
+    IOStatus result;
+    mutex_lockedFiles.Lock();
+    // If we are unlocking, then verify that we had locked it earlier,
+    // it should already exist in lockedFiles. Remove it from lockedFiles.
+    if (lockedFiles.erase(my_lock->filename) != 1) {
+      errno = ENOLCK;
+      result = IOError("unlock", my_lock->filename, errno);
+    } else if (LockOrUnlock(my_lock->fd_, false) == -1) {
+      result = IOError("unlock", my_lock->filename, errno);
+    }
+    close(my_lock->fd_);
+    delete my_lock;
+    mutex_lockedFiles.Unlock();
+    return result;
+  }
+
+  IOStatus GetAbsolutePath(const std::string& db_path,
+                           const IOOptions& /*opts*/, std::string* output_path,
+                           IODebugContext* /*dbg*/) override {
+    if (!db_path.empty() && db_path[0] == '/') {
+      *output_path = db_path;
+      return IOStatus::OK();
+    }
+
+    char the_path[256];
+    char* ret = getcwd(the_path, 256);
+    if (ret == nullptr) {
+      return IOStatus::IOError(strerror(errno));
+    }
+
+    *output_path = ret;
+    return IOStatus::OK();
+  }
+
+  IOStatus GetTestDirectory(const IOOptions& /*opts*/, std::string* result,
+                            IODebugContext* /*dbg*/) override {
+    const char* env = getenv("TEST_TMPDIR");
+    if (env && env[0] != '\0') {
+      *result = env;
+    } else {
+      char buf[100];
+      snprintf(buf, sizeof(buf), "/tmp/rocksdbtest-%d", int(geteuid()));
+      *result = buf;
+    }
+    // Directory may already exist
+    {
+      IOOptions opts;
+      CreateDir(*result, opts, nullptr);
+    }
+    return IOStatus::OK();
+  }
+
+  IOStatus GetFreeSpace(const std::string& fname, const IOOptions& /*opts*/,
+                        uint64_t* free_space,
+                        IODebugContext* /*dbg*/) override {
+    struct statvfs sbuf;
+
+    if (statvfs(fname.c_str(), &sbuf) < 0) {
+      return IOError("While doing statvfs", fname, errno);
+    }
+
+    *free_space = ((uint64_t)sbuf.f_bsize * sbuf.f_bfree);
+    return IOStatus::OK();
+  }
+
+  FileOptions OptimizeForLogWrite(const FileOptions& file_options,
+                                 const DBOptions& db_options) const override {
+    FileOptions optimized = file_options;
+    optimized.use_mmap_writes = false;
+    optimized.use_direct_writes = false;
+    optimized.bytes_per_sync = db_options.wal_bytes_per_sync;
+    // TODO(icanadi) it's faster if fallocate_with_keep_size is false, but it
+    // breaks TransactionLogIteratorStallAtLastRecord unit test. Fix the unit
+    // test and make this false
+    optimized.fallocate_with_keep_size = true;
+    optimized.writable_file_max_buffer_size =
+        db_options.writable_file_max_buffer_size;
+    return optimized;
+  }
+
+  FileOptions OptimizeForManifestWrite(
+      const FileOptions& file_options) const override {
+    FileOptions optimized = file_options;
+    optimized.use_mmap_writes = false;
+    optimized.use_direct_writes = false;
+    optimized.fallocate_with_keep_size = true;
+    return optimized;
+  }
+
+ private:
+  bool checkedDiskForMmap_;
+  bool forceMmapOff_;  // do we override Env options?
+
+  // Returns true iff the named directory exists and is a directory.
+  virtual bool DirExists(const std::string& dname) {
+    struct stat statbuf;
+    if (stat(dname.c_str(), &statbuf) == 0) {
+      return S_ISDIR(statbuf.st_mode);
+    }
+    return false;  // stat() failed return false
+  }
+
+  bool SupportsFastAllocate(const std::string& path) {
+#ifdef ROCKSDB_FALLOCATE_PRESENT
+    struct statfs s;
+    if (statfs(path.c_str(), &s)) {
+      return false;
+    }
+    switch (s.f_type) {
+      case EXT4_SUPER_MAGIC:
+        return true;
+      case XFS_SUPER_MAGIC:
+        return true;
+      case TMPFS_MAGIC:
+        return true;
+      default:
+        return false;
+    }
+#else
+    (void)path;
+    return false;
+#endif
+  }
+
+#if defined(ROCKSDB_IOURING_PRESENT)
+  // io_uring instance
+  std::unique_ptr<ThreadLocalPtr> thread_local_io_urings_;
+#endif
+
+  size_t page_size_;
+
+  // If true, allow non owner read access for db files. Otherwise, non-owner
+  //  has no access to db files.
+  bool allow_non_owner_access_;
+};
+
+PosixFileSystem::PosixFileSystem()
+    : checkedDiskForMmap_(false),
+      forceMmapOff_(false),
+      page_size_(getpagesize()),
+      allow_non_owner_access_(true) {
+#if defined(ROCKSDB_IOURING_PRESENT)
+  // Test whether IOUring is supported, and if it does, create a managing
+  // object for thread local point so that in the future thread-local
+  // io_uring can be created.
+  struct io_uring* new_io_uring = CreateIOUring();
+  if (new_io_uring != nullptr) {
+    thread_local_io_urings_.reset(new ThreadLocalPtr(DeleteIOUring));
+    delete new_io_uring;
+  }
+#endif
+}
+
+}  // namespace
+
+//
+// Default Posix FileSystem
+//
+std::shared_ptr<FileSystem> FileSystem::Default() {
+  static PosixFileSystem default_fs;
+  static std::shared_ptr<PosixFileSystem> default_fs_ptr(
+      &default_fs, [](PosixFileSystem*) {});
+  return default_fs_ptr;
+}
+
+}  // namespace rocksdb
@@ -266,9 +266,11 @@ PosixSequentialFile::~PosixSequentialFile() {
  }
 }

-Status PosixSequentialFile::Read(size_t n, Slice* result, char* scratch) {
+IOStatus PosixSequentialFile::Read(size_t n, const IOOptions& /*opts*/,
+                                   Slice* result, char* scratch,
+                                   IODebugContext* /*dbg*/) {
  assert(result != nullptr && !use_direct_io());
-  Status s;
+  IOStatus s;
  size_t r = 0;
  do {
    r = fread_unlocked(scratch, 1, n, file_);
@@ -288,14 +290,16 @@ Status PosixSequentialFile::Read(size_t n, Slice* result, char* scratch) {
  return s;
 }

-Status PosixSequentialFile::PositionedRead(uint64_t offset, size_t n,
-                                           Slice* result, char* scratch) {
+IOStatus PosixSequentialFile::PositionedRead(uint64_t offset, size_t n,
+                                             const IOOptions& /*opts*/,
+                                             Slice* result, char* scratch,
+                                             IODebugContext* /*dbg*/) {
  assert(use_direct_io());
  assert(IsSectorAligned(offset, GetRequiredBufferAlignment()));
  assert(IsSectorAligned(n, GetRequiredBufferAlignment()));
  assert(IsSectorAligned(scratch, GetRequiredBufferAlignment()));

-  Status s;
+  IOStatus s;
  ssize_t r = -1;
  size_t left = n;
  char* ptr = scratch;
@@ -326,19 +330,19 @@ Status PosixSequentialFile::PositionedRead(uint64_t offset, size_t n,
  return s;
 }

-Status PosixSequentialFile::Skip(uint64_t n) {
+IOStatus PosixSequentialFile::Skip(uint64_t n) {
  if (fseek(file_, static_cast<long int>(n), SEEK_CUR)) {
    return IOError("While fseek to skip " + ToString(n) + " bytes", filename_,
                   errno);
  }
-  return Status::OK();
+  return IOStatus::OK();
 }

-Status PosixSequentialFile::InvalidateCache(size_t offset, size_t length) {
+IOStatus PosixSequentialFile::InvalidateCache(size_t offset, size_t length) {
 #ifndef OS_LINUX
  (void)offset;
  (void)length;
-  return Status::OK();
+  return IOStatus::OK();
 #else
  if (!use_direct_io()) {
    // free OS pages
@@ -349,7 +353,7 @@ Status PosixSequentialFile::InvalidateCache(size_t offset, size_t length) {
                     filename_, errno);
    }
  }
-  return Status::OK();
+  return IOStatus::OK();
 #endif
 }

@@ -432,14 +436,16 @@ PosixRandomAccessFile::PosixRandomAccessFile(

 PosixRandomAccessFile::~PosixRandomAccessFile() { close(fd_); }

-Status PosixRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result,
-                                   char* scratch) const {
+IOStatus PosixRandomAccessFile::Read(uint64_t offset, size_t n,
+                                     const IOOptions& /*opts*/, Slice* result,
+                                     char* scratch,
+                                     IODebugContext* /*dbg*/) const {
  if (use_direct_io()) {
    assert(IsSectorAligned(offset, GetRequiredBufferAlignment()));
    assert(IsSectorAligned(n, GetRequiredBufferAlignment()));
    assert(IsSectorAligned(scratch, GetRequiredBufferAlignment()));
  }
-  Status s;
+  IOStatus s;
  ssize_t r = -1;
  size_t left = n;
  char* ptr = scratch;
@@ -471,7 +477,10 @@ Status PosixRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result,
  return s;
 }

-Status PosixRandomAccessFile::MultiRead(ReadRequest* reqs, size_t num_reqs) {
+IOStatus PosixRandomAccessFile::MultiRead(FSReadRequest* reqs,
+                                          size_t num_reqs,
+                                          const IOOptions& options,
+                                          IODebugContext* dbg) {
 #if defined(ROCKSDB_IOURING_PRESENT)
  size_t reqs_off;
  ssize_t ret __attribute__((__unused__));
@@ -490,13 +499,13 @@ Status PosixRandomAccessFile::MultiRead(ReadRequest* reqs, size_t num_reqs) {
  // Init failed, platform doesn't support io_uring. Fall back to
  // serialized reads
  if (iu == nullptr) {
-    return RandomAccessFile::MultiRead(reqs, num_reqs);
+    return FSRandomAccessFile::MultiRead(reqs, num_reqs, options, dbg);
  }

  struct WrappedReadRequest {
-    ReadRequest* req;
+    FSReadRequest* req;
    struct iovec iov;
-    explicit WrappedReadRequest(ReadRequest* r) : req(r) {}
+    explicit WrappedReadRequest(FSReadRequest* r) : req(r) {}
  };

  autovector<WrappedReadRequest, 32> req_wraps;
@@ -539,10 +548,10 @@ Status PosixRandomAccessFile::MultiRead(ReadRequest* reqs, size_t num_reqs) {
      ret = io_uring_wait_cqe(iu, &cqe);
      assert(!ret);
      req_wrap = static_cast<WrappedReadRequest*>(io_uring_cqe_get_data(cqe));
-      ReadRequest* req = req_wrap->req;
+      FSReadRequest* req = req_wrap->req;
      if (static_cast<size_t>(cqe->res) == req_wrap->iov.iov_len) {
        req->result = Slice(req->scratch, cqe->res);
-        req->status = Status::OK();
+        req->status = IOStatus::OK();
      } else if (cqe->res >= 0) {
        req->result = Slice(req->scratch, req_wrap->iov.iov_len - cqe->res);
      } else {
@@ -554,14 +563,16 @@ Status PosixRandomAccessFile::MultiRead(ReadRequest* reqs, size_t num_reqs) {
    num_reqs -= this_reqs;
    reqs_off += this_reqs;
  }
-  return Status::OK();
+  return IOStatus::OK();
 #else
-  return RandomAccessFile::MultiRead(reqs, num_reqs);
+  return FSRandomAccessFile::MultiRead(reqs, num_reqs, options, dbg);
 #endif
 }

-Status PosixRandomAccessFile::Prefetch(uint64_t offset, size_t n) {
-  Status s;
+IOStatus PosixRandomAccessFile::Prefetch(uint64_t offset, size_t n,
+                                         const IOOptions& /*opts*/,
+                                         IODebugContext* /*dbg*/) {
+  IOStatus s;
  if (!use_direct_io()) {
    ssize_t r = 0;
 #ifdef OS_LINUX
@@ -593,19 +604,19 @@ void PosixRandomAccessFile::Hint(AccessPattern pattern) {
    return;
  }
  switch (pattern) {
-    case NORMAL:
+    case kNormal:
      Fadvise(fd_, 0, 0, POSIX_FADV_NORMAL);
      break;
-    case RANDOM:
+    case kRandom:
      Fadvise(fd_, 0, 0, POSIX_FADV_RANDOM);
      break;
-    case SEQUENTIAL:
+    case kSequential:
      Fadvise(fd_, 0, 0, POSIX_FADV_SEQUENTIAL);
      break;
-    case WILLNEED:
+    case kWillNeed:
      Fadvise(fd_, 0, 0, POSIX_FADV_WILLNEED);
      break;
-    case DONTNEED:
+    case kWontNeed:
      Fadvise(fd_, 0, 0, POSIX_FADV_DONTNEED);
      break;
    default:
@@ -614,19 +625,19 @@ void PosixRandomAccessFile::Hint(AccessPattern pattern) {
  }
 }

-Status PosixRandomAccessFile::InvalidateCache(size_t offset, size_t length) {
+IOStatus PosixRandomAccessFile::InvalidateCache(size_t offset, size_t length) {
  if (use_direct_io()) {
-    return Status::OK();
+    return IOStatus::OK();
  }
 #ifndef OS_LINUX
  (void)offset;
  (void)length;
-  return Status::OK();
+  return IOStatus::OK();
 #else
  // free OS pages
  int ret = Fadvise(fd_, offset, length, POSIX_FADV_DONTNEED);
  if (ret == 0) {
-    return Status::OK();
+    return IOStatus::OK();
  }
  return IOError("While fadvise NotNeeded offset " + ToString(offset) +
                     " len " + ToString(length),
@@ -662,9 +673,11 @@ PosixMmapReadableFile::~PosixMmapReadableFile() {
  close(fd_);
 }

-Status PosixMmapReadableFile::Read(uint64_t offset, size_t n, Slice* result,
-                                   char* /*scratch*/) const {
-  Status s;
+IOStatus PosixMmapReadableFile::Read(uint64_t offset, size_t n,
+                                     const IOOptions& /*opts*/, Slice* result,
+                                     char* /*scratch*/,
+                                     IODebugContext* /*dbg*/) const {
+  IOStatus s;
  if (offset > length_) {
    *result = Slice();
    return IOError("While mmap read offset " + ToString(offset) +
@@ -677,16 +690,16 @@ Status PosixMmapReadableFile::Read(uint64_t offset, size_t n, Slice* result,
  return s;
 }

-Status PosixMmapReadableFile::InvalidateCache(size_t offset, size_t length) {
+IOStatus PosixMmapReadableFile::InvalidateCache(size_t offset, size_t length) {
 #ifndef OS_LINUX
  (void)offset;
  (void)length;
-  return Status::OK();
+  return IOStatus::OK();
 #else
  // free OS pages
  int ret = Fadvise(fd_, offset, length, POSIX_FADV_DONTNEED);
  if (ret == 0) {
-    return Status::OK();
+    return IOStatus::OK();
  }
  return IOError("While fadvise not needed. Offset " + ToString(offset) +
                     " len" + ToString(length),
@@ -702,7 +715,7 @@ Status PosixMmapReadableFile::InvalidateCache(size_t offset, size_t length) {
 * file before reading from it, or for log files, the reading code
 * knows enough to skip zero suffixes.
 */
-Status PosixMmapFile::UnmapCurrentRegion() {
+IOStatus PosixMmapFile::UnmapCurrentRegion() {
  TEST_KILL_RANDOM("PosixMmapFile::UnmapCurrentRegion:0", rocksdb_kill_odds);
  if (base_ != nullptr) {
    int munmap_status = munmap(base_, limit_ - base_);
@@ -720,10 +733,10 @@ Status PosixMmapFile::UnmapCurrentRegion() {
      map_size_ *= 2;
    }
  }
-  return Status::OK();
+  return IOStatus::OK();
 }

-Status PosixMmapFile::MapNewRegion() {
+IOStatus PosixMmapFile::MapNewRegion() {
 #ifdef ROCKSDB_FALLOCATE_PRESENT
  assert(base_ == nullptr);
  TEST_KILL_RANDOM("PosixMmapFile::UnmapCurrentRegion:0", rocksdb_kill_odds);
@@ -736,8 +749,8 @@ Status PosixMmapFile::MapNewRegion() {
      alloc_status = posix_fallocate(fd_, file_offset_, map_size_);
    }
    if (alloc_status != 0) {
-      return Status::IOError("Error allocating space to file : " + filename_ +
-                             "Error : " + strerror(alloc_status));
+      return IOStatus::IOError("Error allocating space to file : " + filename_ +
+                               "Error : " + strerror(alloc_status));
    }
  }

@@ -745,7 +758,7 @@ Status PosixMmapFile::MapNewRegion() {
  void* ptr = mmap(nullptr, map_size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_,
                   file_offset_);
  if (ptr == MAP_FAILED) {
-    return Status::IOError("MMap failed on " + filename_);
+    return IOStatus::IOError("MMap failed on " + filename_);
  }
  TEST_KILL_RANDOM("PosixMmapFile::Append:2", rocksdb_kill_odds);

@@ -753,15 +766,15 @@ Status PosixMmapFile::MapNewRegion() {
  limit_ = base_ + map_size_;
  dst_ = base_;
  last_sync_ = base_;
-  return Status::OK();
+  return IOStatus::OK();
 #else
-  return Status::NotSupported("This platform doesn't support fallocate()");
+  return IOStatus::NotSupported("This platform doesn't support fallocate()");
 #endif
 }

-Status PosixMmapFile::Msync() {
+IOStatus PosixMmapFile::Msync() {
  if (dst_ == last_sync_) {
-    return Status::OK();
+    return IOStatus::OK();
  }
  // Find the beginnings of the pages that contain the first and last
  // bytes to be synced.
@@ -772,7 +785,7 @@ Status PosixMmapFile::Msync() {
  if (msync(base_ + p1, p2 - p1 + page_size_, MS_SYNC) < 0) {
    return IOError("While msync", filename_, errno);
  }
-  return Status::OK();
+  return IOStatus::OK();
 }

 PosixMmapFile::PosixMmapFile(const std::string& fname, int fd, size_t page_size,
@@ -799,11 +812,12 @@ PosixMmapFile::PosixMmapFile(const std::string& fname, int fd, size_t page_size,

 PosixMmapFile::~PosixMmapFile() {
  if (fd_ >= 0) {
-    PosixMmapFile::Close();
+    PosixMmapFile::Close(IOOptions(), nullptr);
  }
 }

-Status PosixMmapFile::Append(const Slice& data) {
+IOStatus PosixMmapFile::Append(const Slice& data, const IOOptions& /*opts*/,
+                               IODebugContext* /*dbg*/) {
  const char* src = data.data();
  size_t left = data.size();
  while (left > 0) {
@@ -811,7 +825,7 @@ Status PosixMmapFile::Append(const Slice& data) {
    assert(dst_ <= limit_);
    size_t avail = limit_ - dst_;
    if (avail == 0) {
-      Status s = UnmapCurrentRegion();
+      IOStatus s = UnmapCurrentRegion();
      if (!s.ok()) {
        return s;
      }
@@ -829,11 +843,12 @@ Status PosixMmapFile::Append(const Slice& data) {
    src += n;
    left -= n;
  }
-  return Status::OK();
+  return IOStatus::OK();
 }

-Status PosixMmapFile::Close() {
-  Status s;
+IOStatus PosixMmapFile::Close(const IOOptions& /*opts*/,
+                              IODebugContext* /*dbg*/) {
+  IOStatus s;
  size_t unused = limit_ - dst_;

  s = UnmapCurrentRegion();
@@ -858,9 +873,13 @@ Status PosixMmapFile::Close() {
  return s;
 }

-Status PosixMmapFile::Flush() { return Status::OK(); }
+IOStatus PosixMmapFile::Flush(const IOOptions& /*opts*/,
+                              IODebugContext* /*dbg*/) {
+  return IOStatus::OK();
+}

-Status PosixMmapFile::Sync() {
+IOStatus PosixMmapFile::Sync(const IOOptions& /*opts*/,
+                             IODebugContext* /*dbg*/) {
  if (fdatasync(fd_) < 0) {
    return IOError("While fdatasync mmapped file", filename_, errno);
  }
@@ -871,7 +890,8 @@ Status PosixMmapFile::Sync() {
 /**
 * Flush data as well as metadata to stable storage.
 */
-Status PosixMmapFile::Fsync() {
+IOStatus PosixMmapFile::Fsync(const IOOptions& /*opts*/,
+                              IODebugContext* /*dbg*/) {
  if (fsync(fd_) < 0) {
    return IOError("While fsync mmaped file", filename_, errno);
  }
@@ -884,28 +904,31 @@ Status PosixMmapFile::Fsync() {
 * size that is returned from the filesystem because we use mmap
 * to extend file by map_size every time.
 */
-uint64_t PosixMmapFile::GetFileSize() {
+uint64_t PosixMmapFile::GetFileSize(const IOOptions& /*opts*/,
+                                    IODebugContext* /*dbg*/) {
  size_t used = dst_ - base_;
  return file_offset_ + used;
 }

-Status PosixMmapFile::InvalidateCache(size_t offset, size_t length) {
+IOStatus PosixMmapFile::InvalidateCache(size_t offset, size_t length) {
 #ifndef OS_LINUX
  (void)offset;
  (void)length;
-  return Status::OK();
+  return IOStatus::OK();
 #else
  // free OS pages
  int ret = Fadvise(fd_, offset, length, POSIX_FADV_DONTNEED);
  if (ret == 0) {
-    return Status::OK();
+    return IOStatus::OK();
  }
  return IOError("While fadvise NotNeeded mmapped file", filename_, errno);
 #endif
 }

 #ifdef ROCKSDB_FALLOCATE_PRESENT
-Status PosixMmapFile::Allocate(uint64_t offset, uint64_t len) {
+IOStatus PosixMmapFile::Allocate(uint64_t offset, uint64_t len,
+                                 const IOOptions& /*opts*/,
+                                 IODebugContext* /*dbg*/) {
  assert(offset <= static_cast<uint64_t>(std::numeric_limits<off_t>::max()));
  assert(len <= static_cast<uint64_t>(std::numeric_limits<off_t>::max()));
  TEST_KILL_RANDOM("PosixMmapFile::Allocate:0", rocksdb_kill_odds);
@@ -916,7 +939,7 @@ Status PosixMmapFile::Allocate(uint64_t offset, uint64_t len) {
                  static_cast<off_t>(offset), static_cast<off_t>(len));
  }
  if (alloc_status == 0) {
-    return Status::OK();
+    return IOStatus::OK();
  } else {
    return IOError(
        "While fallocate offset " + ToString(offset) + " len " + ToString(len),
@@ -932,7 +955,7 @@ Status PosixMmapFile::Allocate(uint64_t offset, uint64_t len) {
 */
 PosixWritableFile::PosixWritableFile(const std::string& fname, int fd,
                                     const EnvOptions& options)
-    : WritableFile(options),
+    : FSWritableFile(options),
      filename_(fname),
      use_direct_io_(options.use_direct_writes),
      fd_(fd),
@@ -950,11 +973,12 @@ PosixWritableFile::PosixWritableFile(const std::string& fname, int fd,

 PosixWritableFile::~PosixWritableFile() {
  if (fd_ >= 0) {
-    PosixWritableFile::Close();
+    PosixWritableFile::Close(IOOptions(), nullptr);
  }
 }

-Status PosixWritableFile::Append(const Slice& data) {
+IOStatus PosixWritableFile::Append(const Slice& data, const IOOptions& /*opts*/,
+                                   IODebugContext* /*dbg*/) {
  if (use_direct_io()) {
    assert(IsSectorAligned(data.size(), GetRequiredBufferAlignment()));
    assert(IsSectorAligned(data.data(), GetRequiredBufferAlignment()));
@@ -967,10 +991,12 @@ Status PosixWritableFile::Append(const Slice& data) {
  }

  filesize_ += nbytes;
-  return Status::OK();
+  return IOStatus::OK();
 }

-Status PosixWritableFile::PositionedAppend(const Slice& data, uint64_t offset) {
+IOStatus PosixWritableFile::PositionedAppend(const Slice& data, uint64_t offset,
+                                             const IOOptions& /*opts*/,
+                                             IODebugContext* /*dbg*/) {
  if (use_direct_io()) {
    assert(IsSectorAligned(offset, GetRequiredBufferAlignment()));
    assert(IsSectorAligned(data.size(), GetRequiredBufferAlignment()));
@@ -984,11 +1010,12 @@ Status PosixWritableFile::PositionedAppend(const Slice& data, uint64_t offset) {
                   filename_, errno);
  }
  filesize_ = offset + nbytes;
-  return Status::OK();
+  return IOStatus::OK();
 }

-Status PosixWritableFile::Truncate(uint64_t size) {
-  Status s;
+IOStatus PosixWritableFile::Truncate(uint64_t size, const IOOptions& /*opts*/,
+                                     IODebugContext* /*dbg*/) {
+  IOStatus s;
  int r = ftruncate(fd_, size);
  if (r < 0) {
    s = IOError("While ftruncate file to size " + ToString(size), filename_,
@@ -999,8 +1026,9 @@ Status PosixWritableFile::Truncate(uint64_t size) {
  return s;
 }

-Status PosixWritableFile::Close() {
-  Status s;
+IOStatus PosixWritableFile::Close(const IOOptions& /*opts*/,
+                                  IODebugContext* /*dbg*/) {
+  IOStatus s;

  size_t block_size;
  size_t last_allocated_block;
@@ -1054,25 +1082,33 @@ Status PosixWritableFile::Close() {
 }

 // write out the cached data to the OS cache
-Status PosixWritableFile::Flush() { return Status::OK(); }
+IOStatus PosixWritableFile::Flush(const IOOptions& /*opts*/,
+                                  IODebugContext* /*dbg*/) {
+  return IOStatus::OK();
+}

-Status PosixWritableFile::Sync() {
+IOStatus PosixWritableFile::Sync(const IOOptions& /*opts*/,
+                                 IODebugContext* /*dbg*/) {
  if (fdatasync(fd_) < 0) {
    return IOError("While fdatasync", filename_, errno);
  }
-  return Status::OK();
+  return IOStatus::OK();
 }

-Status PosixWritableFile::Fsync() {
+IOStatus PosixWritableFile::Fsync(const IOOptions& /*opts*/,
+                                  IODebugContext* /*dbg*/) {
  if (fsync(fd_) < 0) {
    return IOError("While fsync", filename_, errno);
  }
-  return Status::OK();
+  return IOStatus::OK();
 }

 bool PosixWritableFile::IsSyncThreadSafe() const { return true; }

-uint64_t PosixWritableFile::GetFileSize() { return filesize_; }
+uint64_t PosixWritableFile::GetFileSize(const IOOptions& /*opts*/,
+                                        IODebugContext* /*dbg*/) {
+  return filesize_;
+}

 void PosixWritableFile::SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) {
 #ifdef OS_LINUX
@@ -1092,26 +1128,28 @@ void PosixWritableFile::SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) {
 #endif  // OS_LINUX
 }

-Status PosixWritableFile::InvalidateCache(size_t offset, size_t length) {
+IOStatus PosixWritableFile::InvalidateCache(size_t offset, size_t length) {
  if (use_direct_io()) {
-    return Status::OK();
+    return IOStatus::OK();
  }
 #ifndef OS_LINUX
  (void)offset;
  (void)length;
-  return Status::OK();
+  return IOStatus::OK();
 #else
  // free OS pages
  int ret = Fadvise(fd_, offset, length, POSIX_FADV_DONTNEED);
  if (ret == 0) {
-    return Status::OK();
+    return IOStatus::OK();
  }
  return IOError("While fadvise NotNeeded", filename_, errno);
 #endif
 }

 #ifdef ROCKSDB_FALLOCATE_PRESENT
-Status PosixWritableFile::Allocate(uint64_t offset, uint64_t len) {
+IOStatus PosixWritableFile::Allocate(uint64_t offset, uint64_t len,
+                                     const IOOptions& /*opts*/,
+                                     IODebugContext* /*dbg*/) {
  assert(offset <= static_cast<uint64_t>(std::numeric_limits<off_t>::max()));
  assert(len <= static_cast<uint64_t>(std::numeric_limits<off_t>::max()));
  TEST_KILL_RANDOM("PosixWritableFile::Allocate:0", rocksdb_kill_odds);
@@ -1123,7 +1161,7 @@ Status PosixWritableFile::Allocate(uint64_t offset, uint64_t len) {
                  static_cast<off_t>(offset), static_cast<off_t>(len));
  }
  if (alloc_status == 0) {
-    return Status::OK();
+    return IOStatus::OK();
  } else {
    return IOError(
        "While fallocate offset " + ToString(offset) + " len " + ToString(len),
@@ -1132,7 +1170,9 @@ Status PosixWritableFile::Allocate(uint64_t offset, uint64_t len) {
 }
 #endif

-Status PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) {
+IOStatus PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes,
+                                      const IOOptions& opts,
+                                      IODebugContext* dbg) {
 #ifdef ROCKSDB_RANGESYNC_PRESENT
  assert(offset <= static_cast<uint64_t>(std::numeric_limits<off_t>::max()));
  assert(nbytes <= static_cast<uint64_t>(std::numeric_limits<off_t>::max()));
@@ -1153,10 +1193,10 @@ Status PosixWritableFile::RangeSync(uint64_t offset, uint64_t nbytes) {
      return IOError("While sync_file_range returned " + ToString(ret),
                     filename_, errno);
    }
-    return Status::OK();
+    return IOStatus::OK();
  }
 #endif  // ROCKSDB_RANGESYNC_PRESENT
-  return WritableFile::RangeSync(offset, nbytes);
+  return FSWritableFile::RangeSync(offset, nbytes, opts, dbg);
 }

 #ifdef OS_LINUX
@@ -1175,11 +1215,13 @@ PosixRandomRWFile::PosixRandomRWFile(const std::string& fname, int fd,

 PosixRandomRWFile::~PosixRandomRWFile() {
  if (fd_ >= 0) {
-    Close();
+    Close(IOOptions(), nullptr);
  }
 }

-Status PosixRandomRWFile::Write(uint64_t offset, const Slice& data) {
+IOStatus PosixRandomRWFile::Write(uint64_t offset, const Slice& data,
+                                  const IOOptions& /*opts*/,
+                                  IODebugContext* /*dbg*/) {
  const char* src = data.data();
  size_t nbytes = data.size();
  if (!PosixPositionedWrite(fd_, src, nbytes, static_cast<off_t>(offset))) {
@@ -1188,11 +1230,12 @@ Status PosixRandomRWFile::Write(uint64_t offset, const Slice& data) {
        filename_, errno);
  }

-  return Status::OK();
+  return IOStatus::OK();
 }

-Status PosixRandomRWFile::Read(uint64_t offset, size_t n, Slice* result,
-                               char* scratch) const {
+IOStatus PosixRandomRWFile::Read(uint64_t offset, size_t n,
+                                 const IOOptions& /*opts*/, Slice* result,
+                                 char* scratch, IODebugContext* /*dbg*/) const {
  size_t left = n;
  char* ptr = scratch;
  while (left > 0) {
@@ -1218,31 +1261,37 @@ Status PosixRandomRWFile::Read(uint64_t offset, size_t n, Slice* result,
  }

  *result = Slice(scratch, n - left);
-  return Status::OK();
+  return IOStatus::OK();
 }

-Status PosixRandomRWFile::Flush() { return Status::OK(); }
+IOStatus PosixRandomRWFile::Flush(const IOOptions& /*opts*/,
+                                  IODebugContext* /*dbg*/) {
+  return IOStatus::OK();
+}

-Status PosixRandomRWFile::Sync() {
+IOStatus PosixRandomRWFile::Sync(const IOOptions& /*opts*/,
+                                 IODebugContext* /*dbg*/) {
  if (fdatasync(fd_) < 0) {
    return IOError("While fdatasync random read/write file", filename_, errno);
  }
-  return Status::OK();
+  return IOStatus::OK();
 }

-Status PosixRandomRWFile::Fsync() {
+IOStatus PosixRandomRWFile::Fsync(const IOOptions& /*opts*/,
+                                  IODebugContext* /*dbg*/) {
  if (fsync(fd_) < 0) {
    return IOError("While fsync random read/write file", filename_, errno);
  }
-  return Status::OK();
+  return IOStatus::OK();
 }

-Status PosixRandomRWFile::Close() {
+IOStatus PosixRandomRWFile::Close(const IOOptions& /*opts*/,
+                                  IODebugContext* /*dbg*/) {
  if (close(fd_) < 0) {
    return IOError("While close random read/write file", filename_, errno);
  }
  fd_ = -1;
-  return Status::OK();
+  return IOStatus::OK();
 }

 PosixMemoryMappedFileBuffer::~PosixMemoryMappedFileBuffer() {
@@ -1256,13 +1305,14 @@ PosixMemoryMappedFileBuffer::~PosixMemoryMappedFileBuffer() {

 PosixDirectory::~PosixDirectory() { close(fd_); }

-Status PosixDirectory::Fsync() {
+IOStatus PosixDirectory::Fsync(const IOOptions& /*opts*/,
+                               IODebugContext* /*dbg*/) {
 #ifndef OS_AIX
  if (fsync(fd_) == -1) {
    return IOError("While fsync", "a directory", errno);
  }
 #endif
-  return Status::OK();
+  return IOStatus::OK();
 }
 }  // namespace rocksdb
 #endif
@@ -17,6 +17,8 @@
 #include <string>
 #include "rocksdb/env.h"
 #include "util/thread_local.h"
+#include "rocksdb/file_system.h"
+#include "rocksdb/io_status.h"

 // For non linux platform, the following macros are used only as place
 // holder.
@@ -38,20 +40,23 @@ static std::string IOErrorMsg(const std::string& context,
 }

 // file_name can be left empty if it is not unkown.
-static Status IOError(const std::string& context, const std::string& file_name,
-                      int err_number) {
+static IOStatus IOError(const std::string& context,
+                        const std::string& file_name, int err_number) {
  switch (err_number) {
-  case ENOSPC:
-    return Status::NoSpace(IOErrorMsg(context, file_name),
-                           strerror(err_number));
+    case ENOSPC: {
+      IOStatus s = IOStatus::NoSpace(IOErrorMsg(context, file_name),
+                                     strerror(err_number));
+      s.SetRetryable(true);
+      return s;
+    }
  case ESTALE:
-    return Status::IOError(Status::kStaleFile);
+    return IOStatus::IOError(IOStatus::kStaleFile);
  case ENOENT:
-    return Status::PathNotFound(IOErrorMsg(context, file_name),
-                                strerror(err_number));
+    return IOStatus::PathNotFound(IOErrorMsg(context, file_name),
+                                  strerror(err_number));
  default:
-    return Status::IOError(IOErrorMsg(context, file_name),
-                           strerror(err_number));
+    return IOStatus::IOError(IOErrorMsg(context, file_name),
+                             strerror(err_number));
  }
 }

@@ -60,7 +65,7 @@ class PosixHelper {
  static size_t GetUniqueIdFromFile(int fd, char* id, size_t max_size);
 };

-class PosixSequentialFile : public SequentialFile {
+class PosixSequentialFile : public FSSequentialFile {
 private:
  std::string filename_;
  FILE* file_;
@@ -73,11 +78,13 @@ class PosixSequentialFile : public SequentialFile {
                      const EnvOptions& options);
  virtual ~PosixSequentialFile();

-  virtual Status Read(size_t n, Slice* result, char* scratch) override;
-  virtual Status PositionedRead(uint64_t offset, size_t n, Slice* result,
-                                char* scratch) override;
-  virtual Status Skip(uint64_t n) override;
-  virtual Status InvalidateCache(size_t offset, size_t length) override;
+  virtual IOStatus Read(size_t n, const IOOptions& opts, Slice* result,
+                        char* scratch, IODebugContext* dbg) override;
+  virtual IOStatus PositionedRead(uint64_t offset, size_t n,
+                                  const IOOptions& opts, Slice* result,
+                                  char* scratch, IODebugContext* dbg) override;
+  virtual IOStatus Skip(uint64_t n) override;
+  virtual IOStatus InvalidateCache(size_t offset, size_t length) override;
  virtual bool use_direct_io() const override { return use_direct_io_; }
  virtual size_t GetRequiredBufferAlignment() const override {
    return logical_sector_size_;
@@ -104,7 +111,7 @@ inline struct io_uring* CreateIOUring() {
 }
 #endif  // defined(ROCKSDB_IOURING_PRESENT)

-class PosixRandomAccessFile : public RandomAccessFile {
+class PosixRandomAccessFile : public FSRandomAccessFile {
 protected:
  std::string filename_;
  int fd_;
@@ -124,25 +131,29 @@ class PosixRandomAccessFile : public RandomAccessFile {
  );
  virtual ~PosixRandomAccessFile();

-  virtual Status Read(uint64_t offset, size_t n, Slice* result,
-                      char* scratch) const override;
+  virtual IOStatus Read(uint64_t offset, size_t n, const IOOptions& opts,
+                        Slice* result, char* scratch,
+                        IODebugContext* dbg) const override;

-  virtual Status MultiRead(ReadRequest* reqs, size_t num_reqs) override;
+  virtual IOStatus MultiRead(FSReadRequest* reqs, size_t num_reqs,
+                             const IOOptions& options,
+                             IODebugContext* dbg) override;

-  virtual Status Prefetch(uint64_t offset, size_t n) override;
+  virtual IOStatus Prefetch(uint64_t offset, size_t n, const IOOptions& opts,
+                            IODebugContext* dbg) override;

 #if defined(OS_LINUX) || defined(OS_MACOSX) || defined(OS_AIX)
  virtual size_t GetUniqueId(char* id, size_t max_size) const override;
 #endif
  virtual void Hint(AccessPattern pattern) override;
-  virtual Status InvalidateCache(size_t offset, size_t length) override;
+  virtual IOStatus InvalidateCache(size_t offset, size_t length) override;
  virtual bool use_direct_io() const override { return use_direct_io_; }
  virtual size_t GetRequiredBufferAlignment() const override {
    return logical_sector_size_;
  }
 };

-class PosixWritableFile : public WritableFile {
+class PosixWritableFile : public FSWritableFile {
 protected:
  const std::string filename_;
  const bool use_direct_io_;
@@ -166,32 +177,41 @@ class PosixWritableFile : public WritableFile {

  // Need to implement this so the file is truncated correctly
  // with direct I/O
-  virtual Status Truncate(uint64_t size) override;
-  virtual Status Close() override;
-  virtual Status Append(const Slice& data) override;
-  virtual Status PositionedAppend(const Slice& data, uint64_t offset) override;
-  virtual Status Flush() override;
-  virtual Status Sync() override;
-  virtual Status Fsync() override;
+  virtual IOStatus Truncate(uint64_t size, const IOOptions& opts,
+                            IODebugContext* dbg) override;
+  virtual IOStatus Close(const IOOptions& opts, IODebugContext* dbg) override;
+  virtual IOStatus Append(const Slice& data, const IOOptions& opts,
+                          IODebugContext* dbg) override;
+  virtual IOStatus PositionedAppend(const Slice& data, uint64_t offset,
+                                    const IOOptions& opts,
+                                    IODebugContext* dbg) override;
+  virtual IOStatus Flush(const IOOptions& opts, IODebugContext* dbg) override;
+  virtual IOStatus Sync(const IOOptions& opts, IODebugContext* dbg) override;
+  virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override;
  virtual bool IsSyncThreadSafe() const override;
  virtual bool use_direct_io() const override { return use_direct_io_; }
  virtual void SetWriteLifeTimeHint(Env::WriteLifeTimeHint hint) override;
-  virtual uint64_t GetFileSize() override;
-  virtual Status InvalidateCache(size_t offset, size_t length) override;
+  virtual uint64_t GetFileSize(const IOOptions& opts,
+                               IODebugContext* dbg) override;
+  virtual IOStatus InvalidateCache(size_t offset, size_t length) override;
  virtual size_t GetRequiredBufferAlignment() const override {
    return logical_sector_size_;
  }
 #ifdef ROCKSDB_FALLOCATE_PRESENT
-  virtual Status Allocate(uint64_t offset, uint64_t len) override;
+  virtual IOStatus Allocate(uint64_t offset, uint64_t len,
+                            const IOOptions& opts,
+                            IODebugContext* dbg) override;
 #endif
-  virtual Status RangeSync(uint64_t offset, uint64_t nbytes) override;
+  virtual IOStatus RangeSync(uint64_t offset, uint64_t nbytes,
+                             const IOOptions& opts,
+                             IODebugContext* dbg) override;
 #ifdef OS_LINUX
  virtual size_t GetUniqueId(char* id, size_t max_size) const override;
 #endif
 };

 // mmap() based random-access
-class PosixMmapReadableFile : public RandomAccessFile {
+class PosixMmapReadableFile : public FSRandomAccessFile {
 private:
  int fd_;
  std::string filename_;
@@ -202,12 +222,13 @@ class PosixMmapReadableFile : public RandomAccessFile {
  PosixMmapReadableFile(const int fd, const std::string& fname, void* base,
                        size_t length, const EnvOptions& options);
  virtual ~PosixMmapReadableFile();
-  virtual Status Read(uint64_t offset, size_t n, Slice* result,
-                      char* scratch) const override;
-  virtual Status InvalidateCache(size_t offset, size_t length) override;
+  virtual IOStatus Read(uint64_t offset, size_t n, const IOOptions& opts,
+                        Slice* result, char* scratch,
+                        IODebugContext* dbg) const override;
+  virtual IOStatus InvalidateCache(size_t offset, size_t length) override;
 };

-class PosixMmapFile : public WritableFile {
+class PosixMmapFile : public FSWritableFile {
 private:
  std::string filename_;
  int fd_;
@@ -232,9 +253,9 @@ class PosixMmapFile : public WritableFile {
    return s;
  }

-  Status MapNewRegion();
-  Status UnmapCurrentRegion();
-  Status Msync();
+  IOStatus MapNewRegion();
+  IOStatus UnmapCurrentRegion();
+  IOStatus Msync();

 public:
  PosixMmapFile(const std::string& fname, int fd, size_t page_size,
@@ -243,34 +264,43 @@ class PosixMmapFile : public WritableFile {

  // Means Close() will properly take care of truncate
  // and it does not need any additional information
-  virtual Status Truncate(uint64_t /*size*/) override { return Status::OK(); }
-  virtual Status Close() override;
-  virtual Status Append(const Slice& data) override;
-  virtual Status Flush() override;
-  virtual Status Sync() override;
-  virtual Status Fsync() override;
-  virtual uint64_t GetFileSize() override;
-  virtual Status InvalidateCache(size_t offset, size_t length) override;
+  virtual IOStatus Truncate(uint64_t /*size*/, const IOOptions& /*opts*/,
+                            IODebugContext* /*dbg*/) override {
+    return IOStatus::OK();
+  }
+  virtual IOStatus Close(const IOOptions& opts, IODebugContext* dbg) override;
+  virtual IOStatus Append(const Slice& data, const IOOptions& opts,
+                          IODebugContext* dbg) override;
+  virtual IOStatus Flush(const IOOptions& opts, IODebugContext* dbg) override;
+  virtual IOStatus Sync(const IOOptions& opts, IODebugContext* dbg) override;
+  virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override;
+  virtual uint64_t GetFileSize(const IOOptions& opts,
+                               IODebugContext* dbg) override;
+  virtual IOStatus InvalidateCache(size_t offset, size_t length) override;
 #ifdef ROCKSDB_FALLOCATE_PRESENT
-  virtual Status Allocate(uint64_t offset, uint64_t len) override;
+  virtual IOStatus Allocate(uint64_t offset, uint64_t len,
+                            const IOOptions& opts,
+                            IODebugContext* dbg) override;
 #endif
 };

-class PosixRandomRWFile : public RandomRWFile {
+class PosixRandomRWFile : public FSRandomRWFile {
 public:
  explicit PosixRandomRWFile(const std::string& fname, int fd,
                             const EnvOptions& options);
  virtual ~PosixRandomRWFile();

-  virtual Status Write(uint64_t offset, const Slice& data) override;
+  virtual IOStatus Write(uint64_t offset, const Slice& data,
+                         const IOOptions& opts, IODebugContext* dbg) override;

-  virtual Status Read(uint64_t offset, size_t n, Slice* result,
-                      char* scratch) const override;
+  virtual IOStatus Read(uint64_t offset, size_t n, const IOOptions& opts,
+                        Slice* result, char* scratch,
+                        IODebugContext* dbg) const override;

-  virtual Status Flush() override;
-  virtual Status Sync() override;
-  virtual Status Fsync() override;
-  virtual Status Close() override;
+  virtual IOStatus Flush(const IOOptions& opts, IODebugContext* dbg) override;
+  virtual IOStatus Sync(const IOOptions& opts, IODebugContext* dbg) override;
+  virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override;
+  virtual IOStatus Close(const IOOptions& opts, IODebugContext* dbg) override;

 private:
  const std::string filename_;
@@ -283,11 +313,11 @@ struct PosixMemoryMappedFileBuffer : public MemoryMappedFileBuffer {
  virtual ~PosixMemoryMappedFileBuffer();
 };

-class PosixDirectory : public Directory {
+class PosixDirectory : public FSDirectory {
 public:
  explicit PosixDirectory(int fd) : fd_(fd) {}
  ~PosixDirectory();
-  virtual Status Fsync() override;
+  virtual IOStatus Fsync(const IOOptions& opts, IODebugContext* dbg) override;

 private:
  int fd_;
@@ -19,12 +19,13 @@

 namespace rocksdb {

-DeleteScheduler::DeleteScheduler(Env* env, int64_t rate_bytes_per_sec,
-                                 Logger* info_log,
+DeleteScheduler::DeleteScheduler(Env* env, FileSystem* fs,
+                                 int64_t rate_bytes_per_sec, Logger* info_log,
                                 SstFileManagerImpl* sst_file_manager,
                                 double max_trash_db_ratio,
                                 uint64_t bytes_max_delete_chunk)
    : env_(env),
+      fs_(fs),
      total_trash_size_(0),
      rate_bytes_per_sec_(rate_bytes_per_sec),
      pending_files_(0),
@@ -61,7 +62,7 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path,
    // Rate limiting is disabled or trash size makes up more than
    // max_trash_db_ratio_ (default 25%) of the total DB size
    TEST_SYNC_POINT("DeleteScheduler::DeleteFile");
-    s = env_->DeleteFile(file_path);
+    s = fs_->DeleteFile(file_path, IOOptions(), nullptr);
    if (s.ok()) {
      sst_file_manager_->OnDeleteFile(file_path);
    }
@@ -73,8 +74,9 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path,
  s = MarkAsTrash(file_path, &trash_file);

  if (!s.ok()) {
-    ROCKS_LOG_ERROR(info_log_, "Failed to mark %s as trash", file_path.c_str());
-    s = env_->DeleteFile(file_path);
+    ROCKS_LOG_ERROR(info_log_, "Failed to mark %s as trash -- %s",
+                    file_path.c_str(), s.ToString().c_str());
+    s = fs_->DeleteFile(file_path, IOOptions(), nullptr);
    if (s.ok()) {
      sst_file_manager_->OnDeleteFile(file_path);
    }
@@ -83,7 +85,7 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path,

  // Update the total trash size
  uint64_t trash_file_size = 0;
-  env_->GetFileSize(trash_file, &trash_file_size);
+  fs_->GetFileSize(trash_file, IOOptions(), &trash_file_size, nullptr);
  total_trash_size_.fetch_add(trash_file_size);

  // Add file to delete queue
@@ -165,10 +167,10 @@ Status DeleteScheduler::MarkAsTrash(const std::string& file_path,
  int cnt = 0;
  InstrumentedMutexLock l(&file_move_mu_);
  while (true) {
-    s = env_->FileExists(*trash_file);
+    s = fs_->FileExists(*trash_file, IOOptions(), nullptr);
    if (s.IsNotFound()) {
      // We found a path for our file in trash
-      s = env_->RenameFile(file_path, *trash_file);
+      s = fs_->RenameFile(file_path, *trash_file, IOOptions(), nullptr);
      break;
    } else if (s.ok()) {
      // Name conflict, generate new random suffix
@@ -262,7 +264,7 @@ Status DeleteScheduler::DeleteTrashFile(const std::string& path_in_trash,
                                        uint64_t* deleted_bytes,
                                        bool* is_complete) {
  uint64_t file_size;
-  Status s = env_->GetFileSize(path_in_trash, &file_size);
+  Status s = fs_->GetFileSize(path_in_trash, IOOptions(), &file_size, nullptr);
  *is_complete = true;
  TEST_SYNC_POINT("DeleteScheduler::DeleteTrashFile:DeleteFile");
  if (s.ok()) {
@@ -273,17 +275,19 @@ Status DeleteScheduler::DeleteTrashFile(const std::string& path_in_trash,
      // file after the number of file link check and ftruncte because
      // the file is now in trash and no hardlink is supposed to create
      // to trash files by RocksDB.
-      Status my_status = env_->NumFileLinks(path_in_trash, &num_hard_links);
+      Status my_status = fs_->NumFileLinks(path_in_trash, IOOptions(),
+                                           &num_hard_links, nullptr);
      if (my_status.ok()) {
        if (num_hard_links == 1) {
-          std::unique_ptr<WritableFile> wf;
-          my_status =
-              env_->ReopenWritableFile(path_in_trash, &wf, EnvOptions());
+          std::unique_ptr<FSWritableFile> wf;
+          my_status = fs_->ReopenWritableFile(path_in_trash, FileOptions(),
+                                              &wf, nullptr);
          if (my_status.ok()) {
-            my_status = wf->Truncate(file_size - bytes_max_delete_chunk_);
+            my_status = wf->Truncate(file_size - bytes_max_delete_chunk_,
+                                     IOOptions(), nullptr);
            if (my_status.ok()) {
              TEST_SYNC_POINT("DeleteScheduler::DeleteTrashFile:Fsync");
-              my_status = wf->Fsync();
+              my_status = wf->Fsync(IOOptions(), nullptr);
            }
          }
          if (my_status.ok()) {
@@ -312,14 +316,14 @@ Status DeleteScheduler::DeleteTrashFile(const std::string& path_in_trash,
    }

    if (need_full_delete) {
-      s = env_->DeleteFile(path_in_trash);
+      s = fs_->DeleteFile(path_in_trash, IOOptions(), nullptr);
      if (!dir_to_sync.empty()) {
-        std::unique_ptr<Directory> dir_obj;
+        std::unique_ptr<FSDirectory> dir_obj;
        if (s.ok()) {
-          s = env_->NewDirectory(dir_to_sync, &dir_obj);
+          s = fs_->NewDirectory(dir_to_sync, IOOptions(), &dir_obj, nullptr);
        }
        if (s.ok()) {
-          s = dir_obj->Fsync();
+          s = dir_obj->Fsync(IOOptions(), nullptr);
          TEST_SYNC_POINT_CALLBACK(
              "DeleteScheduler::DeleteTrashFile::AfterSyncDir",
              reinterpret_cast<void*>(const_cast<std::string*>(&dir_to_sync)));
@@ -15,6 +15,7 @@
 #include "monitoring/instrumented_mutex.h"
 #include "port/port.h"

+#include "rocksdb/file_system.h"
 #include "rocksdb/status.h"

 namespace rocksdb {
@@ -32,8 +33,8 @@ class SstFileManagerImpl;
 // case DeleteScheduler will delete files immediately.
 class DeleteScheduler {
 public:
-  DeleteScheduler(Env* env, int64_t rate_bytes_per_sec, Logger* info_log,
-                  SstFileManagerImpl* sst_file_manager,
+  DeleteScheduler(Env* env, FileSystem* fs, int64_t rate_bytes_per_sec,
+                  Logger* info_log, SstFileManagerImpl* sst_file_manager,
                  double max_trash_db_ratio, uint64_t bytes_max_delete_chunk);

  ~DeleteScheduler();
@@ -91,6 +92,8 @@ class DeleteScheduler {
  void BackgroundEmptyTrash();

  Env* env_;
+  FileSystem* fs_;
+
  // total size of trash files
  std::atomic<uint64_t> total_trash_size_;
  // Maximum number of bytes that should be deleted per second
@@ -93,8 +93,10 @@ class DeleteSchedulerTest : public testing::Test {
    // Tests in this file are for DeleteScheduler component and dont create any
    // DBs, so we need to set max_trash_db_ratio to 100% (instead of default
    // 25%)
+    std::shared_ptr<FileSystem>
+                fs(std::make_shared<LegacyFileSystemWrapper>(env_));
    sst_file_mgr_.reset(
-        new SstFileManagerImpl(env_, nullptr, rate_bytes_per_sec_,
+        new SstFileManagerImpl(env_, fs, nullptr, rate_bytes_per_sec_,
                               /* max_trash_db_ratio= */ 1.1, 128 * 1024));
    delete_scheduler_ = sst_file_mgr_->delete_scheduler();
  }