TESTING ppc build based on known good base revision

This commit is actually a cherry-pick of #6643 but for testing purposes only (see Issue #6653)
Exclude more Travis builds for each pull request (#6557 )
2026-11-06 11:02:36 -05:00 · 2020-04-06 14:36:41 -07:00 · 2020-03-19 12:53:37 -07:00 · 2020-03-19 11:30:26 -07:00 · 2020-03-19 10:44:16 -07:00 · 2020-03-18 17:14:15 -07:00
190 changed files with 8023 additions and 3207 deletions
@@ -0,0 +1,56 @@
+version: 2.1
+
+orbs:
+  win: circleci/windows@2.4.0
+
+executors:
+  windows-2xlarge:
+    machine:
+      image: 'windows-server-2019-vs2019:stable'
+      resource_class: windows.2xlarge
+      shell: bash.exe
+
+jobs:
+  build:
+    executor: windows-2xlarge
+    
+    environment:
+      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
+      CMAKE_HOME: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64
+      CMAKE_BIN: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64/bin/cmake.exe
+      CMAKE_GENERATOR: Visual Studio 16 2019
+      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.7
+      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.7;C:/Users/circleci/thirdparty/snappy-1.1.7/build
+      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.7/build/Debug/snappy.lib
+
+    steps:
+      - checkout
+      - run:
+          name: "Install thirdparty dependencies"
+          command: |
+            mkdir ${THIRDPARTY_HOME}
+            cd ${THIRDPARTY_HOME}
+            echo "Installing CMake..."
+            curl --fail --silent --show-error --output cmake-3.16.4-win64-x64.zip --location https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-win64-x64.zip
+            unzip -q cmake-3.16.4-win64-x64.zip
+            echo "Building Snappy dependency..."
+            curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
+            unzip -q snappy-1.1.7.zip
+            cd snappy-1.1.7
+            mkdir build
+            cd build
+            ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" ..
+            msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+      - run:
+          name: "Build RocksDB"
+          command: |
+            mkdir build
+            cd build
+            ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 ..
+            cd ..
+            msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
+      - run:
+          name: "Test RocksDB"
+          shell: powershell.exe
+          command: |
+            build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,env_basic_test,env_test,db_merge_operand_test -Concurrency 16
@@ -34,6 +34,7 @@ manifest_dump
 sst_dump
 blob_dump
 block_cache_trace_analyzer
+db_with_timestamp_basic_test
 tools/block_cache_analyzer/*.pyc
 column_aware_encoding_exp
 util/build_version.cc
@@ -82,3 +83,5 @@ fbcode/
 fbcode
 buckifier/*.pyc
 buckifier/__pycache__
+
+compile_commands.json
@@ -3,12 +3,14 @@ language: cpp
 os:
  - linux
  - osx
+arch:
+  - amd64
+  - arm64
+  - ppc64le
 compiler:
  - clang
  - gcc
 osx_image: xcode9.4
-jdk:
-  - openjdk7
 cache:
  - ccache

@@ -52,22 +54,99 @@ env:

 matrix:
  exclude:
-  - os: osx
-    env: TEST_GROUP=1
-  - os: osx
-    env: TEST_GROUP=2
-  - os: osx
-    env: TEST_GROUP=3
-  - os: osx
-    env: TEST_GROUP=4
  - os: osx
    env: JOB_NAME=cmake-gcc8
-  - os : osx
+  - os: osx
    env: JOB_NAME=cmake-mingw
-  - os : linux
-    compiler: clang
-  - os : osx
+  - os: osx
+    arch: ppc64le
+  - os: osx
    compiler: gcc
+  - os : linux
+    arch: arm64
+    env: JOB_NAME=cmake-mingw
+  - os: linux
+    arch: ppc64le
+    env: JOB_NAME=cmake-mingw
+  - os: linux
+    compiler: clang
+  # Exclude most osx, arm64 and ppc64le tests for pull requests, but build in branches
+  - if: type != pull_request
+    os: osx
+    env: TEST_GROUP=1
+  - if: type != pull_request
+    os : linux
+    arch: arm64
+    env: TEST_GROUP=1
+  - if: type != pull_request
+    os: linux
+    arch: ppc64le
+    env: TEST_GROUP=1
+  - if: type != pull_request
+    os: osx
+    env: TEST_GROUP=2
+  - if: type != pull_request
+    os : linux
+    arch: arm64
+    env: TEST_GROUP=2
+  - if: type != pull_request
+    os: linux
+    arch: ppc64le
+    env: TEST_GROUP=2
+  - if: type != pull_request
+    os: osx
+    env: TEST_GROUP=3
+  - if: type != pull_request
+    os : linux
+    arch: arm64
+    env: TEST_GROUP=3
+  - if: type != pull_request
+    os: linux
+    arch: ppc64le
+    env: TEST_GROUP=3
+  - if: type != pull_request
+    os: osx
+    env: TEST_GROUP=4
+  - if: type != pull_request
+    os : linux
+    arch: arm64
+    env: TEST_GROUP=4
+  - if: type != pull_request
+    os: linux
+    arch: ppc64le
+    env: TEST_GROUP=4
+  - if: type != pull_request
+    os : linux
+    arch: arm64
+    env: JOB_NAME=java-test
+  - if: type != pull_request
+    os: linux
+    arch: ppc64le
+    env: JOB_NAME=java-test
+  - if: type != pull_request
+    os : linux
+    arch: arm64
+    env: JOB_NAME=lite-build
+  - if: type != pull_request
+    os: linux
+    arch: ppc64le
+    env: JOB_NAME=lite-build
+  - if: type != pull_request
+    os : linux
+    arch: arm64
+    env: JOB_NAME=examples
+  - if: type != pull_request
+    os: linux
+    arch: ppc64le
+    env: JOB_NAME=examples
+  - if: type != pull_request
+    os : linux
+    arch: arm64
+    env: JOB_NAME=cmake-gcc8
+  - if: type != pull_request
+    os: linux
+    arch: ppc64le
+    env: JOB_NAME=cmake-gcc8

 install:
  - if [ "${TRAVIS_OS_NAME}" == osx ]; then
@@ -83,10 +162,27 @@ install:
  - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then
      CMAKE_DIST_URL="https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/cmake-3.14.5-Linux-$(uname -m).tar.bz2";
      TAR_OPT="--strip-components=1 -xj";
+      if [ "aarch64" == "$(uname -m)" ]; then
+        sudo apt-get install -y libuv1 librhash0;
+        sudo apt-get upgrade -y libstdc++6;
+      fi;
      mkdir cmake-dist && curl --silent --fail --show-error --location "${CMAKE_DIST_URL}" | tar -C cmake-dist ${TAR_OPT} && export PATH=$PWD/cmake-dist/bin:$PATH;
    fi
-  - if [[ "${JOB_NAME}" == java_test ]]; then
-      java -version && echo "JAVA_HOME=${JAVA_HOME}";
+  - |
+    if [[ "${JOB_NAME}" == java_test || "${JOB_NAME}" == cmake* ]]; then
+      # Ensure JDK 8
+      if [ "${TRAVIS_OS_NAME}" == osx ]; then
+        brew tap AdoptOpenJDK/openjdk
+        brew cask install adoptopenjdk8
+        export JAVA_HOME=$(/usr/libexec/java_home)
+      else
+        sudo apt-get install -y openjdk-8-jdk
+        export PATH=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin:$PATH
+        export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)
+      fi
+      echo "JAVA_HOME=${JAVA_HOME}"
+      which java && java -version
+      which javac && javac -version
    fi

 before_script:
@@ -95,7 +191,7 @@ before_script:
  - ulimit -n 8192

 script:
-  - ${CXX} --version
+  - date; ${CXX} --version
  - if [ `command -v ccache` ]; then ccache -C; fi
  - case $TEST_GROUP in
    platform_dependent)
@@ -203,16 +203,25 @@ else()
 endif()

 include(CheckCCompilerFlag)
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
+  CHECK_C_COMPILER_FLAG("-mcpu=power9" HAS_POWER9)
+  if(HAS_POWER9)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power9 -mtune=power9")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power9 -mtune=power9")
+  else()
+    CHECK_C_COMPILER_FLAG("-mcpu=power8" HAS_POWER8)
+    if(HAS_POWER8)
+      set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power8 -mtune=power8")
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power8 -mtune=power8")
+    endif(HAS_POWER8)
+  endif(HAS_POWER9)
  CHECK_C_COMPILER_FLAG("-maltivec" HAS_ALTIVEC)
  if(HAS_ALTIVEC)
    message(STATUS " HAS_ALTIVEC yes")
    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maltivec")
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maltivec")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power8")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power8")
  endif(HAS_ALTIVEC)
-endif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
+endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")

 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
        CHECK_C_COMPILER_FLAG("-march=armv8-a+crc+crypto" HAS_ARMV8_CRC)
@@ -235,7 +244,7 @@ else()
  if(MSVC)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
  else()
-    if(NOT HAVE_POWER8 AND NOT HAS_ARMV8_CRC)
+    if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64" AND NOT HAS_ARMV8_CRC)
      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
    endif()
  endif()
@@ -481,6 +490,11 @@ if(HAVE_SCHED_GETCPU)
  add_definitions(-DROCKSDB_SCHED_GETCPU_PRESENT)
 endif()

+check_cxx_symbol_exists(getauxval auvx.h HAVE_AUXV_GETAUXVAL)
+if(HAVE_AUXV_GETAUXVAL)
+  add_definitions(-DROCKSDB_AUXV_GETAUXVAL_PRESENT)
+endif()
+
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/include)
 include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.8.1/fused-src)
@@ -496,6 +510,8 @@ set(SOURCES
        cache/lru_cache.cc
        cache/sharded_cache.cc
        db/arena_wrapped_db_iter.cc
+        db/blob/blob_file_addition.cc
+        db/blob/blob_file_garbage.cc
        db/builder.cc
        db/c.cc
        db/column_family.cc
@@ -604,12 +620,15 @@ set(SOURCES
        options/options_sanity_check.cc
        port/stack_trace.cc
        table/adaptive/adaptive_table_factory.cc
+        table/block_based/binary_search_index_reader.cc
        table/block_based/block.cc
        table/block_based/block_based_filter_block.cc
        table/block_based/block_based_table_builder.cc
        table/block_based/block_based_table_factory.cc
+        table/block_based/block_based_table_iterator.cc
        table/block_based/block_based_table_reader.cc
        table/block_based/block_builder.cc
+        table/block_based/block_prefetcher.cc
        table/block_based/block_prefix_index.cc
        table/block_based/data_block_hash_index.cc
        table/block_based/data_block_footer.cc
@@ -617,9 +636,14 @@ set(SOURCES
        table/block_based/filter_policy.cc
        table/block_based/flush_block_policy.cc
        table/block_based/full_filter_block.cc
+        table/block_based/hash_index_reader.cc
        table/block_based/index_builder.cc
+        table/block_based/index_reader_common.cc
        table/block_based/parsed_full_filter_block.cc
        table/block_based/partitioned_filter_block.cc
+        table/block_based/partitioned_index_iterator.cc
+        table/block_based/partitioned_index_reader.cc
+        table/block_based/reader_common.cc
        table/block_based/uncompression_dict_reader.cc
        table/block_fetcher.cc
        table/cuckoo/cuckoo_table_builder.cc
@@ -734,11 +758,11 @@ if(HAVE_SSE42 AND NOT MSVC)
    PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
 endif()

-if(HAVE_POWER8)
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
  list(APPEND SOURCES
    util/crc32c_ppc.c
    util/crc32c_ppc_asm.S)
-endif(HAVE_POWER8)
+endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")

 if(HAS_ARMV8_CRC)
  list(APPEND SOURCES
@@ -919,6 +943,9 @@ if(WITH_TESTS)
  set(TESTS
        cache/cache_test.cc
        cache/lru_cache_test.cc
+        db/blob/blob_file_addition_test.cc
+        db/blob/blob_file_garbage_test.cc
+        db/blob/db_blob_index_test.cc
        db/column_family_test.cc
        db/compact_files_test.cc
        db/compaction/compaction_job_stats_test.cc
@@ -929,7 +956,7 @@ if(WITH_TESTS)
        db/corruption_test.cc
        db/cuckoo_table_db_test.cc
        db/db_basic_test.cc
-        db/db_blob_index_test.cc
+        db/db_with_timestamp_basic_test.cc
        db/db_block_cache_test.cc
        db/db_bloom_filter_test.cc
        db/db_compaction_filter_test.cc
@@ -955,12 +982,13 @@ if(WITH_TESTS)
        db/db_tailing_iter_test.cc
        db/db_test.cc
        db/db_test2.cc
+        db/db_logical_block_size_cache_test.cc
        db/db_universal_compaction_test.cc
        db/db_wal_test.cc
        db/db_write_test.cc
        db/dbformat_test.cc
        db/deletefile_test.cc
-        db/error_handler_test.cc
+        db/error_handler_fs_test.cc
        db/obsolete_files_test.cc
        db/external_sst_file_basic_test.cc
        db/external_sst_file_test.cc
@@ -991,6 +1019,7 @@ if(WITH_TESTS)
        db/write_controller_test.cc
        env/env_basic_test.cc
        env/env_test.cc
+        env/io_posix_test.cc
        env/mock_env_test.cc
        file/delete_scheduler_test.cc
        logging/auto_roll_logger_test.cc
@@ -1077,6 +1106,7 @@ if(WITH_TESTS)
      monitoring/thread_status_updater_debug.cc
      table/mock_table.cc
      test_util/fault_injection_test_env.cc
+      test_util/fault_injection_test_fs.cc
      utilities/cassandra/test_utils.cc
  )
  enable_testing()
@@ -1,11 +1,18 @@
 # Rocksdb Change Log
 ## Unreleased
-### Bug Fixes
-* Upgraded version of bzip library (1.0.6 -> 1.0.8) used with RocksJava to address potential vulnerabilities if an attacker can manipulate compressed data saved and loaded by RocksDB (not normal). See issue #6703.
+### Public API Change
+* Fix spelling so that API now has correctly spelled transaction state name `COMMITTED`, while the old misspelled `COMMITED` is still available as an alias.

-## 6.8.1 (03/30/2020)
-### Behavior changes
-* Since RocksDB 6.8.0, ttl-based FIFO compaction can drop a file whose oldest key becomes older than options.ttl while others have not. This fix reverts this and makes ttl-based FIFO compaction use the file's flush time as the criterion. This fix also requires that max_open_files = -1 and compaction_options_fifo.allow_compaction = false to function properly.
+### Bug Fixes
+* Fix a bug where range tombstone blocks in ingested files were cached incorrectly during ingestion. If range tombstones were read from those incorrectly cached blocks, the keys they covered would be exposed.
+* Fix a data race that might cause crash when calling DB::GetCreationTimeOfOldestFile() by a small chance. The bug was introduced in 6.6 Release.
+
+### Performance Improvements
+* In CompactRange, for levels starting from 0, if the level does not have any file with any key falling in the specified range, the level is skipped. So instead of always compacting from level 0, the compaction starts from the first level with keys in the specified range until the last such level.
+
+### New Features
+* Basic support for user timestamp in iterator. Seek/SeekToFirst/Next and lower/upper bounds are supported. Reverse iteration is not supported. Merge is not considered.
+* When file lock failure when the lock is held by the current process, return acquiring time and thread ID in the error message.

 ## 6.8.0 (02/24/2020)
 ### Java API Changes
@@ -207,6 +207,7 @@ AM_LINK = $(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS
 dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; export PORTABLE="$(PORTABLE)"; "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
 # this file is generated by the previous line to set build flags and sources
 include make_config.mk
+export JAVAC_ARGS
 CLEAN_FILES += make_config.mk

 missing_make_config_paths := $(shell				\
@@ -216,7 +217,7 @@ missing_make_config_paths := $(shell				\
 	done | sort | uniq)

 $(foreach path, $(missing_make_config_paths), \
-	$(warning Warning: $(path) dont exist))
+	$(warning Warning: $(path) does not exist))

 ifeq ($(PLATFORM), OS_AIX)
 # no debug info
@@ -445,6 +446,7 @@ EXPOBJECTS = $(LIBOBJECTS) $(TESTUTIL)

 TESTS = \
 	db_basic_test \
+	db_with_timestamp_basic_test \
 	db_encryption_test \
 	db_test2 \
 	external_sst_file_basic_test \
@@ -459,6 +461,7 @@ TESTS = \
 	env_basic_test \
 	env_test \
 	env_logger_test \
+	io_posix_test \
 	hash_test \
 	random_test \
 	thread_local_test \
@@ -468,6 +471,7 @@ TESTS = \
 	db_wal_test \
 	db_block_cache_test \
 	db_test \
+	db_logical_block_size_cache_test \
 	db_blob_index_test \
 	db_iter_test \
 	db_iter_stress_test \
@@ -492,7 +496,7 @@ TESTS = \
 	db_table_properties_test \
 	db_statistics_test \
 	db_write_test \
-	error_handler_test \
+	error_handler_fs_test \
 	autovector_test \
 	blob_db_test \
 	cleanable_test \
@@ -597,6 +601,8 @@ TESTS = \
 	block_cache_tracer_test \
 	block_cache_trace_analyzer_test \
 	defer_test \
+	blob_file_addition_test \
+	blob_file_garbage_test \

 ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
 	TESTS += folly_synchronization_distributed_mutex_test
@@ -937,7 +943,7 @@ check: all
 	    $(MAKE) T="$$t" TMPD=$(TMPD) check_0;                       \
 	else                                                            \
 	    for t in $(TESTS); do                                       \
-	      echo "===== Running $$t"; ./$$t || exit 1; done;          \
+	      echo "===== Running $$t (`date`)"; ./$$t || exit 1; done;          \
 	fi
 	rm -rf $(TMPD)
 ifneq ($(PLATFORM), OS_AIX)
@@ -950,7 +956,7 @@ endif

 # TODO add ldb_tests
 check_some: $(SUBSET)
-	for t in $(SUBSET); do echo "===== Running $$t"; ./$$t || exit 1; done
+	for t in $(SUBSET); do echo "===== Running $$t (`date`)"; ./$$t || exit 1; done

 .PHONY: ldb_tests
 ldb_tests: ldb
@@ -1056,7 +1062,7 @@ ifneq ($(PAR_TEST),)
 parloop:
 	ret_bad=0;							\
 	for t in $(PAR_TEST); do		\
-		echo "===== Running $$t in parallel $(NUM_PAR)";\
+		echo "===== Running $$t in parallel $(NUM_PAR) (`date`)";\
 		if [ $(db_test) -eq 1 ]; then \
 			seq $(J) | v="$$t" build_tools/gnu_parallel --gnu --plain 's=$(TMPD)/rdb-{};  export TEST_TMPDIR=$$s;' \
 				'timeout 2m ./db_test --gtest_filter=$$v >> $$s/log-{} 2>1'; \
@@ -1302,6 +1308,9 @@ slice_transform_test: util/slice_transform_test.o $(LIBOBJECTS) $(TESTHARNESS)
 db_basic_test: db/db_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

+db_with_timestamp_basic_test: db/db_with_timestamp_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 db_encryption_test: db/db_encryption_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -1311,7 +1320,10 @@ db_test: db/db_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 db_test2: db/db_test2.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-db_blob_index_test: db/db_blob_index_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+db_logical_block_size_cache_test: db/db_logical_block_size_cache_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
+db_blob_index_test: db/blob/db_blob_index_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 db_block_cache_test: db/db_block_cache_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1365,7 +1377,7 @@ db_statistics_test: db/db_statistics_test.o db/db_test_util.o $(LIBOBJECTS) $(TE
 db_write_test: db/db_write_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-error_handler_test: db/error_handler_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+error_handler_fs_test: db/error_handler_fs_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 external_sst_file_basic_test: db/external_sst_file_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1478,6 +1490,9 @@ env_basic_test: env/env_basic_test.o $(LIBOBJECTS) $(TESTHARNESS)
 env_test: env/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

+io_posix_test: env/io_posix_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 fault_injection_test: db/fault_injection_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -1718,6 +1733,12 @@ block_cache_trace_analyzer_test: tools/block_cache_analyzer/block_cache_trace_an
 defer_test: util/defer_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

+blob_file_addition_test: db/blob/blob_file_addition_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
+blob_file_garbage_test: db/blob/blob_file_garbage_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 #-------------------------------------------------
 # make install related stuff
 INSTALL_PATH ?= /usr/local
@@ -1797,9 +1818,9 @@ SHA256_CMD = sha256sum
 ZLIB_VER ?= 1.2.11
 ZLIB_SHA256 ?= c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
 ZLIB_DOWNLOAD_BASE ?= http://zlib.net
-BZIP2_VER ?= 1.0.8
-BZIP2_SHA256 ?= ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
-BZIP2_DOWNLOAD_BASE ?= https://sourceware.org/pub/bzip2
+BZIP2_VER ?= 1.0.6
+BZIP2_SHA256 ?= a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
+BZIP2_DOWNLOAD_BASE ?= https://downloads.sourceforge.net/project/bzip2
 SNAPPY_VER ?= 1.1.8
 SNAPPY_SHA256 ?= 16b677f07832a612b0836178db7f374e414f94657c138e6993cbfc5dcc58651f
 SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/archive
@@ -116,6 +116,8 @@ cpp_library(
        "cache/lru_cache.cc",
        "cache/sharded_cache.cc",
        "db/arena_wrapped_db_iter.cc",
+        "db/blob/blob_file_addition.cc",
+        "db/blob/blob_file_garbage.cc",
        "db/builder.cc",
        "db/c.cc",
        "db/column_family.cc",
@@ -229,12 +231,15 @@ cpp_library(
        "port/port_posix.cc",
        "port/stack_trace.cc",
        "table/adaptive/adaptive_table_factory.cc",
+        "table/block_based/binary_search_index_reader.cc",
        "table/block_based/block.cc",
        "table/block_based/block_based_filter_block.cc",
        "table/block_based/block_based_table_builder.cc",
        "table/block_based/block_based_table_factory.cc",
+        "table/block_based/block_based_table_iterator.cc",
        "table/block_based/block_based_table_reader.cc",
        "table/block_based/block_builder.cc",
+        "table/block_based/block_prefetcher.cc",
        "table/block_based/block_prefix_index.cc",
        "table/block_based/data_block_footer.cc",
        "table/block_based/data_block_hash_index.cc",
@@ -242,9 +247,14 @@ cpp_library(
        "table/block_based/filter_policy.cc",
        "table/block_based/flush_block_policy.cc",
        "table/block_based/full_filter_block.cc",
+        "table/block_based/hash_index_reader.cc",
        "table/block_based/index_builder.cc",
+        "table/block_based/index_reader_common.cc",
        "table/block_based/parsed_full_filter_block.cc",
        "table/block_based/partitioned_filter_block.cc",
+        "table/block_based/partitioned_index_iterator.cc",
+        "table/block_based/partitioned_index_reader.cc",
+        "table/block_based/reader_common.cc",
        "table/block_based/uncompression_dict_reader.cc",
        "table/block_fetcher.cc",
        "table/cuckoo/cuckoo_table_builder.cc",
@@ -367,6 +377,7 @@ cpp_library(
        "db/db_test_util.cc",
        "table/mock_table.cc",
        "test_util/fault_injection_test_env.cc",
+        "test_util/fault_injection_test_fs.cc",
        "test_util/testharness.cc",
        "test_util/testutil.cc",
        "tools/block_cache_analyzer/block_cache_trace_analyzer.cc",
@@ -477,6 +488,20 @@ ROCKS_TESTS = [
        [],
        [],
    ],
+    [
+        "blob_file_addition_test",
+        "db/blob/blob_file_addition_test.cc",
+        "serial",
+        [],
+        [],
+    ],
+    [
+        "blob_file_garbage_test",
+        "db/blob/blob_file_garbage_test.cc",
+        "serial",
+        [],
+        [],
+    ],
    [
        "block_based_filter_block_test",
        "table/block_based/block_based_filter_block_test.cc",
@@ -689,7 +714,7 @@ ROCKS_TESTS = [
    ],
    [
        "db_blob_index_test",
-        "db/db_blob_index_test.cc",
+        "db/blob/db_blob_index_test.cc",
        "serial",
        [],
        [],
@@ -785,6 +810,13 @@ ROCKS_TESTS = [
        [],
        [],
    ],
+    [
+        "db_logical_block_size_cache_test",
+        "db/db_logical_block_size_cache_test.cc",
+        "serial",
+        [],
+        [],
+    ],
    [
        "db_memtable_test",
        "db/db_memtable_test.cc",
@@ -890,6 +922,13 @@ ROCKS_TESTS = [
        [],
        [],
    ],
+    [
+        "db_with_timestamp_basic_test",
+        "db/db_with_timestamp_basic_test.cc",
+        "serial",
+        [],
+        [],
+    ],
    [
        "db_write_test",
        "db/db_write_test.cc",
@@ -961,8 +1000,8 @@ ROCKS_TESTS = [
        [],
    ],
    [
-        "error_handler_test",
-        "db/error_handler_test.cc",
+        "error_handler_fs_test",
+        "db/error_handler_fs_test.cc",
        "serial",
        [],
        [],
@@ -1079,6 +1118,13 @@ ROCKS_TESTS = [
        [],
        [],
    ],
+    [
+        "io_posix_test",
+        "env/io_posix_test.cc",
+        "serial",
+        [],
+        [],
+    ],
    [
        "iostats_context_test",
        "monitoring/iostats_context_test.cc",
@@ -68,7 +68,7 @@ build:
 test:

 test_script:
-  - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test,db_merge_operand_test -Concurrency 8
+  - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_with_timestamp_basic_test,db_test2,db_test,env_basic_test,env_test,db_merge_operand_test -Concurrency 8

 on_failure:
  - cmd: 7z a build-failed.zip %APPVEYOR_BUILD_FOLDER%\build\ && appveyor PushArtifact build-failed.zip
@@ -9,6 +9,7 @@
 #   PLATFORM_LDFLAGS            Linker flags
 #   JAVA_LDFLAGS                Linker flags for RocksDBJava
 #   JAVA_STATIC_LDFLAGS         Linker flags for RocksDBJava static build
+#   JAVAC_ARGS                  Arguments for javac
 #   PLATFORM_SHARED_EXT         Extension for shared libraries
 #   PLATFORM_SHARED_LDFLAGS     Flags for building shared library
 #   PLATFORM_SHARED_CFLAGS      Flags for compiling objects for shared library
@@ -239,6 +240,7 @@ esac
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}"
 JAVA_LDFLAGS="$PLATFORM_LDFLAGS"
 JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS"
+JAVAC_ARGS="-source 7"

 if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
    # Cross-compiling; do not try any compilation tests.
@@ -508,6 +510,20 @@ EOF
        fi
    fi

+    if ! test $ROCKSDB_DISABLE_AUXV_GETAUXVAL; then
+        # Test whether getauxval is supported
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+          #include <sys/auxv.h>
+          int main() {
+            uint64_t auxv = getauxval(AT_HWCAP);
+            (void)auxv;
+          }
+EOF
+        if [ "$?" = 0 ]; then
+            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_AUXV_GETAUXVAL_PRESENT"
+        fi
+    fi
+
    if ! test $ROCKSDB_DISABLE_ALIGNED_NEW; then
        # Test whether c++17 aligned-new is supported
        $CXX $PLATFORM_CXXFLAGS -faligned-new -x c++ - -o /dev/null 2>/dev/null <<EOF
@@ -696,6 +712,7 @@ echo "PLATFORM=$PLATFORM" >> "$OUTPUT"
 echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT"
 echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT"
 echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT"
+echo "JAVAC_ARGS=$JAVAC_ARGS" >> "$OUTPUT"
 echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT"
 echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT"
 echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT"
@@ -5801,7 +5801,7 @@ sub workdir {
 		    . "-" . $self->seq();
 	    } else {
 		$workdir = $opt::workdir;
-		# Rsync treats /./ special. We dont want that
+		# Rsync treats /./ special. We don't want that
 		$workdir =~ s:/\./:/:g; # Remove /./
 		$workdir =~ s:/+$::; # Remove ending / if any
 		$workdir =~ s:^\./::g; # Remove starting ./ if any
@@ -51,6 +51,8 @@ class ArenaWrappedDBIter : public Iterator {
  bool Valid() const override { return db_iter_->Valid(); }
  void SeekToFirst() override { db_iter_->SeekToFirst(); }
  void SeekToLast() override { db_iter_->SeekToLast(); }
+  // 'target' does not contain timestamp, even if user timestamp feature is
+  // enabled.
  void Seek(const Slice& target) override { db_iter_->Seek(target); }
  void SeekForPrev(const Slice& target) override {
    db_iter_->SeekForPrev(target);
@@ -60,6 +62,7 @@ class ArenaWrappedDBIter : public Iterator {
  Slice key() const override { return db_iter_->key(); }
  Slice value() const override { return db_iter_->value(); }
  Status status() const override { return db_iter_->status(); }
+  Slice timestamp() const override { return db_iter_->timestamp(); }
  bool IsBlob() const { return db_iter_->IsBlob(); }

  Status GetProperty(std::string prop_name, std::string* prop) override;
@@ -0,0 +1,16 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <cstdint>
+
+#include "rocksdb/rocksdb_namespace.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+constexpr uint64_t kInvalidBlobFileNumber = 0;
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,154 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "db/blob/blob_file_addition.h"
+
+#include <ostream>
+#include <sstream>
+
+#include "logging/event_logger.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
+#include "test_util/sync_point.h"
+#include "util/coding.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// Tags for custom fields. Note that these get persisted in the manifest,
+// so existing tags should not be modified.
+enum BlobFileAddition::CustomFieldTags : uint32_t {
+  kEndMarker,
+
+  // Add forward compatible fields here
+
+  /////////////////////////////////////////////////////////////////////
+
+  kForwardIncompatibleMask = 1 << 6,
+
+  // Add forward incompatible fields here
+};
+
+void BlobFileAddition::EncodeTo(std::string* output) const {
+  PutVarint64(output, blob_file_number_);
+  PutVarint64(output, total_blob_count_);
+  PutVarint64(output, total_blob_bytes_);
+  PutLengthPrefixedSlice(output, checksum_method_);
+  PutLengthPrefixedSlice(output, checksum_value_);
+
+  // Encode any custom fields here. The format to use is a Varint32 tag (see
+  // CustomFieldTags above) followed by a length prefixed slice. Unknown custom
+  // fields will be ignored during decoding unless they're in the forward
+  // incompatible range.
+
+  TEST_SYNC_POINT_CALLBACK("BlobFileAddition::EncodeTo::CustomFields", output);
+
+  PutVarint32(output, kEndMarker);
+}
+
+Status BlobFileAddition::DecodeFrom(Slice* input) {
+  constexpr char class_name[] = "BlobFileAddition";
+
+  if (!GetVarint64(input, &blob_file_number_)) {
+    return Status::Corruption(class_name, "Error decoding blob file number");
+  }
+
+  if (!GetVarint64(input, &total_blob_count_)) {
+    return Status::Corruption(class_name, "Error decoding total blob count");
+  }
+
+  if (!GetVarint64(input, &total_blob_bytes_)) {
+    return Status::Corruption(class_name, "Error decoding total blob bytes");
+  }
+
+  Slice checksum_method;
+  if (!GetLengthPrefixedSlice(input, &checksum_method)) {
+    return Status::Corruption(class_name, "Error decoding checksum method");
+  }
+  checksum_method_ = checksum_method.ToString();
+
+  Slice checksum_value;
+  if (!GetLengthPrefixedSlice(input, &checksum_value)) {
+    return Status::Corruption(class_name, "Error decoding checksum value");
+  }
+  checksum_value_ = checksum_value.ToString();
+
+  while (true) {
+    uint32_t custom_field_tag = 0;
+    if (!GetVarint32(input, &custom_field_tag)) {
+      return Status::Corruption(class_name, "Error decoding custom field tag");
+    }
+
+    if (custom_field_tag == kEndMarker) {
+      break;
+    }
+
+    if (custom_field_tag & kForwardIncompatibleMask) {
+      return Status::Corruption(
+          class_name, "Forward incompatible custom field encountered");
+    }
+
+    Slice custom_field_value;
+    if (!GetLengthPrefixedSlice(input, &custom_field_value)) {
+      return Status::Corruption(class_name,
+                                "Error decoding custom field value");
+    }
+  }
+
+  return Status::OK();
+}
+
+std::string BlobFileAddition::DebugString() const {
+  std::ostringstream oss;
+
+  oss << *this;
+
+  return oss.str();
+}
+
+std::string BlobFileAddition::DebugJSON() const {
+  JSONWriter jw;
+
+  jw << *this;
+
+  jw.EndObject();
+
+  return jw.Get();
+}
+
+bool operator==(const BlobFileAddition& lhs, const BlobFileAddition& rhs) {
+  return lhs.GetBlobFileNumber() == rhs.GetBlobFileNumber() &&
+         lhs.GetTotalBlobCount() == rhs.GetTotalBlobCount() &&
+         lhs.GetTotalBlobBytes() == rhs.GetTotalBlobBytes() &&
+         lhs.GetChecksumMethod() == rhs.GetChecksumMethod() &&
+         lhs.GetChecksumValue() == rhs.GetChecksumValue();
+}
+
+bool operator!=(const BlobFileAddition& lhs, const BlobFileAddition& rhs) {
+  return !(lhs == rhs);
+}
+
+std::ostream& operator<<(std::ostream& os,
+                         const BlobFileAddition& blob_file_addition) {
+  os << "blob_file_number: " << blob_file_addition.GetBlobFileNumber()
+     << " total_blob_count: " << blob_file_addition.GetTotalBlobCount()
+     << " total_blob_bytes: " << blob_file_addition.GetTotalBlobBytes()
+     << " checksum_method: " << blob_file_addition.GetChecksumMethod()
+     << " checksum_value: " << blob_file_addition.GetChecksumValue();
+
+  return os;
+}
+
+JSONWriter& operator<<(JSONWriter& jw,
+                       const BlobFileAddition& blob_file_addition) {
+  jw << "BlobFileNumber" << blob_file_addition.GetBlobFileNumber()
+     << "TotalBlobCount" << blob_file_addition.GetTotalBlobCount()
+     << "TotalBlobBytes" << blob_file_addition.GetTotalBlobBytes()
+     << "ChecksumMethod" << blob_file_addition.GetChecksumMethod()
+     << "ChecksumValue" << blob_file_addition.GetChecksumValue();
+
+  return jw;
+}
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,67 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <iosfwd>
+#include <string>
+
+#include "db/blob/blob_constants.h"
+#include "rocksdb/rocksdb_namespace.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+class JSONWriter;
+class Slice;
+class Status;
+
+class BlobFileAddition {
+ public:
+  BlobFileAddition() = default;
+
+  BlobFileAddition(uint64_t blob_file_number, uint64_t total_blob_count,
+                   uint64_t total_blob_bytes, std::string checksum_method,
+                   std::string checksum_value)
+      : blob_file_number_(blob_file_number),
+        total_blob_count_(total_blob_count),
+        total_blob_bytes_(total_blob_bytes),
+        checksum_method_(std::move(checksum_method)),
+        checksum_value_(std::move(checksum_value)) {
+    assert(checksum_method_.empty() == checksum_value_.empty());
+  }
+
+  uint64_t GetBlobFileNumber() const { return blob_file_number_; }
+  uint64_t GetTotalBlobCount() const { return total_blob_count_; }
+  uint64_t GetTotalBlobBytes() const { return total_blob_bytes_; }
+  const std::string& GetChecksumMethod() const { return checksum_method_; }
+  const std::string& GetChecksumValue() const { return checksum_value_; }
+
+  void EncodeTo(std::string* output) const;
+  Status DecodeFrom(Slice* input);
+
+  std::string DebugString() const;
+  std::string DebugJSON() const;
+
+ private:
+  enum CustomFieldTags : uint32_t;
+
+  uint64_t blob_file_number_ = kInvalidBlobFileNumber;
+  uint64_t total_blob_count_ = 0;
+  uint64_t total_blob_bytes_ = 0;
+  std::string checksum_method_;
+  std::string checksum_value_;
+};
+
+bool operator==(const BlobFileAddition& lhs, const BlobFileAddition& rhs);
+bool operator!=(const BlobFileAddition& lhs, const BlobFileAddition& rhs);
+
+std::ostream& operator<<(std::ostream& os,
+                         const BlobFileAddition& blob_file_addition);
+JSONWriter& operator<<(JSONWriter& jw,
+                       const BlobFileAddition& blob_file_addition);
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,206 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "db/blob/blob_file_addition.h"
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+
+#include "test_util/sync_point.h"
+#include "test_util/testharness.h"
+#include "util/coding.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+class BlobFileAdditionTest : public testing::Test {
+ public:
+  static void TestEncodeDecode(const BlobFileAddition& blob_file_addition) {
+    std::string encoded;
+    blob_file_addition.EncodeTo(&encoded);
+
+    BlobFileAddition decoded;
+    Slice input(encoded);
+    ASSERT_OK(decoded.DecodeFrom(&input));
+
+    ASSERT_EQ(blob_file_addition, decoded);
+  }
+};
+
+TEST_F(BlobFileAdditionTest, Empty) {
+  BlobFileAddition blob_file_addition;
+
+  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), kInvalidBlobFileNumber);
+  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 0);
+  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), 0);
+  ASSERT_TRUE(blob_file_addition.GetChecksumMethod().empty());
+  ASSERT_TRUE(blob_file_addition.GetChecksumValue().empty());
+
+  TestEncodeDecode(blob_file_addition);
+}
+
+TEST_F(BlobFileAdditionTest, NonEmpty) {
+  constexpr uint64_t blob_file_number = 123;
+  constexpr uint64_t total_blob_count = 2;
+  constexpr uint64_t total_blob_bytes = 123456;
+  const std::string checksum_method("SHA1");
+  const std::string checksum_value("bdb7f34a59dfa1592ce7f52e99f98c570c525cbd");
+
+  BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
+                                      total_blob_bytes, checksum_method,
+                                      checksum_value);
+
+  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
+  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), total_blob_count);
+  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), total_blob_bytes);
+  ASSERT_EQ(blob_file_addition.GetChecksumMethod(), checksum_method);
+  ASSERT_EQ(blob_file_addition.GetChecksumValue(), checksum_value);
+
+  TestEncodeDecode(blob_file_addition);
+}
+
+TEST_F(BlobFileAdditionTest, DecodeErrors) {
+  std::string str;
+  Slice slice(str);
+
+  BlobFileAddition blob_file_addition;
+
+  {
+    const Status s = blob_file_addition.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "blob file number"));
+  }
+
+  constexpr uint64_t blob_file_number = 123;
+  PutVarint64(&str, blob_file_number);
+  slice = str;
+
+  {
+    const Status s = blob_file_addition.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "total blob count"));
+  }
+
+  constexpr uint64_t total_blob_count = 4567;
+  PutVarint64(&str, total_blob_count);
+  slice = str;
+
+  {
+    const Status s = blob_file_addition.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "total blob bytes"));
+  }
+
+  constexpr uint64_t total_blob_bytes = 12345678;
+  PutVarint64(&str, total_blob_bytes);
+  slice = str;
+
+  {
+    const Status s = blob_file_addition.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "checksum method"));
+  }
+
+  constexpr char checksum_method[] = "SHA1";
+  PutLengthPrefixedSlice(&str, checksum_method);
+  slice = str;
+
+  {
+    const Status s = blob_file_addition.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "checksum value"));
+  }
+
+  constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd";
+  PutLengthPrefixedSlice(&str, checksum_value);
+  slice = str;
+
+  {
+    const Status s = blob_file_addition.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "custom field tag"));
+  }
+
+  constexpr uint32_t custom_tag = 2;
+  PutVarint32(&str, custom_tag);
+  slice = str;
+
+  {
+    const Status s = blob_file_addition.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "custom field value"));
+  }
+}
+
+TEST_F(BlobFileAdditionTest, ForwardCompatibleCustomField) {
+  SyncPoint::GetInstance()->SetCallBack(
+      "BlobFileAddition::EncodeTo::CustomFields", [&](void* arg) {
+        std::string* output = static_cast<std::string*>(arg);
+
+        constexpr uint32_t forward_compatible_tag = 2;
+        PutVarint32(output, forward_compatible_tag);
+
+        PutLengthPrefixedSlice(output, "deadbeef");
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  constexpr uint64_t blob_file_number = 678;
+  constexpr uint64_t total_blob_count = 9999;
+  constexpr uint64_t total_blob_bytes = 100000000;
+  const std::string checksum_method("CRC32");
+  const std::string checksum_value("3d87ff57");
+
+  BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
+                                      total_blob_bytes, checksum_method,
+                                      checksum_value);
+
+  TestEncodeDecode(blob_file_addition);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
+TEST_F(BlobFileAdditionTest, ForwardIncompatibleCustomField) {
+  SyncPoint::GetInstance()->SetCallBack(
+      "BlobFileAddition::EncodeTo::CustomFields", [&](void* arg) {
+        std::string* output = static_cast<std::string*>(arg);
+
+        constexpr uint32_t forward_incompatible_tag = (1 << 6) + 1;
+        PutVarint32(output, forward_incompatible_tag);
+
+        PutLengthPrefixedSlice(output, "foobar");
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  constexpr uint64_t blob_file_number = 456;
+  constexpr uint64_t total_blob_count = 100;
+  constexpr uint64_t total_blob_bytes = 2000000;
+  const std::string checksum_method("CRC32B");
+  const std::string checksum_value("6dbdf23a");
+
+  BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
+                                      total_blob_bytes, checksum_method,
+                                      checksum_value);
+
+  std::string encoded;
+  blob_file_addition.EncodeTo(&encoded);
+
+  BlobFileAddition decoded_blob_file_addition;
+  Slice input(encoded);
+  const Status s = decoded_blob_file_addition.DecodeFrom(&input);
+
+  ASSERT_TRUE(s.IsCorruption());
+  ASSERT_TRUE(std::strstr(s.getState(), "Forward incompatible"));
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
+}  // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -0,0 +1,134 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "db/blob/blob_file_garbage.h"
+
+#include <ostream>
+#include <sstream>
+
+#include "logging/event_logger.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
+#include "test_util/sync_point.h"
+#include "util/coding.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// Tags for custom fields. Note that these get persisted in the manifest,
+// so existing tags should not be modified.
+enum BlobFileGarbage::CustomFieldTags : uint32_t {
+  kEndMarker,
+
+  // Add forward compatible fields here
+
+  /////////////////////////////////////////////////////////////////////
+
+  kForwardIncompatibleMask = 1 << 6,
+
+  // Add forward incompatible fields here
+};
+
+void BlobFileGarbage::EncodeTo(std::string* output) const {
+  PutVarint64(output, blob_file_number_);
+  PutVarint64(output, garbage_blob_count_);
+  PutVarint64(output, garbage_blob_bytes_);
+
+  // Encode any custom fields here. The format to use is a Varint32 tag (see
+  // CustomFieldTags above) followed by a length prefixed slice. Unknown custom
+  // fields will be ignored during decoding unless they're in the forward
+  // incompatible range.
+
+  TEST_SYNC_POINT_CALLBACK("BlobFileGarbage::EncodeTo::CustomFields", output);
+
+  PutVarint32(output, kEndMarker);
+}
+
+Status BlobFileGarbage::DecodeFrom(Slice* input) {
+  constexpr char class_name[] = "BlobFileGarbage";
+
+  if (!GetVarint64(input, &blob_file_number_)) {
+    return Status::Corruption(class_name, "Error decoding blob file number");
+  }
+
+  if (!GetVarint64(input, &garbage_blob_count_)) {
+    return Status::Corruption(class_name, "Error decoding garbage blob count");
+  }
+
+  if (!GetVarint64(input, &garbage_blob_bytes_)) {
+    return Status::Corruption(class_name, "Error decoding garbage blob bytes");
+  }
+
+  while (true) {
+    uint32_t custom_field_tag = 0;
+    if (!GetVarint32(input, &custom_field_tag)) {
+      return Status::Corruption(class_name, "Error decoding custom field tag");
+    }
+
+    if (custom_field_tag == kEndMarker) {
+      break;
+    }
+
+    if (custom_field_tag & kForwardIncompatibleMask) {
+      return Status::Corruption(
+          class_name, "Forward incompatible custom field encountered");
+    }
+
+    Slice custom_field_value;
+    if (!GetLengthPrefixedSlice(input, &custom_field_value)) {
+      return Status::Corruption(class_name,
+                                "Error decoding custom field value");
+    }
+  }
+
+  return Status::OK();
+}
+
+std::string BlobFileGarbage::DebugString() const {
+  std::ostringstream oss;
+
+  oss << *this;
+
+  return oss.str();
+}
+
+std::string BlobFileGarbage::DebugJSON() const {
+  JSONWriter jw;
+
+  jw << *this;
+
+  jw.EndObject();
+
+  return jw.Get();
+}
+
+bool operator==(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs) {
+  return lhs.GetBlobFileNumber() == rhs.GetBlobFileNumber() &&
+         lhs.GetGarbageBlobCount() == rhs.GetGarbageBlobCount() &&
+         lhs.GetGarbageBlobBytes() == rhs.GetGarbageBlobBytes();
+}
+
+bool operator!=(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs) {
+  return !(lhs == rhs);
+}
+
+std::ostream& operator<<(std::ostream& os,
+                         const BlobFileGarbage& blob_file_garbage) {
+  os << "blob_file_number: " << blob_file_garbage.GetBlobFileNumber()
+     << " garbage_blob_count: " << blob_file_garbage.GetGarbageBlobCount()
+     << " garbage_blob_bytes: " << blob_file_garbage.GetGarbageBlobBytes();
+
+  return os;
+}
+
+JSONWriter& operator<<(JSONWriter& jw,
+                       const BlobFileGarbage& blob_file_garbage) {
+  jw << "BlobFileNumber" << blob_file_garbage.GetBlobFileNumber()
+     << "GarbageBlobCount" << blob_file_garbage.GetGarbageBlobCount()
+     << "GarbageBlobBytes" << blob_file_garbage.GetGarbageBlobBytes();
+
+  return jw;
+}
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,57 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include <cstdint>
+#include <iosfwd>
+#include <string>
+
+#include "db/blob/blob_constants.h"
+#include "rocksdb/rocksdb_namespace.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+class JSONWriter;
+class Slice;
+class Status;
+
+class BlobFileGarbage {
+ public:
+  BlobFileGarbage() = default;
+
+  BlobFileGarbage(uint64_t blob_file_number, uint64_t garbage_blob_count,
+                  uint64_t garbage_blob_bytes)
+      : blob_file_number_(blob_file_number),
+        garbage_blob_count_(garbage_blob_count),
+        garbage_blob_bytes_(garbage_blob_bytes) {}
+
+  uint64_t GetBlobFileNumber() const { return blob_file_number_; }
+  uint64_t GetGarbageBlobCount() const { return garbage_blob_count_; }
+  uint64_t GetGarbageBlobBytes() const { return garbage_blob_bytes_; }
+
+  void EncodeTo(std::string* output) const;
+  Status DecodeFrom(Slice* input);
+
+  std::string DebugString() const;
+  std::string DebugJSON() const;
+
+ private:
+  enum CustomFieldTags : uint32_t;
+
+  uint64_t blob_file_number_ = kInvalidBlobFileNumber;
+  uint64_t garbage_blob_count_ = 0;
+  uint64_t garbage_blob_bytes_ = 0;
+};
+
+bool operator==(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs);
+bool operator!=(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs);
+
+std::ostream& operator<<(std::ostream& os,
+                         const BlobFileGarbage& blob_file_garbage);
+JSONWriter& operator<<(JSONWriter& jw,
+                       const BlobFileGarbage& blob_file_garbage);
+
+}  // namespace ROCKSDB_NAMESPACE
@@ -0,0 +1,173 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "db/blob/blob_file_garbage.h"
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+
+#include "test_util/sync_point.h"
+#include "test_util/testharness.h"
+#include "util/coding.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+class BlobFileGarbageTest : public testing::Test {
+ public:
+  static void TestEncodeDecode(const BlobFileGarbage& blob_file_garbage) {
+    std::string encoded;
+    blob_file_garbage.EncodeTo(&encoded);
+
+    BlobFileGarbage decoded;
+    Slice input(encoded);
+    ASSERT_OK(decoded.DecodeFrom(&input));
+
+    ASSERT_EQ(blob_file_garbage, decoded);
+  }
+};
+
+TEST_F(BlobFileGarbageTest, Empty) {
+  BlobFileGarbage blob_file_garbage;
+
+  ASSERT_EQ(blob_file_garbage.GetBlobFileNumber(), kInvalidBlobFileNumber);
+  ASSERT_EQ(blob_file_garbage.GetGarbageBlobCount(), 0);
+  ASSERT_EQ(blob_file_garbage.GetGarbageBlobBytes(), 0);
+
+  TestEncodeDecode(blob_file_garbage);
+}
+
+TEST_F(BlobFileGarbageTest, NonEmpty) {
+  constexpr uint64_t blob_file_number = 123;
+  constexpr uint64_t garbage_blob_count = 1;
+  constexpr uint64_t garbage_blob_bytes = 9876;
+
+  BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count,
+                                    garbage_blob_bytes);
+
+  ASSERT_EQ(blob_file_garbage.GetBlobFileNumber(), blob_file_number);
+  ASSERT_EQ(blob_file_garbage.GetGarbageBlobCount(), garbage_blob_count);
+  ASSERT_EQ(blob_file_garbage.GetGarbageBlobBytes(), garbage_blob_bytes);
+
+  TestEncodeDecode(blob_file_garbage);
+}
+
+TEST_F(BlobFileGarbageTest, DecodeErrors) {
+  std::string str;
+  Slice slice(str);
+
+  BlobFileGarbage blob_file_garbage;
+
+  {
+    const Status s = blob_file_garbage.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "blob file number"));
+  }
+
+  constexpr uint64_t blob_file_number = 123;
+  PutVarint64(&str, blob_file_number);
+  slice = str;
+
+  {
+    const Status s = blob_file_garbage.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "garbage blob count"));
+  }
+
+  constexpr uint64_t garbage_blob_count = 4567;
+  PutVarint64(&str, garbage_blob_count);
+  slice = str;
+
+  {
+    const Status s = blob_file_garbage.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "garbage blob bytes"));
+  }
+
+  constexpr uint64_t garbage_blob_bytes = 12345678;
+  PutVarint64(&str, garbage_blob_bytes);
+  slice = str;
+
+  {
+    const Status s = blob_file_garbage.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "custom field tag"));
+  }
+
+  constexpr uint32_t custom_tag = 2;
+  PutVarint32(&str, custom_tag);
+  slice = str;
+
+  {
+    const Status s = blob_file_garbage.DecodeFrom(&slice);
+    ASSERT_TRUE(s.IsCorruption());
+    ASSERT_TRUE(std::strstr(s.getState(), "custom field value"));
+  }
+}
+
+TEST_F(BlobFileGarbageTest, ForwardCompatibleCustomField) {
+  SyncPoint::GetInstance()->SetCallBack(
+      "BlobFileGarbage::EncodeTo::CustomFields", [&](void* arg) {
+        std::string* output = static_cast<std::string*>(arg);
+
+        constexpr uint32_t forward_compatible_tag = 2;
+        PutVarint32(output, forward_compatible_tag);
+
+        PutLengthPrefixedSlice(output, "deadbeef");
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  constexpr uint64_t blob_file_number = 678;
+  constexpr uint64_t garbage_blob_count = 9999;
+  constexpr uint64_t garbage_blob_bytes = 100000000;
+
+  BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count,
+                                    garbage_blob_bytes);
+
+  TestEncodeDecode(blob_file_garbage);
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
+TEST_F(BlobFileGarbageTest, ForwardIncompatibleCustomField) {
+  SyncPoint::GetInstance()->SetCallBack(
+      "BlobFileGarbage::EncodeTo::CustomFields", [&](void* arg) {
+        std::string* output = static_cast<std::string*>(arg);
+
+        constexpr uint32_t forward_incompatible_tag = (1 << 6) + 1;
+        PutVarint32(output, forward_incompatible_tag);
+
+        PutLengthPrefixedSlice(output, "foobar");
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  constexpr uint64_t blob_file_number = 456;
+  constexpr uint64_t garbage_blob_count = 100;
+  constexpr uint64_t garbage_blob_bytes = 2000000;
+
+  BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count,
+                                    garbage_blob_bytes);
+
+  std::string encoded;
+  blob_file_garbage.EncodeTo(&encoded);
+
+  BlobFileGarbage decoded_blob_file_addition;
+  Slice input(encoded);
+  const Status s = decoded_blob_file_addition.DecodeFrom(&input);
+
+  ASSERT_TRUE(s.IsCorruption());
+  ASSERT_TRUE(std::strstr(s.getState(), "Forward incompatible"));
+
+  SyncPoint::GetInstance()->DisableProcessing();
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+}
+
+}  // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -471,6 +471,17 @@ void SuperVersionUnrefHandle(void* ptr) {
 }
 }  // anonymous namespace

+std::vector<std::string> ColumnFamilyData::GetDbPaths() const {
+  std::vector<std::string> paths;
+  paths.reserve(ioptions_.cf_paths.size());
+  for (const DbPath& db_path : ioptions_.cf_paths) {
+    paths.emplace_back(db_path.path);
+  }
+  return paths;
+}
+
+const uint32_t ColumnFamilyData::kDummyColumnFamilyDataId = port::kMaxUint32;
+
 ColumnFamilyData::ColumnFamilyData(
    uint32_t id, const std::string& name, Version* _dummy_versions,
    Cache* _table_cache, WriteBufferManager* write_buffer_manager,
@@ -507,7 +518,23 @@ ColumnFamilyData::ColumnFamilyData(
      queued_for_compaction_(false),
      prev_compaction_needed_bytes_(0),
      allow_2pc_(db_options.allow_2pc),
-      last_memtable_id_(0) {
+      last_memtable_id_(0),
+      db_paths_registered_(false) {
+  if (id_ != kDummyColumnFamilyDataId) {
+    // TODO(cc): RegisterDbPaths can be expensive, considering moving it
+    // outside of this constructor which might be called with db mutex held.
+    // TODO(cc): considering using ioptions_.fs, currently some tests rely on
+    // EnvWrapper, that's the main reason why we use env here.
+    Status s = ioptions_.env->RegisterDbPaths(GetDbPaths());
+    if (s.ok()) {
+      db_paths_registered_ = true;
+    } else {
+      ROCKS_LOG_ERROR(
+          ioptions_.info_log,
+          "Failed to register data paths of column family (id: %d, name: %s)",
+          id_, name_.c_str());
+    }
+  }
  Ref();

  // Convert user defined table properties collector factories to internal ones.
@@ -601,6 +628,18 @@ ColumnFamilyData::~ColumnFamilyData() {
  for (MemTable* m : to_delete) {
    delete m;
  }
+
+  if (db_paths_registered_) {
+    // TODO(cc): considering using ioptions_.fs, currently some tests rely on
+    // EnvWrapper, that's the main reason why we use env here.
+    Status s = ioptions_.env->UnregisterDbPaths(GetDbPaths());
+    if (!s.ok()) {
+      ROCKS_LOG_ERROR(
+          ioptions_.info_log,
+          "Failed to unregister data paths of column family (id: %d, name: %s)",
+          id_, name_.c_str());
+    }
+  }
 }

 bool ColumnFamilyData::UnrefAndTryDelete() {
@@ -1321,7 +1360,7 @@ Env::WriteLifeTimeHint ColumnFamilyData::CalculateSSTWriteHint(int level) {
 }

 Status ColumnFamilyData::AddDirectories(
-    std::map<std::string, std::shared_ptr<Directory>>* created_dirs) {
+    std::map<std::string, std::shared_ptr<FSDirectory>>* created_dirs) {
  Status s;
  assert(created_dirs != nullptr);
  assert(data_dirs_.empty());
@@ -1329,8 +1368,8 @@ Status ColumnFamilyData::AddDirectories(
    auto existing_dir = created_dirs->find(p.path);

    if (existing_dir == created_dirs->end()) {
-      std::unique_ptr<Directory> path_directory;
-      s = DBImpl::CreateAndNewDirectory(ioptions_.env, p.path, &path_directory);
+      std::unique_ptr<FSDirectory> path_directory;
+      s = DBImpl::CreateAndNewDirectory(ioptions_.fs, p.path, &path_directory);
      if (!s.ok()) {
        return s;
      }
@@ -1345,7 +1384,7 @@ Status ColumnFamilyData::AddDirectories(
  return s;
 }

-Directory* ColumnFamilyData::GetDataDir(size_t path_id) const {
+FSDirectory* ColumnFamilyData::GetDataDir(size_t path_id) const {
  if (data_dirs_.empty()) {
    return nullptr;
  }
@@ -1363,8 +1402,9 @@ ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
                                 BlockCacheTracer* const block_cache_tracer)
    : max_column_family_(0),
      dummy_cfd_(new ColumnFamilyData(
-          0, "", nullptr, nullptr, nullptr, ColumnFamilyOptions(), *db_options,
-          file_options, nullptr, block_cache_tracer)),
+          ColumnFamilyData::kDummyColumnFamilyDataId, "", nullptr, nullptr,
+          nullptr, ColumnFamilyOptions(), *db_options, file_options, nullptr,
+          block_cache_tracer)),
      default_cfd_cache_(nullptr),
      db_name_(dbname),
      db_options_(db_options),
@@ -500,14 +500,15 @@ class ColumnFamilyData {
  // created_dirs remembers directory created, so that we don't need to call
  // the same data creation operation again.
  Status AddDirectories(
-      std::map<std::string, std::shared_ptr<Directory>>* created_dirs);
+      std::map<std::string, std::shared_ptr<FSDirectory>>* created_dirs);

-  Directory* GetDataDir(size_t path_id) const;
+  FSDirectory* GetDataDir(size_t path_id) const;

  ThreadLocalPtr* TEST_GetLocalSV() { return local_sv_.get(); }

 private:
  friend class ColumnFamilySet;
+  static const uint32_t kDummyColumnFamilyDataId;
  ColumnFamilyData(uint32_t id, const std::string& name,
                   Version* dummy_versions, Cache* table_cache,
                   WriteBufferManager* write_buffer_manager,
@@ -517,6 +518,8 @@ class ColumnFamilyData {
                   ColumnFamilySet* column_family_set,
                   BlockCacheTracer* const block_cache_tracer);

+  std::vector<std::string> GetDbPaths() const;
+
  uint32_t id_;
  const std::string name_;
  Version* dummy_versions_;  // Head of circular doubly-linked list of versions.
@@ -592,7 +595,9 @@ class ColumnFamilyData {
  std::atomic<uint64_t> last_memtable_id_;

  // Directories corresponding to cf_paths.
-  std::vector<std::shared_ptr<Directory>> data_dirs_;
+  std::vector<std::shared_ptr<FSDirectory>> data_dirs_;
+
+  bool db_paths_registered_;
 };

 // ColumnFamilySet has interesting thread-safety requirements
@@ -821,7 +821,7 @@ TEST_P(ColumnFamilyTest, BulkAddDrop) {
 }

 TEST_P(ColumnFamilyTest, DropTest) {
-  // first iteration - dont reopen DB before dropping
+  // first iteration - don't reopen DB before dropping
  // second iteration - reopen DB before dropping
  for (int iter = 0; iter < 2; ++iter) {
    Open({"default"});
@@ -37,7 +37,7 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
                            const Slice& key, PinnableSlice* value) {
  GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
                         GetContext::kNotFound, key, value, nullptr, nullptr,
-                         true, nullptr, nullptr);
+                         nullptr, true, nullptr, nullptr);
  LookupKey lkey(key, kMaxSequenceNumber);
  files_.files[FindFile(key)].fd.table_reader->Get(options, lkey.internal_key(),
                                                   &get_context, nullptr);
@@ -70,7 +70,7 @@ std::vector<Status> CompactedDBImpl::MultiGet(const ReadOptions& options,
      std::string& value = (*values)[idx];
      GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
                             GetContext::kNotFound, keys[idx], &pinnable_val,
-                             nullptr, nullptr, true, nullptr, nullptr);
+                             nullptr, nullptr, nullptr, true, nullptr, nullptr);
      LookupKey lkey(keys[idx], kMaxSequenceNumber);
      r->Get(options, lkey.internal_key(), &get_context, nullptr);
      value.assign(pinnable_val.data(), pinnable_val.size());
@@ -298,7 +298,7 @@ CompactionJob::CompactionJob(
    const FileOptions& file_options, VersionSet* versions,
    const std::atomic<bool>* shutting_down,
    const SequenceNumber preserve_deletes_seqnum, LogBuffer* log_buffer,
-    Directory* db_directory, Directory* output_directory, Statistics* stats,
+    FSDirectory* db_directory, FSDirectory* output_directory, Statistics* stats,
    InstrumentedMutex* db_mutex, ErrorHandler* db_error_handler,
    std::vector<SequenceNumber> existing_snapshots,
    SequenceNumber earliest_write_conflict_snapshot,
@@ -614,7 +614,7 @@ Status CompactionJob::Run() {
  }

  if (status.ok() && output_directory_) {
-    status = output_directory_->Fsync();
+    status = output_directory_->Fsync(IOOptions(), nullptr);
  }

  if (status.ok()) {
@@ -67,8 +67,8 @@ class CompactionJob {
                const FileOptions& file_options, VersionSet* versions,
                const std::atomic<bool>* shutting_down,
                const SequenceNumber preserve_deletes_seqnum,
-                LogBuffer* log_buffer, Directory* db_directory,
-                Directory* output_directory, Statistics* stats,
+                LogBuffer* log_buffer, FSDirectory* db_directory,
+                FSDirectory* output_directory, Statistics* stats,
                InstrumentedMutex* db_mutex, ErrorHandler* db_error_handler,
                std::vector<SequenceNumber> existing_snapshots,
                SequenceNumber earliest_write_conflict_snapshot,
@@ -161,8 +161,8 @@ class CompactionJob {
  const std::atomic<bool>* manual_compaction_paused_;
  const SequenceNumber preserve_deletes_seqnum_;
  LogBuffer* log_buffer_;
-  Directory* db_directory_;
-  Directory* output_directory_;
+  FSDirectory* db_directory_;
+  FSDirectory* output_directory_;
  Statistics* stats_;
  InstrumentedMutex* db_mutex_;
  ErrorHandler* db_error_handler_;
@@ -12,7 +12,7 @@
 #include <string>
 #include <tuple>

-#include "db/blob_index.h"
+#include "db/blob/blob_index.h"
 #include "db/column_family.h"
 #include "db/compaction/compaction_job.h"
 #include "db/db_impl/db_impl.h"
@@ -71,16 +71,13 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
  if (current_time > mutable_cf_options.ttl) {
    for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
      FileMetaData* f = *ritr;
-      if (f->fd.table_reader && f->fd.table_reader->GetTableProperties()) {
-        uint64_t creation_time =
-            f->fd.table_reader->GetTableProperties()->creation_time;
-        if (creation_time == 0 ||
-            creation_time >= (current_time - mutable_cf_options.ttl)) {
-          break;
-        }
-        total_size -= f->compensated_file_size;
-        inputs[0].files.push_back(f);
+      uint64_t creation_time = f->TryGetFileCreationTime();
+      if (creation_time == kUnknownFileCreationTime ||
+          creation_time >= (current_time - mutable_cf_options.ttl)) {
+        break;
      }
+      total_size -= f->compensated_file_size;
+      inputs[0].files.push_back(f);
    }
  }

@@ -95,14 +92,11 @@ Compaction* FIFOCompactionPicker::PickTTLCompaction(
  }

  for (const auto& f : inputs[0].files) {
-    uint64_t creation_time = 0;
-    if (f && f->fd.table_reader && f->fd.table_reader->GetTableProperties()) {
-      creation_time = f->fd.table_reader->GetTableProperties()->creation_time;
-    }
    ROCKS_LOG_BUFFER(log_buffer,
                     "[%s] FIFO compaction: picking file %" PRIu64
                     " with creation time %" PRIu64 " for deletion",
-                     cf_name.c_str(), f->fd.GetNumber(), creation_time);
+                     cf_name.c_str(), f->fd.GetNumber(),
+                     f->TryGetFileCreationTime());
  }

  Compaction* c = new Compaction(
@@ -324,7 +324,12 @@ TEST_F(DBBasicTest, CheckLock) {
    ASSERT_OK(TryReopen(options));

    // second open should fail
-    ASSERT_TRUE(!(DB::Open(options, dbname_, &localdb)).ok());
+    Status s = DB::Open(options, dbname_, &localdb);
+    ASSERT_NOK(s);
+#ifdef OS_LINUX
+    ASSERT_TRUE(s.ToString().find("lock hold by current process") !=
+                std::string::npos);
+#endif  // OS_LINUX
  } while (ChangeCompactOptions());
 }

@@ -392,7 +397,7 @@ TEST_F(DBBasicTest, FlushEmptyColumnFamily) {
  sleeping_task_low.WaitUntilDone();
 }

-TEST_F(DBBasicTest, FLUSH) {
+TEST_F(DBBasicTest, Flush) {
  do {
    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
    WriteOptions writeOpt = WriteOptions();
@@ -1973,6 +1978,8 @@ class DBBasicTestWithParallelIO
  bool fill_cache_;
 };

+// TODO: fails on CircleCI's Windows env
+#ifndef OS_WIN
 TEST_P(DBBasicTestWithParallelIO, MultiGet) {
  std::vector<std::string> key_data(10);
  std::vector<Slice> keys;
@@ -2095,6 +2102,7 @@ TEST_P(DBBasicTestWithParallelIO, MultiGet) {
    }
  }
 }
+#endif // OS_WIN

 TEST_P(DBBasicTestWithParallelIO, MultiGetWithChecksumMismatch) {
  std::vector<std::string> key_data(10);
@@ -2190,342 +2198,6 @@ INSTANTIATE_TEST_CASE_P(
    ::testing::Combine(::testing::Bool(), ::testing::Bool(),
                       ::testing::Bool(), ::testing::Bool()));

-class DBBasicTestWithTimestampBase : public DBTestBase {
- public:
-  explicit DBBasicTestWithTimestampBase(const std::string& dbname)
-      : DBTestBase(dbname) {}
-
- protected:
-  class TestComparatorBase : public Comparator {
-   public:
-    explicit TestComparatorBase(size_t ts_sz) : Comparator(ts_sz) {}
-
-    const char* Name() const override { return "TestComparator"; }
-
-    void FindShortSuccessor(std::string*) const override {}
-
-    void FindShortestSeparator(std::string*, const Slice&) const override {}
-
-    int Compare(const Slice& a, const Slice& b) const override {
-      int r = CompareWithoutTimestamp(a, b);
-      if (r != 0 || 0 == timestamp_size()) {
-        return r;
-      }
-      return CompareTimestamp(
-          Slice(a.data() + a.size() - timestamp_size(), timestamp_size()),
-          Slice(b.data() + b.size() - timestamp_size(), timestamp_size()));
-    }
-
-    virtual int CompareImpl(const Slice& a, const Slice& b) const = 0;
-
-    int CompareWithoutTimestamp(const Slice& a, const Slice& b) const override {
-      assert(a.size() >= timestamp_size());
-      assert(b.size() >= timestamp_size());
-      Slice k1 = StripTimestampFromUserKey(a, timestamp_size());
-      Slice k2 = StripTimestampFromUserKey(b, timestamp_size());
-
-      return CompareImpl(k1, k2);
-    }
-
-    int CompareTimestamp(const Slice& ts1, const Slice& ts2) const override {
-      if (!ts1.data() && !ts2.data()) {
-        return 0;
-      } else if (ts1.data() && !ts2.data()) {
-        return 1;
-      } else if (!ts1.data() && ts2.data()) {
-        return -1;
-      }
-      assert(ts1.size() == ts2.size());
-      uint64_t low1 = 0;
-      uint64_t low2 = 0;
-      uint64_t high1 = 0;
-      uint64_t high2 = 0;
-      auto* ptr1 = const_cast<Slice*>(&ts1);
-      auto* ptr2 = const_cast<Slice*>(&ts2);
-      if (!GetFixed64(ptr1, &low1) || !GetFixed64(ptr1, &high1) ||
-          !GetFixed64(ptr2, &low2) || !GetFixed64(ptr2, &high2)) {
-        assert(false);
-      }
-      if (high1 < high2) {
-        return 1;
-      } else if (high1 > high2) {
-        return -1;
-      }
-      if (low1 < low2) {
-        return 1;
-      } else if (low1 > low2) {
-        return -1;
-      }
-      return 0;
-    }
-  };
-
-  Slice EncodeTimestamp(uint64_t low, uint64_t high, std::string* ts) {
-    assert(nullptr != ts);
-    ts->clear();
-    PutFixed64(ts, low);
-    PutFixed64(ts, high);
-    assert(ts->size() == sizeof(low) + sizeof(high));
-    return Slice(*ts);
-  }
-};
-
-class DBBasicTestWithTimestamp : public DBBasicTestWithTimestampBase {
- public:
-  DBBasicTestWithTimestamp()
-      : DBBasicTestWithTimestampBase("/db_basic_test_with_timestamp") {}
-
- protected:
-  class TestComparator : public TestComparatorBase {
-   public:
-    const int kKeyPrefixLength =
-        3;  // 3: length of "key" in generated keys ("key" + std::to_string(j))
-    explicit TestComparator(size_t ts_sz) : TestComparatorBase(ts_sz) {}
-
-    int CompareImpl(const Slice& a, const Slice& b) const override {
-      int n1 = atoi(
-          std::string(a.data() + kKeyPrefixLength, a.size() - kKeyPrefixLength)
-              .c_str());
-      int n2 = atoi(
-          std::string(b.data() + kKeyPrefixLength, b.size() - kKeyPrefixLength)
-              .c_str());
-      return (n1 < n2) ? -1 : (n1 > n2) ? 1 : 0;
-    }
-  };
-};
-
-#ifndef ROCKSDB_LITE
-// A class which remembers the name of each flushed file.
-class FlushedFileCollector : public EventListener {
- public:
-  FlushedFileCollector() {}
-  ~FlushedFileCollector() override {}
-
-  void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
-    InstrumentedMutexLock lock(&mutex_);
-    flushed_files_.push_back(info.file_path);
-  }
-
-  std::vector<std::string> GetFlushedFiles() {
-    std::vector<std::string> result;
-    {
-      InstrumentedMutexLock lock(&mutex_);
-      result = flushed_files_;
-    }
-    return result;
-  }
-
-  void ClearFlushedFiles() {
-    InstrumentedMutexLock lock(&mutex_);
-    flushed_files_.clear();
-  }
-
- private:
-  std::vector<std::string> flushed_files_;
-  InstrumentedMutex mutex_;
-};
-
-TEST_F(DBBasicTestWithTimestamp, PutAndGetWithCompaction) {
-  const int kNumKeysPerFile = 8192;
-  const size_t kNumTimestamps = 2;
-  const size_t kNumKeysPerTimestamp = (kNumKeysPerFile - 1) / kNumTimestamps;
-  const size_t kSplitPosBase = kNumKeysPerTimestamp / 2;
-  Options options = CurrentOptions();
-  options.create_if_missing = true;
-  options.env = env_;
-  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
-
-  FlushedFileCollector* collector = new FlushedFileCollector();
-  options.listeners.emplace_back(collector);
-
-  std::string tmp;
-  size_t ts_sz = EncodeTimestamp(0, 0, &tmp).size();
-  TestComparator test_cmp(ts_sz);
-  options.comparator = &test_cmp;
-  BlockBasedTableOptions bbto;
-  bbto.filter_policy.reset(NewBloomFilterPolicy(
-      10 /*bits_per_key*/, false /*use_block_based_builder*/));
-  bbto.whole_key_filtering = true;
-  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  size_t num_cfs = handles_.size();
-  ASSERT_EQ(2, num_cfs);
-  std::vector<std::string> write_ts_strs(kNumTimestamps);
-  std::vector<std::string> read_ts_strs(kNumTimestamps);
-  std::vector<Slice> write_ts_list;
-  std::vector<Slice> read_ts_list;
-
-  for (size_t i = 0; i != kNumTimestamps; ++i) {
-    write_ts_list.emplace_back(EncodeTimestamp(i * 2, 0, &write_ts_strs[i]));
-    read_ts_list.emplace_back(EncodeTimestamp(1 + i * 2, 0, &read_ts_strs[i]));
-    const Slice& write_ts = write_ts_list.back();
-    WriteOptions wopts;
-    wopts.timestamp = &write_ts;
-    for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
-      for (size_t j = 0; j != kNumKeysPerTimestamp; ++j) {
-        ASSERT_OK(Put(cf, "key" + std::to_string(j),
-                      "value_" + std::to_string(j) + "_" + std::to_string(i),
-                      wopts));
-        if (j == kSplitPosBase + i || j == kNumKeysPerTimestamp - 1) {
-          // flush all keys with the same timestamp to two sst files, split at
-          // incremental positions such that lowerlevel[1].smallest.userkey ==
-          // higherlevel[0].largest.userkey
-          ASSERT_OK(Flush(cf));
-
-          // compact files (2 at each level) to a lower level such that all keys
-          // with the same timestamp is at one level, with newer versions at
-          // higher levels.
-          CompactionOptions compact_opt;
-          compact_opt.compression = kNoCompression;
-          db_->CompactFiles(compact_opt, handles_[cf],
-                            collector->GetFlushedFiles(),
-                            static_cast<int>(kNumTimestamps - i));
-          collector->ClearFlushedFiles();
-        }
-      }
-    }
-  }
-  const auto& verify_db_func = [&]() {
-    for (size_t i = 0; i != kNumTimestamps; ++i) {
-      ReadOptions ropts;
-      ropts.timestamp = &read_ts_list[i];
-      for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
-        ColumnFamilyHandle* cfh = handles_[cf];
-        for (size_t j = 0; j != kNumKeysPerTimestamp; ++j) {
-          std::string value;
-          ASSERT_OK(db_->Get(ropts, cfh, "key" + std::to_string(j), &value));
-          ASSERT_EQ("value_" + std::to_string(j) + "_" + std::to_string(i),
-                    value);
-        }
-      }
-    }
-  };
-  verify_db_func();
-}
-#endif  // !ROCKSDB_LITE
-
-class DBBasicTestWithTimestampWithParam
-    : public DBBasicTestWithTimestampBase,
-      public testing::WithParamInterface<bool> {
- public:
-  DBBasicTestWithTimestampWithParam()
-      : DBBasicTestWithTimestampBase(
-            "/db_basic_test_with_timestamp_with_param") {}
-
- protected:
-  class TestComparator : public TestComparatorBase {
-   private:
-    const Comparator* cmp_without_ts_;
-
-   public:
-    explicit TestComparator(size_t ts_sz)
-        : TestComparatorBase(ts_sz), cmp_without_ts_(nullptr) {
-      cmp_without_ts_ = BytewiseComparator();
-    }
-
-    int CompareImpl(const Slice& a, const Slice& b) const override {
-      return cmp_without_ts_->Compare(a, b);
-    }
-  };
-};
-
-TEST_P(DBBasicTestWithTimestampWithParam, PutAndGet) {
-  const int kNumKeysPerFile = 8192;
-  const size_t kNumTimestamps = 6;
-  bool memtable_only = GetParam();
-  Options options = CurrentOptions();
-  options.create_if_missing = true;
-  options.env = env_;
-  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
-  std::string tmp;
-  size_t ts_sz = EncodeTimestamp(0, 0, &tmp).size();
-  TestComparator test_cmp(ts_sz);
-  options.comparator = &test_cmp;
-  BlockBasedTableOptions bbto;
-  bbto.filter_policy.reset(NewBloomFilterPolicy(
-      10 /*bits_per_key*/, false /*use_block_based_builder*/));
-  bbto.whole_key_filtering = true;
-  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-
-  std::vector<CompressionType> compression_types;
-  compression_types.push_back(kNoCompression);
-  if (Zlib_Supported()) {
-    compression_types.push_back(kZlibCompression);
-  }
-#if LZ4_VERSION_NUMBER >= 10400  // r124+
-  compression_types.push_back(kLZ4Compression);
-  compression_types.push_back(kLZ4HCCompression);
-#endif  // LZ4_VERSION_NUMBER >= 10400
-  if (ZSTD_Supported()) {
-    compression_types.push_back(kZSTD);
-  }
-
-  // Switch compression dictionary on/off to check key extraction
-  // correctness in kBuffered state
-  std::vector<uint32_t> max_dict_bytes_list = {0, 1 << 14};  // 0 or 16KB
-
-  for (auto compression_type : compression_types) {
-    for (uint32_t max_dict_bytes : max_dict_bytes_list) {
-      options.compression = compression_type;
-      options.compression_opts.max_dict_bytes = max_dict_bytes;
-      if (compression_type == kZSTD) {
-        options.compression_opts.zstd_max_train_bytes = max_dict_bytes;
-      }
-      options.target_file_size_base = 1 << 26;  // 64MB
-
-      DestroyAndReopen(options);
-      CreateAndReopenWithCF({"pikachu"}, options);
-      size_t num_cfs = handles_.size();
-      ASSERT_EQ(2, num_cfs);
-      std::vector<std::string> write_ts_strs(kNumTimestamps);
-      std::vector<std::string> read_ts_strs(kNumTimestamps);
-      std::vector<Slice> write_ts_list;
-      std::vector<Slice> read_ts_list;
-
-      for (size_t i = 0; i != kNumTimestamps; ++i) {
-        write_ts_list.emplace_back(
-            EncodeTimestamp(i * 2, 0, &write_ts_strs[i]));
-        read_ts_list.emplace_back(
-            EncodeTimestamp(1 + i * 2, 0, &read_ts_strs[i]));
-        const Slice& write_ts = write_ts_list.back();
-        WriteOptions wopts;
-        wopts.timestamp = &write_ts;
-        for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
-          for (size_t j = 0; j != (kNumKeysPerFile - 1) / kNumTimestamps; ++j) {
-            ASSERT_OK(Put(
-                cf, "key" + std::to_string(j),
-                "value_" + std::to_string(j) + "_" + std::to_string(i), wopts));
-          }
-          if (!memtable_only) {
-            ASSERT_OK(Flush(cf));
-          }
-        }
-      }
-      const auto& verify_db_func = [&]() {
-        for (size_t i = 0; i != kNumTimestamps; ++i) {
-          ReadOptions ropts;
-          ropts.timestamp = &read_ts_list[i];
-          for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
-            ColumnFamilyHandle* cfh = handles_[cf];
-            for (size_t j = 0; j != (kNumKeysPerFile - 1) / kNumTimestamps;
-                 ++j) {
-              std::string value;
-              ASSERT_OK(
-                  db_->Get(ropts, cfh, "key" + std::to_string(j), &value));
-              ASSERT_EQ("value_" + std::to_string(j) + "_" + std::to_string(i),
-                        value);
-            }
-          }
-        }
-      };
-      verify_db_func();
-    }
-  }
-}
-
-INSTANTIATE_TEST_CASE_P(Timestamp, DBBasicTestWithTimestampWithParam,
-                        ::testing::Bool());

 }  // namespace ROCKSDB_NAMESPACE

@@ -2459,7 +2459,7 @@ TEST_P(DBCompactionTestWithParam, ManualCompaction) {
    ASSERT_EQ("1,1,1", FilesPerLevel(1));

    // Compaction range overlaps files
-    Compact(1, "p1", "p9");
+    Compact(1, "p", "q");
    ASSERT_EQ("0,0,1", FilesPerLevel(1));

    // Populate a different range
@@ -2526,7 +2526,7 @@ TEST_P(DBCompactionTestWithParam, ManualLevelCompactionOutputPathId) {
    ASSERT_EQ("3", FilesPerLevel(1));

    // Compaction range overlaps files
-    Compact(1, "p1", "p9", 1);
+    Compact(1, "p", "q", 1);
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ("0,1", FilesPerLevel(1));
    ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path));
@@ -4655,7 +4655,7 @@ TEST_P(DBCompactionDirectIOTest, DirectIO) {
  CreateAndReopenWithCF({"pikachu"}, options);
  MakeTables(3, "p", "q", 1);
  ASSERT_EQ("1,1,1", FilesPerLevel(1));
-  Compact(1, "p1", "p9");
+  Compact(1, "p", "q");
  ASSERT_EQ(readahead, options.use_direct_reads);
  ASSERT_EQ("0,0,1", FilesPerLevel(1));
  Destroy(options);
@@ -5151,6 +5151,31 @@ TEST_P(DBCompactionTestWithParam,
  }
 }

+TEST_F(DBCompactionTest, FifoCompactionGetFileCreationTime) {
+  MockEnv mock_env(env_);
+  do {
+    Options options = CurrentOptions();
+    options.table_factory.reset(new BlockBasedTableFactory());
+    options.env = &mock_env;
+    options.ttl = static_cast<uint64_t>(24) * 3600;
+    options.compaction_style = kCompactionStyleFIFO;
+    constexpr size_t kNumFiles = 24;
+    options.max_open_files = 20;
+    constexpr size_t kNumKeysPerFile = 10;
+    DestroyAndReopen(options);
+    for (size_t i = 0; i < kNumFiles; ++i) {
+      for (size_t j = 0; j < kNumKeysPerFile; ++j) {
+        ASSERT_OK(Put(std::to_string(j), "value_" + std::to_string(i)));
+      }
+      ASSERT_OK(Flush());
+    }
+    mock_env.FakeSleepForMicroseconds(
+        static_cast<uint64_t>(1000 * 1000 * (1 + options.ttl)));
+    ASSERT_OK(Put("foo", "value"));
+    ASSERT_OK(Flush());
+  } while (ChangeOptions());
+}
+
 #endif // !defined(ROCKSDB_LITE)
 }  // namespace ROCKSDB_NAMESPACE

@@ -21,7 +21,6 @@
 #include <stdexcept>
 #include <string>
 #include <unordered_map>
-#include <unordered_set>
 #include <utility>
 #include <vector>

@@ -890,9 +889,9 @@ void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) {
  }
 }

-Directory* DBImpl::GetDataDir(ColumnFamilyData* cfd, size_t path_id) const {
+FSDirectory* DBImpl::GetDataDir(ColumnFamilyData* cfd, size_t path_id) const {
  assert(cfd);
-  Directory* ret_dir = cfd->GetDataDir(path_id);
+  FSDirectory* ret_dir = cfd->GetDataDir(path_id);
  if (ret_dir == nullptr) {
    return directories_.GetDataDir(path_id);
  }
@@ -1224,7 +1223,7 @@ Status DBImpl::SyncWAL() {
    }
  }
  if (status.ok() && need_log_dir_sync) {
-    status = directories_.GetWalDir()->Fsync();
+    status = directories_.GetWalDir()->Fsync(IOOptions(), nullptr);
  }
  TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2");

@@ -1496,14 +1495,22 @@ ColumnFamilyHandle* DBImpl::PersistentStatsColumnFamily() const {
 Status DBImpl::Get(const ReadOptions& read_options,
                   ColumnFamilyHandle* column_family, const Slice& key,
                   PinnableSlice* value) {
+  return Get(read_options, column_family, key, value, /*timestamp=*/nullptr);
+}
+
+Status DBImpl::Get(const ReadOptions& read_options,
+                   ColumnFamilyHandle* column_family, const Slice& key,
+                   PinnableSlice* value, std::string* timestamp) {
  GetImplOptions get_impl_options;
  get_impl_options.column_family = column_family;
  get_impl_options.value = value;
-  return GetImpl(read_options, key, get_impl_options);
+  get_impl_options.timestamp = timestamp;
+  Status s = GetImpl(read_options, key, get_impl_options);
+  return s;
 }

 Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
-                       GetImplOptions get_impl_options) {
+                       GetImplOptions& get_impl_options) {
  assert(get_impl_options.value != nullptr ||
         get_impl_options.merge_operands != nullptr);
  PERF_CPU_TIMER_GUARD(get_cpu_nanos, env_);
@@ -1583,10 +1590,14 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
  bool skip_memtable = (read_options.read_tier == kPersistedTier &&
                        has_unpersisted_data_.load(std::memory_order_relaxed));
  bool done = false;
+  const Comparator* comparator =
+      get_impl_options.column_family->GetComparator();
+  size_t ts_sz = comparator->timestamp_size();
+  std::string* timestamp = ts_sz > 0 ? get_impl_options.timestamp : nullptr;
  if (!skip_memtable) {
    // Get value associated with key
    if (get_impl_options.get_value) {
-      if (sv->mem->Get(lkey, get_impl_options.value->GetSelf(), &s,
+      if (sv->mem->Get(lkey, get_impl_options.value->GetSelf(), timestamp, &s,
                       &merge_context, &max_covering_tombstone_seq,
                       read_options, get_impl_options.callback,
                       get_impl_options.is_blob_index)) {
@@ -1594,9 +1605,10 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
        get_impl_options.value->PinSelf();
        RecordTick(stats_, MEMTABLE_HIT);
      } else if ((s.ok() || s.IsMergeInProgress()) &&
-                 sv->imm->Get(lkey, get_impl_options.value->GetSelf(), &s,
-                              &merge_context, &max_covering_tombstone_seq,
-                              read_options, get_impl_options.callback,
+                 sv->imm->Get(lkey, get_impl_options.value->GetSelf(),
+                              timestamp, &s, &merge_context,
+                              &max_covering_tombstone_seq, read_options,
+                              get_impl_options.callback,
                              get_impl_options.is_blob_index)) {
        done = true;
        get_impl_options.value->PinSelf();
@@ -1605,9 +1617,9 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
    } else {
      // Get Merge Operands associated with key, Merge Operands should not be
      // merged and raw values should be returned to the user.
-      if (sv->mem->Get(lkey, nullptr, &s, &merge_context,
-                       &max_covering_tombstone_seq, read_options, nullptr,
-                       nullptr, false)) {
+      if (sv->mem->Get(lkey, /*value*/ nullptr, /*timestamp=*/nullptr, &s,
+                       &merge_context, &max_covering_tombstone_seq,
+                       read_options, nullptr, nullptr, false)) {
        done = true;
        RecordTick(stats_, MEMTABLE_HIT);
      } else if ((s.ok() || s.IsMergeInProgress()) &&
@@ -1626,8 +1638,8 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,
  if (!done) {
    PERF_TIMER_GUARD(get_from_output_files_time);
    sv->current->Get(
-        read_options, lkey, get_impl_options.value, &s, &merge_context,
-        &max_covering_tombstone_seq,
+        read_options, lkey, get_impl_options.value, timestamp, &s,
+        &merge_context, &max_covering_tombstone_seq,
        get_impl_options.get_value ? get_impl_options.value_found : nullptr,
        nullptr, nullptr,
        get_impl_options.get_value ? get_impl_options.callback : nullptr,
@@ -1738,13 +1750,14 @@ std::vector<Status> DBImpl::MultiGet(
         has_unpersisted_data_.load(std::memory_order_relaxed));
    bool done = false;
    if (!skip_memtable) {
-      if (super_version->mem->Get(lkey, value, &s, &merge_context,
-                                  &max_covering_tombstone_seq, read_options)) {
+      if (super_version->mem->Get(lkey, value, /*timestamp=*/nullptr, &s,
+                                  &merge_context, &max_covering_tombstone_seq,
+                                  read_options)) {
        done = true;
        RecordTick(stats_, MEMTABLE_HIT);
-      } else if (super_version->imm->Get(lkey, value, &s, &merge_context,
-                                         &max_covering_tombstone_seq,
-                                         read_options)) {
+      } else if (super_version->imm->Get(
+                     lkey, value, nullptr, &s, &merge_context,
+                     &max_covering_tombstone_seq, read_options)) {
        done = true;
        RecordTick(stats_, MEMTABLE_HIT);
      }
@@ -1752,8 +1765,9 @@ std::vector<Status> DBImpl::MultiGet(
    if (!done) {
      PinnableSlice pinnable_val;
      PERF_TIMER_GUARD(get_from_output_files_time);
-      super_version->current->Get(read_options, lkey, &pinnable_val, &s,
-                                  &merge_context, &max_covering_tombstone_seq);
+      super_version->current->Get(read_options, lkey, &pinnable_val,
+                                  /*timestamp=*/nullptr, &s, &merge_context,
+                                  &max_covering_tombstone_seq);
      value->assign(pinnable_val.data(), pinnable_val.size());
      RecordTick(stats_, MEMTABLE_MISS);
    }
@@ -2301,7 +2315,7 @@ Status DBImpl::CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options,
      auto* cfd =
          versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name);
      assert(cfd != nullptr);
-      std::map<std::string, std::shared_ptr<Directory>> dummy_created_dirs;
+      std::map<std::string, std::shared_ptr<FSDirectory>> dummy_created_dirs;
      s = cfd->AddDirectories(&dummy_created_dirs);
    }
    if (s.ok()) {
@@ -2436,7 +2450,8 @@ Status DBImpl::DropColumnFamilyImpl(ColumnFamilyHandle* column_family) {

 bool DBImpl::KeyMayExist(const ReadOptions& read_options,
                         ColumnFamilyHandle* column_family, const Slice& key,
-                         std::string* value, bool* value_found) {
+                         std::string* value, std::string* timestamp,
+                         bool* value_found) {
  assert(value != nullptr);
  if (value_found != nullptr) {
    // falsify later if key-may-exist but can't fetch value
@@ -2449,6 +2464,7 @@ bool DBImpl::KeyMayExist(const ReadOptions& read_options,
  get_impl_options.column_family = column_family;
  get_impl_options.value = &pinnable_val;
  get_impl_options.value_found = value_found;
+  get_impl_options.timestamp = timestamp;
  auto s = GetImpl(roptions, key, get_impl_options);
  value->assign(pinnable_val.data(), pinnable_val.size());

@@ -3511,24 +3527,15 @@ Status DestroyDB(const std::string& dbname, const Options& options,
      }
    }

-    std::vector<std::string> paths;
-
-    for (const auto& path : options.db_paths) {
-      paths.emplace_back(path.path);
+    std::set<std::string> paths;
+    for (const DbPath& db_path : options.db_paths) {
+      paths.insert(db_path.path);
    }
-    for (const auto& cf : column_families) {
-      for (const auto& path : cf.options.cf_paths) {
-        paths.emplace_back(path.path);
+    for (const ColumnFamilyDescriptor& cf : column_families) {
+      for (const DbPath& cf_path : cf.options.cf_paths) {
+        paths.insert(cf_path.path);
      }
    }
-
-    // Remove duplicate paths.
-    // Note that we compare only the actual paths but not path ids.
-    // This reason is that same path can appear at different path_ids
-    // for different column families.
-    std::sort(paths.begin(), paths.end());
-    paths.erase(std::unique(paths.begin(), paths.end()), paths.end());
-
    for (const auto& path : paths) {
      if (env->GetChildren(path, &filenames).ok()) {
        for (const auto& fname : filenames) {
@@ -3819,8 +3826,9 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
  *found_record_for_key = false;

  // Check if there is a record for this key in the latest memtable
-  sv->mem->Get(lkey, nullptr, &s, &merge_context, &max_covering_tombstone_seq,
-               seq, read_options, nullptr /*read_callback*/, is_blob_index);
+  sv->mem->Get(lkey, nullptr, nullptr, &s, &merge_context,
+               &max_covering_tombstone_seq, seq, read_options,
+               nullptr /*read_callback*/, is_blob_index);

  if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
    // unexpected error reading memtable.
@@ -3845,8 +3853,9 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
  }

  // Check if there is a record for this key in the immutable memtables
-  sv->imm->Get(lkey, nullptr, &s, &merge_context, &max_covering_tombstone_seq,
-               seq, read_options, nullptr /*read_callback*/, is_blob_index);
+  sv->imm->Get(lkey, nullptr, nullptr, &s, &merge_context,
+               &max_covering_tombstone_seq, seq, read_options,
+               nullptr /*read_callback*/, is_blob_index);

  if (!(s.ok() || s.IsNotFound() || s.IsMergeInProgress())) {
    // unexpected error reading memtable.
@@ -3871,7 +3880,7 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
  }

  // Check if there is a record for this key in the immutable memtables
-  sv->imm->GetFromHistory(lkey, nullptr, &s, &merge_context,
+  sv->imm->GetFromHistory(lkey, nullptr, nullptr, &s, &merge_context,
                          &max_covering_tombstone_seq, seq, read_options,
                          is_blob_index);

@@ -3899,7 +3908,7 @@ Status DBImpl::GetLatestSequenceForKey(SuperVersion* sv, const Slice& key,
  // SST files if cache_only=true?
  if (!cache_only) {
    // Check tables
-    sv->current->Get(read_options, lkey, nullptr, &s, &merge_context,
+    sv->current->Get(read_options, lkey, nullptr, nullptr, &s, &merge_context,
                     &max_covering_tombstone_seq, nullptr /* value_found */,
                     found_record_for_key, seq, nullptr /*read_callback*/,
                     is_blob_index);
@@ -82,13 +82,13 @@ struct MemTableInfo;
 // Class to maintain directories for all database paths other than main one.
 class Directories {
 public:
-  Status SetDirectories(Env* env, const std::string& dbname,
-                        const std::string& wal_dir,
-                        const std::vector<DbPath>& data_paths);
+  IOStatus SetDirectories(FileSystem* fs, const std::string& dbname,
+                          const std::string& wal_dir,
+                          const std::vector<DbPath>& data_paths);

-  Directory* GetDataDir(size_t path_id) const {
+  FSDirectory* GetDataDir(size_t path_id) const {
    assert(path_id < data_dirs_.size());
-    Directory* ret_dir = data_dirs_[path_id].get();
+    FSDirectory* ret_dir = data_dirs_[path_id].get();
    if (ret_dir == nullptr) {
      // Should use db_dir_
      return db_dir_.get();
@@ -96,19 +96,19 @@ class Directories {
    return ret_dir;
  }

-  Directory* GetWalDir() {
+  FSDirectory* GetWalDir() {
    if (wal_dir_) {
      return wal_dir_.get();
    }
    return db_dir_.get();
  }

-  Directory* GetDbDir() { return db_dir_.get(); }
+  FSDirectory* GetDbDir() { return db_dir_.get(); }

 private:
-  std::unique_ptr<Directory> db_dir_;
-  std::vector<std::unique_ptr<Directory>> data_dirs_;
-  std::unique_ptr<Directory> wal_dir_;
+  std::unique_ptr<FSDirectory> db_dir_;
+  std::vector<std::unique_ptr<FSDirectory>> data_dirs_;
+  std::unique_ptr<FSDirectory> wal_dir_;
 };

 // While DB is the public interface of RocksDB, and DBImpl is the actual
@@ -163,6 +163,9 @@ class DBImpl : public DB {
  virtual Status Get(const ReadOptions& options,
                     ColumnFamilyHandle* column_family, const Slice& key,
                     PinnableSlice* value) override;
+  virtual Status Get(const ReadOptions& options,
+                     ColumnFamilyHandle* column_family, const Slice& key,
+                     PinnableSlice* value, std::string* timestamp) override;

  using DB::GetMergeOperands;
  Status GetMergeOperands(const ReadOptions& options,
@@ -230,7 +233,7 @@ class DBImpl : public DB {
  using DB::KeyMayExist;
  virtual bool KeyMayExist(const ReadOptions& options,
                           ColumnFamilyHandle* column_family, const Slice& key,
-                           std::string* value,
+                           std::string* value, std::string* timestamp,
                           bool* value_found = nullptr) override;

  using DB::NewIterator;
@@ -433,6 +436,7 @@ class DBImpl : public DB {
  struct GetImplOptions {
    ColumnFamilyHandle* column_family = nullptr;
    PinnableSlice* value = nullptr;
+    std::string* timestamp = nullptr;
    bool* value_found = nullptr;
    ReadCallback* callback = nullptr;
    bool* is_blob_index = nullptr;
@@ -455,7 +459,7 @@ class DBImpl : public DB {
  // If get_impl_options.get_value = false get merge operands associated with
  // get_impl_options.key via get_impl_options.merge_operands
  Status GetImpl(const ReadOptions& options, const Slice& key,
-                 GetImplOptions get_impl_options);
+                 GetImplOptions& get_impl_options);

  ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& options,
                                      ColumnFamilyData* cfd,
@@ -826,8 +830,9 @@ class DBImpl : public DB {
                     std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
                     const bool seq_per_batch, const bool batch_per_txn);

-  static Status CreateAndNewDirectory(Env* env, const std::string& dirname,
-                                      std::unique_ptr<Directory>* directory);
+  static IOStatus CreateAndNewDirectory(
+      FileSystem* fs, const std::string& dirname,
+      std::unique_ptr<FSDirectory>* directory);

  // find stats map from stats_history_ with smallest timestamp in
  // the range of [start_time, end_time)
@@ -1598,7 +1603,7 @@ class DBImpl : public DB {

  uint64_t GetMaxTotalWalSize() const;

-  Directory* GetDataDir(ColumnFamilyData* cfd, size_t path_id) const;
+  FSDirectory* GetDataDir(ColumnFamilyData* cfd, size_t path_id) const;

  Status CloseHelper();

@@ -2021,7 +2026,7 @@ class DBImpl : public DB {
  // REQUIRES: mutex locked
  std::unique_ptr<ROCKSDB_NAMESPACE::RepeatableThread> thread_persist_stats_;

-  // When set, we use a separate queue for writes that dont write to memtable.
+  // When set, we use a separate queue for writes that don't write to memtable.
  // In 2PC these are the writes at Prepare phase.
  const bool two_write_queues_;
  const bool manual_wal_flush_;
@@ -117,7 +117,7 @@ Status DBImpl::SyncClosedLogs(JobContext* job_context) {
      }
    }
    if (s.ok()) {
-      s = directories_.GetWalDir()->Fsync();
+      s = directories_.GetWalDir()->Fsync(IOOptions(), nullptr);
    }

    mutex_.Lock();
@@ -301,7 +301,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
  GetSnapshotContext(job_context, &snapshot_seqs,
                     &earliest_write_conflict_snapshot, &snapshot_checker);

-  autovector<Directory*> distinct_output_dirs;
+  autovector<FSDirectory*> distinct_output_dirs;
  autovector<std::string> distinct_output_dir_paths;
  std::vector<std::unique_ptr<FlushJob>> jobs;
  std::vector<MutableCFOptions> all_mutable_cf_options;
@@ -309,7 +309,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
  all_mutable_cf_options.reserve(num_cfs);
  for (int i = 0; i < num_cfs; ++i) {
    auto cfd = cfds[i];
-    Directory* data_dir = GetDataDir(cfd, 0U);
+    FSDirectory* data_dir = GetDataDir(cfd, 0U);
    const std::string& curr_path = cfd->ioptions()->cf_paths[0].path;

    // Add to distinct output directories if eligible. Use linear search. Since
@@ -413,7 +413,7 @@ Status DBImpl::AtomicFlushMemTablesToOutputFiles(
    // Sync on all distinct output directories.
    for (auto dir : distinct_output_dirs) {
      if (dir != nullptr) {
-        Status error_status = dir->Fsync();
+        Status error_status = dir->Fsync(IOOptions(), nullptr);
        if (!error_status.ok()) {
          s = error_status;
          break;
@@ -652,8 +652,6 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
    return Status::InvalidArgument("Invalid target path ID");
  }

-  bool exclusive = options.exclusive_manual_compaction;
-
  bool flush_needed = true;
  if (begin != nullptr && end != nullptr) {
    // TODO(ajkr): We could also optimize away the flush in certain cases where
@@ -686,25 +684,9 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
    }
  }

-  int max_level_with_files = 0;
-  // max_file_num_to_ignore can be used to filter out newly created SST files,
-  // useful for bottom level compaction in a manual compaction
-  uint64_t max_file_num_to_ignore = port::kMaxUint64;
-  uint64_t next_file_number = port::kMaxUint64;
-  {
-    InstrumentedMutexLock l(&mutex_);
-    Version* base = cfd->current();
-    for (int level = 1; level < base->storage_info()->num_non_empty_levels();
-         level++) {
-      if (base->storage_info()->OverlapInLevel(level, begin, end)) {
-        max_level_with_files = level;
-      }
-    }
-    next_file_number = versions_->current_next_file_number();
-  }
-
-  int final_output_level = 0;
-
+  constexpr int kInvalidLevel = -1;
+  int final_output_level = kInvalidLevel;
+  bool exclusive = options.exclusive_manual_compaction;
  if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal &&
      cfd->NumberLevels() > 1) {
    // Always compact all files together.
@@ -715,58 +697,98 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
    }
    s = RunManualCompaction(cfd, ColumnFamilyData::kCompactAllLevels,
                            final_output_level, options, begin, end, exclusive,
-                            false, max_file_num_to_ignore);
+                            false, port::kMaxUint64);
  } else {
-    for (int level = 0; level <= max_level_with_files; level++) {
+    int first_overlapped_level = kInvalidLevel;
+    int max_overlapped_level = kInvalidLevel;
+    {
+      SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
+      Version* current_version = super_version->current;
+      ReadOptions ro;
+      ro.total_order_seek = true;
+      bool overlap;
+      for (int level = 0;
+           level < current_version->storage_info()->num_non_empty_levels();
+           level++) {
+        overlap = true;
+        if (begin != nullptr && end != nullptr) {
+          Status status = current_version->OverlapWithLevelIterator(
+              ro, file_options_, *begin, *end, level, &overlap);
+          if (!status.ok()) {
+            overlap = current_version->storage_info()->OverlapInLevel(
+                level, begin, end);
+          }
+        } else {
+          overlap = current_version->storage_info()->OverlapInLevel(level,
+                                                                    begin, end);
+        }
+        if (overlap) {
+          if (first_overlapped_level == kInvalidLevel) {
+            first_overlapped_level = level;
+          }
+          max_overlapped_level = level;
+        }
+      }
+      CleanupSuperVersion(super_version);
+    }
+    if (s.ok() && first_overlapped_level != kInvalidLevel) {
+      // max_file_num_to_ignore can be used to filter out newly created SST
+      // files, useful for bottom level compaction in a manual compaction
+      uint64_t max_file_num_to_ignore = port::kMaxUint64;
+      uint64_t next_file_number = versions_->current_next_file_number();
+      final_output_level = max_overlapped_level;
      int output_level;
-      // in case the compaction is universal or if we're compacting the
-      // bottom-most level, the output level will be the same as input one.
-      // level 0 can never be the bottommost level (i.e. if all files are in
-      // level 0, we will compact to level 1)
-      if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
-          cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
-        output_level = level;
-      } else if (level == max_level_with_files && level > 0) {
-        if (options.bottommost_level_compaction ==
-            BottommostLevelCompaction::kSkip) {
-          // Skip bottommost level compaction
-          continue;
-        } else if (options.bottommost_level_compaction ==
-                       BottommostLevelCompaction::kIfHaveCompactionFilter &&
-                   cfd->ioptions()->compaction_filter == nullptr &&
-                   cfd->ioptions()->compaction_filter_factory == nullptr) {
-          // Skip bottommost level compaction since we don't have a compaction
-          // filter
-          continue;
+      for (int level = first_overlapped_level; level <= max_overlapped_level;
+           level++) {
+        // in case the compaction is universal or if we're compacting the
+        // bottom-most level, the output level will be the same as input one.
+        // level 0 can never be the bottommost level (i.e. if all files are in
+        // level 0, we will compact to level 1)
+        if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
+            cfd->ioptions()->compaction_style == kCompactionStyleFIFO) {
+          output_level = level;
+        } else if (level == max_overlapped_level && level > 0) {
+          if (options.bottommost_level_compaction ==
+              BottommostLevelCompaction::kSkip) {
+            // Skip bottommost level compaction
+            continue;
+          } else if (options.bottommost_level_compaction ==
+                         BottommostLevelCompaction::kIfHaveCompactionFilter &&
+                     cfd->ioptions()->compaction_filter == nullptr &&
+                     cfd->ioptions()->compaction_filter_factory == nullptr) {
+            // Skip bottommost level compaction since we don't have a compaction
+            // filter
+            continue;
+          }
+          output_level = level;
+          // update max_file_num_to_ignore only for bottom level compaction
+          // because data in newly compacted files in middle levels may still
+          // need to be pushed down
+          max_file_num_to_ignore = next_file_number;
+        } else {
+          output_level = level + 1;
+          if (cfd->ioptions()->compaction_style == kCompactionStyleLevel &&
+              cfd->ioptions()->level_compaction_dynamic_level_bytes &&
+              level == 0) {
+            output_level = ColumnFamilyData::kCompactToBaseLevel;
+          }
        }
-        output_level = level;
-        // update max_file_num_to_ignore only for bottom level compaction
-        // because data in newly compacted files in middle levels may still need
-        // to be pushed down
-        max_file_num_to_ignore = next_file_number;
-      } else {
-        output_level = level + 1;
-        if (cfd->ioptions()->compaction_style == kCompactionStyleLevel &&
-            cfd->ioptions()->level_compaction_dynamic_level_bytes &&
-            level == 0) {
-          output_level = ColumnFamilyData::kCompactToBaseLevel;
+        s = RunManualCompaction(cfd, level, output_level, options, begin, end,
+                                exclusive, false, max_file_num_to_ignore);
+        if (!s.ok()) {
+          break;
        }
+        if (output_level == ColumnFamilyData::kCompactToBaseLevel) {
+          final_output_level = cfd->NumberLevels() - 1;
+        } else if (output_level > final_output_level) {
+          final_output_level = output_level;
+        }
+        TEST_SYNC_POINT("DBImpl::RunManualCompaction()::1");
+        TEST_SYNC_POINT("DBImpl::RunManualCompaction()::2");
      }
-      s = RunManualCompaction(cfd, level, output_level, options, begin, end,
-                              exclusive, false, max_file_num_to_ignore);
-      if (!s.ok()) {
-        break;
-      }
-      if (output_level == ColumnFamilyData::kCompactToBaseLevel) {
-        final_output_level = cfd->NumberLevels() - 1;
-      } else if (output_level > final_output_level) {
-        final_output_level = output_level;
-      }
-      TEST_SYNC_POINT("DBImpl::RunManualCompaction()::1");
-      TEST_SYNC_POINT("DBImpl::RunManualCompaction()::2");
    }
  }
-  if (!s.ok()) {
+  if (!s.ok() || final_output_level == kInvalidLevel) {
    LogFlush(immutable_db_options_.info_log);
    return s;
  }
@@ -1037,7 +1059,7 @@ Status DBImpl::CompactFilesImpl(
  }

  if (output_file_names != nullptr) {
-    for (const auto newf : c->edit()->GetNewFiles()) {
+    for (const auto& newf : c->edit()->GetNewFiles()) {
      (*output_file_names)
          .push_back(TableFileName(c->immutable_cf_options()->cf_paths,
                                   newf.second.fd.GetNumber(),
@@ -1137,7 +1159,7 @@ void DBImpl::NotifyOnCompactionBegin(ColumnFamilyData* cfd, Compaction* c,
        }
      }
    }
-    for (const auto newf : c->edit()->GetNewFiles()) {
+    for (const auto& newf : c->edit()->GetNewFiles()) {
      const FileMetaData& meta = newf.second;
      const FileDescriptor& desc = meta.fd;
      const uint64_t file_number = desc.GetNumber();
@@ -301,8 +301,9 @@ Status DBImpl::NewDB() {
  return s;
 }

-Status DBImpl::CreateAndNewDirectory(Env* env, const std::string& dirname,
-                                     std::unique_ptr<Directory>* directory) {
+IOStatus DBImpl::CreateAndNewDirectory(
+    FileSystem* fs, const std::string& dirname,
+    std::unique_ptr<FSDirectory>* directory) {
  // We call CreateDirIfMissing() as the directory may already exist (if we
  // are reopening a DB), when this happens we don't want creating the
  // directory to cause an error. However, we need to check if creating the
@@ -310,24 +311,24 @@ Status DBImpl::CreateAndNewDirectory(Env* env, const std::string& dirname,
  // file not existing. One real-world example of this occurring is if
  // env->CreateDirIfMissing() doesn't create intermediate directories, e.g.
  // when dbname_ is "dir/db" but when "dir" doesn't exist.
-  Status s = env->CreateDirIfMissing(dirname);
-  if (!s.ok()) {
-    return s;
+  IOStatus io_s = fs->CreateDirIfMissing(dirname, IOOptions(), nullptr);
+  if (!io_s.ok()) {
+    return io_s;
  }
-  return env->NewDirectory(dirname, directory);
+  return fs->NewDirectory(dirname, IOOptions(), directory, nullptr);
 }

-Status Directories::SetDirectories(Env* env, const std::string& dbname,
-                                   const std::string& wal_dir,
-                                   const std::vector<DbPath>& data_paths) {
-  Status s = DBImpl::CreateAndNewDirectory(env, dbname, &db_dir_);
-  if (!s.ok()) {
-    return s;
+IOStatus Directories::SetDirectories(FileSystem* fs, const std::string& dbname,
+                                     const std::string& wal_dir,
+                                     const std::vector<DbPath>& data_paths) {
+  IOStatus io_s = DBImpl::CreateAndNewDirectory(fs, dbname, &db_dir_);
+  if (!io_s.ok()) {
+    return io_s;
  }
  if (!wal_dir.empty() && dbname != wal_dir) {
-    s = DBImpl::CreateAndNewDirectory(env, wal_dir, &wal_dir_);
-    if (!s.ok()) {
-      return s;
+    io_s = DBImpl::CreateAndNewDirectory(fs, wal_dir, &wal_dir_);
+    if (!io_s.ok()) {
+      return io_s;
    }
  }

@@ -337,16 +338,16 @@ Status Directories::SetDirectories(Env* env, const std::string& dbname,
    if (db_path == dbname) {
      data_dirs_.emplace_back(nullptr);
    } else {
-      std::unique_ptr<Directory> path_directory;
-      s = DBImpl::CreateAndNewDirectory(env, db_path, &path_directory);
-      if (!s.ok()) {
-        return s;
+      std::unique_ptr<FSDirectory> path_directory;
+      io_s = DBImpl::CreateAndNewDirectory(fs, db_path, &path_directory);
+      if (!io_s.ok()) {
+        return io_s;
      }
      data_dirs_.emplace_back(path_directory.release());
    }
  }
  assert(data_dirs_.size() == data_paths.size());
-  return Status::OK();
+  return IOStatus::OK();
 }

 Status DBImpl::Recover(
@@ -358,7 +359,7 @@ Status DBImpl::Recover(
  bool is_new_db = false;
  assert(db_lock_ == nullptr);
  if (!read_only) {
-    Status s = directories_.SetDirectories(env_, dbname_,
+    Status s = directories_.SetDirectories(fs_.get(), dbname_,
                                           immutable_db_options_.wal_dir,
                                           immutable_db_options_.db_paths);
    if (!s.ok()) {
@@ -458,7 +459,7 @@ Status DBImpl::Recover(
    s = CheckConsistency();
  }
  if (s.ok() && !read_only) {
-    std::map<std::string, std::shared_ptr<Directory>> created_dirs;
+    std::map<std::string, std::shared_ptr<FSDirectory>> created_dirs;
    for (auto cfd : *versions_->GetColumnFamilySet()) {
      s = cfd->AddDirectories(&created_dirs);
      if (!s.ok()) {
@@ -1400,13 +1401,9 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
      impl->error_handler_.EnableAutoRecovery();
    }
  }
-
-  if (!s.ok()) {
-    delete impl;
-    return s;
+  if (s.ok()) {
+    s = impl->CreateArchivalDirectory();
  }
-
-  s = impl->CreateArchivalDirectory();
  if (!s.ok()) {
    delete impl;
    return s;
@@ -1477,7 +1474,7 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
      }

      impl->DeleteObsoleteFiles();
-      s = impl->directories_.GetDbDir()->Fsync();
+      s = impl->directories_.GetDbDir()->Fsync(IOOptions(), nullptr);
    }
    if (s.ok()) {
      // In WritePrepared there could be gap in sequence numbers. This breaks
@@ -48,14 +48,16 @@ Status DBImplReadOnly::Get(const ReadOptions& read_options,
  SequenceNumber max_covering_tombstone_seq = 0;
  LookupKey lkey(key, snapshot);
  PERF_TIMER_STOP(get_snapshot_time);
-  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
+  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(),
+                              /*timestamp=*/nullptr, &s, &merge_context,
                              &max_covering_tombstone_seq, read_options)) {
    pinnable_val->PinSelf();
    RecordTick(stats_, MEMTABLE_HIT);
  } else {
    PERF_TIMER_GUARD(get_from_output_files_time);
-    super_version->current->Get(read_options, lkey, pinnable_val, &s,
-                                &merge_context, &max_covering_tombstone_seq);
+    super_version->current->Get(read_options, lkey, pinnable_val,
+                                /*timestamp=*/nullptr, &s, &merge_context,
+                                &max_covering_tombstone_seq);
    RecordTick(stats_, MEMTABLE_MISS);
  }
  RecordTick(stats_, NUMBER_KEYS_READ);
@@ -340,15 +340,16 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
  PERF_TIMER_STOP(get_snapshot_time);

  bool done = false;
-  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
+  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(),
+                              /*timestamp=*/nullptr, &s, &merge_context,
                              &max_covering_tombstone_seq, read_options)) {
    done = true;
    pinnable_val->PinSelf();
    RecordTick(stats_, MEMTABLE_HIT);
  } else if ((s.ok() || s.IsMergeInProgress()) &&
             super_version->imm->Get(
-                 lkey, pinnable_val->GetSelf(), &s, &merge_context,
-                 &max_covering_tombstone_seq, read_options)) {
+                 lkey, pinnable_val->GetSelf(), /*timestamp=*/nullptr, &s,
+                 &merge_context, &max_covering_tombstone_seq, read_options)) {
    done = true;
    pinnable_val->PinSelf();
    RecordTick(stats_, MEMTABLE_HIT);
@@ -359,8 +360,9 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
  }
  if (!done) {
    PERF_TIMER_GUARD(get_from_output_files_time);
-    super_version->current->Get(read_options, lkey, pinnable_val, &s,
-                                &merge_context, &max_covering_tombstone_seq);
+    super_version->current->Get(read_options, lkey, pinnable_val,
+                                /*timestamp=*/nullptr, &s, &merge_context,
+                                &max_covering_tombstone_seq);
    RecordTick(stats_, MEMTABLE_MISS);
  }
  {
@@ -1041,7 +1041,7 @@ Status DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
      // We only sync WAL directory the first time WAL syncing is
      // requested, so that in case users never turn on WAL sync,
      // we can avoid the disk I/O in the write code path.
-      status = directories_.GetWalDir()->Fsync();
+      status = directories_.GetWalDir()->Fsync(IOOptions(), nullptr);
    }
  }

@@ -49,6 +49,8 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options,
      read_callback_(read_callback),
      sequence_(s),
      statistics_(cf_options.statistics),
+      max_skip_(max_sequential_skip_in_iterations),
+      max_skippable_internal_keys_(read_options.max_skippable_internal_keys),
      num_internal_keys_skipped_(0),
      iterate_lower_bound_(read_options.iterate_lower_bound),
      iterate_upper_bound_(read_options.iterate_upper_bound),
@@ -69,16 +71,17 @@ DBIter::DBIter(Env* _env, const ReadOptions& read_options,
      range_del_agg_(&cf_options.internal_comparator, s),
      db_impl_(db_impl),
      cfd_(cfd),
-      start_seqnum_(read_options.iter_start_seqnum) {
+      start_seqnum_(read_options.iter_start_seqnum),
+      timestamp_ub_(read_options.timestamp),
+      timestamp_size_(timestamp_ub_ ? timestamp_ub_->size() : 0) {
  RecordTick(statistics_, NO_ITERATOR_CREATED);
-  max_skip_ = max_sequential_skip_in_iterations;
-  max_skippable_internal_keys_ = read_options.max_skippable_internal_keys;
  if (pin_thru_lifetime_) {
    pinned_iters_mgr_.StartPinning();
  }
  if (iter_.iter()) {
    iter_.iter()->SetPinnedItersMgr(&pinned_iters_mgr_);
  }
+  assert(timestamp_size_ == user_comparator_.timestamp_size());
 }

 Status DBIter::GetProperty(std::string prop_name, std::string* prop) {
@@ -220,9 +223,13 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
    is_key_seqnum_zero_ = (ikey_.sequence == 0);

    assert(iterate_upper_bound_ == nullptr || iter_.MayBeOutOfUpperBound() ||
-           user_comparator_.Compare(ikey_.user_key, *iterate_upper_bound_) < 0);
+           user_comparator_.CompareWithoutTimestamp(
+               ikey_.user_key, /*a_has_ts=*/true, *iterate_upper_bound_,
+               /*b_has_ts=*/false) < 0);
    if (iterate_upper_bound_ != nullptr && iter_.MayBeOutOfUpperBound() &&
-        user_comparator_.Compare(ikey_.user_key, *iterate_upper_bound_) >= 0) {
+        user_comparator_.CompareWithoutTimestamp(
+            ikey_.user_key, /*a_has_ts=*/true, *iterate_upper_bound_,
+            /*b_has_ts=*/false) >= 0) {
      break;
    }

@@ -237,20 +244,25 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
      return false;
    }

-    if (IsVisible(ikey_.sequence)) {
+    assert(ikey_.user_key.size() >= timestamp_size_);
+    Slice ts;
+    if (timestamp_size_ > 0) {
+      ts = ExtractTimestampFromUserKey(ikey_.user_key, timestamp_size_);
+    }
+    if (IsVisible(ikey_.sequence, ts)) {
      // If the previous entry is of seqnum 0, the current entry will not
      // possibly be skipped. This condition can potentially be relaxed to
      // prev_key.seq <= ikey_.sequence. We are cautious because it will be more
      // prone to bugs causing the same user key with the same sequence number.
      if (!is_prev_key_seqnum_zero && skipping_saved_key &&
-          user_comparator_.Compare(ikey_.user_key, saved_key_.GetUserKey()) <=
-              0) {
+          user_comparator_.CompareWithoutTimestamp(
+              ikey_.user_key, saved_key_.GetUserKey()) <= 0) {
        num_skipped++;  // skip this entry
        PERF_COUNTER_ADD(internal_key_skipped_count, 1);
      } else {
        assert(!skipping_saved_key ||
-               user_comparator_.Compare(ikey_.user_key,
-                                        saved_key_.GetUserKey()) > 0);
+               user_comparator_.CompareWithoutTimestamp(
+                   ikey_.user_key, saved_key_.GetUserKey()) > 0);
        num_skipped = 0;
        reseek_done = false;
        switch (ikey_.type) {
@@ -356,8 +368,8 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
      // This key was inserted after our snapshot was taken.
      // If this happens too many times in a row for the same user key, we want
      // to seek to the target sequence number.
-      int cmp =
-          user_comparator_.Compare(ikey_.user_key, saved_key_.GetUserKey());
+      int cmp = user_comparator_.CompareWithoutTimestamp(
+          ikey_.user_key, saved_key_.GetUserKey());
      if (cmp == 0 || (skipping_saved_key && cmp < 0)) {
        num_skipped++;
      } else {
@@ -388,8 +400,17 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
        // We're looking for the next user-key but all we see are the same
        // user-key with decreasing sequence numbers. Fast forward to
        // sequence number 0 and type deletion (the smallest type).
-        AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetUserKey(),
-                                                       0, kTypeDeletion));
+        if (timestamp_size_ == 0) {
+          AppendInternalKey(
+              &last_key,
+              ParsedInternalKey(saved_key_.GetUserKey(), 0, kTypeDeletion));
+        } else {
+          std::string min_ts(timestamp_size_, static_cast<char>(0));
+          AppendInternalKeyWithDifferentTimestamp(
+              &last_key,
+              ParsedInternalKey(saved_key_.GetUserKey(), 0, kTypeDeletion),
+              min_ts);
+        }
        // Don't set skipping_saved_key = false because we may still see more
        // user-keys equal to saved_key_.
      } else {
@@ -398,9 +419,17 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
        // Note that this only covers a case when a higher key was overwritten
        // many times since our snapshot was taken, not the case when a lot of
        // different keys were inserted after our snapshot was taken.
-        AppendInternalKey(&last_key,
-                          ParsedInternalKey(saved_key_.GetUserKey(), sequence_,
-                                            kValueTypeForSeek));
+        if (timestamp_size_ == 0) {
+          AppendInternalKey(
+              &last_key, ParsedInternalKey(saved_key_.GetUserKey(), sequence_,
+                                           kValueTypeForSeek));
+        } else {
+          AppendInternalKeyWithDifferentTimestamp(
+              &last_key,
+              ParsedInternalKey(saved_key_.GetUserKey(), sequence_,
+                                kValueTypeForSeek),
+              *timestamp_ub_);
+        }
      }
      iter_.Seek(last_key);
      RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
@@ -519,6 +548,13 @@ bool DBIter::MergeValuesNewToOld() {
 }

 void DBIter::Prev() {
+  if (timestamp_size_ > 0) {
+    valid_ = false;
+    status_ = Status::NotSupported(
+        "SeekToLast/SeekForPrev/Prev currently not supported with timestamp.");
+    return;
+  }
+
  assert(valid_);
  assert(status_.ok());

@@ -697,7 +733,13 @@ bool DBIter::FindValueForCurrentKey() {
      return false;
    }

-    if (!IsVisible(ikey.sequence) ||
+    assert(ikey.user_key.size() >= timestamp_size_);
+    Slice ts;
+    if (timestamp_size_ > 0) {
+      ts = Slice(ikey.user_key.data() + ikey.user_key.size() - timestamp_size_,
+                 timestamp_size_);
+    }
+    if (!IsVisible(ikey.sequence, ts) ||
        !user_comparator_.Equal(ikey.user_key, saved_key_.GetUserKey())) {
      break;
    }
@@ -853,6 +895,13 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
    if (!ParseKey(&ikey)) {
      return false;
    }
+    assert(ikey.user_key.size() >= timestamp_size_);
+    Slice ts;
+    if (timestamp_size_ > 0) {
+      ts = Slice(ikey.user_key.data() + ikey.user_key.size() - timestamp_size_,
+                 timestamp_size_);
+    }
+
    if (!user_comparator_.Equal(ikey.user_key, saved_key_.GetUserKey())) {
      // No visible values for this key, even though FindValueForCurrentKey()
      // has seen some. This is possible if we're using a tailing iterator, and
@@ -861,7 +910,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
      return true;
    }

-    if (IsVisible(ikey.sequence)) {
+    if (IsVisible(ikey.sequence, ts)) {
      break;
    }

@@ -1001,7 +1050,13 @@ bool DBIter::FindUserKeyBeforeSavedKey() {
    }

    assert(ikey.sequence != kMaxSequenceNumber);
-    if (!IsVisible(ikey.sequence)) {
+    assert(ikey.user_key.size() >= timestamp_size_);
+    Slice ts;
+    if (timestamp_size_ > 0) {
+      ts = Slice(ikey.user_key.data() + ikey.user_key.size() - timestamp_size_,
+                 timestamp_size_);
+    }
+    if (!IsVisible(ikey.sequence, ts)) {
      PERF_COUNTER_ADD(internal_recent_skipped_count, 1);
    } else {
      PERF_COUNTER_ADD(internal_key_skipped_count, 1);
@@ -1046,10 +1101,18 @@ bool DBIter::TooManyInternalKeysSkipped(bool increment) {
  return false;
 }

-bool DBIter::IsVisible(SequenceNumber sequence) {
+bool DBIter::IsVisible(SequenceNumber sequence, const Slice& ts) {
+  // Remember that comparator orders preceding timestamp as larger.
+  int cmp_ts = timestamp_ub_ != nullptr
+                   ? user_comparator_.CompareTimestamp(ts, *timestamp_ub_)
+                   : 0;
+  if (cmp_ts > 0) {
+    return false;
+  }
  if (read_callback_ == nullptr) {
    return sequence <= sequence_;
  } else {
+    // TODO(yanqin): support timestamp in read_callback_.
    return read_callback_->IsVisible(sequence);
  }
 }
@@ -1058,14 +1121,16 @@ void DBIter::SetSavedKeyToSeekTarget(const Slice& target) {
  is_key_seqnum_zero_ = false;
  SequenceNumber seq = sequence_;
  saved_key_.Clear();
-  saved_key_.SetInternalKey(target, seq);
+  saved_key_.SetInternalKey(target, seq, kValueTypeForSeek, timestamp_ub_);

  if (iterate_lower_bound_ != nullptr &&
-      user_comparator_.Compare(saved_key_.GetUserKey(), *iterate_lower_bound_) <
-          0) {
+      user_comparator_.CompareWithoutTimestamp(
+          saved_key_.GetUserKey(), /*a_has_ts=*/true, *iterate_lower_bound_,
+          /*b_has_ts=*/false) < 0) {
    // Seek key is smaller than the lower bound.
    saved_key_.Clear();
-    saved_key_.SetInternalKey(*iterate_lower_bound_, seq);
+    saved_key_.SetInternalKey(*iterate_lower_bound_, seq, kValueTypeForSeek,
+                              timestamp_ub_);
  }
 }

@@ -1155,6 +1220,13 @@ void DBIter::SeekForPrev(const Slice& target) {
  }
 #endif  // ROCKSDB_LITE

+  if (timestamp_size_ > 0) {
+    valid_ = false;
+    status_ = Status::NotSupported(
+        "SeekToLast/SeekForPrev/Prev currently not supported with timestamp.");
+    return;
+  }
+
  status_ = Status::OK();
  ReleaseTempPinnedData();
  ResetInternalKeysSkippedCounter();
@@ -1248,6 +1320,13 @@ void DBIter::SeekToFirst() {
 }

 void DBIter::SeekToLast() {
+  if (timestamp_size_ > 0) {
+    valid_ = false;
+    status_ = Status::NotSupported(
+        "SeekToLast/SeekForPrev/Prev currently not supported with timestamp.");
+    return;
+  }
+
  if (iterate_upper_bound_ != nullptr) {
    // Seek to last key strictly less than ReadOptions.iterate_upper_bound.
    SeekForPrev(*iterate_upper_bound_);
@@ -146,7 +146,8 @@ class DBIter final : public Iterator {
    if (start_seqnum_ > 0) {
      return saved_key_.GetInternalKey();
    } else {
-      return saved_key_.GetUserKey();
+      const Slice ukey_and_ts = saved_key_.GetUserKey();
+      return Slice(ukey_and_ts.data(), ukey_and_ts.size() - timestamp_size_);
    }
  }
  Slice value() const override {
@@ -169,6 +170,13 @@ class DBIter final : public Iterator {
      return status_;
    }
  }
+  Slice timestamp() const override {
+    assert(valid_);
+    assert(timestamp_size_ > 0);
+    const Slice ukey_and_ts = saved_key_.GetUserKey();
+    assert(timestamp_size_ < ukey_and_ts.size());
+    return ExtractTimestampFromUserKey(ukey_and_ts, timestamp_size_);
+  }
  bool IsBlob() const {
    assert(valid_ && (allow_blob_ || !is_blob_));
    return is_blob_;
@@ -178,6 +186,8 @@ class DBIter final : public Iterator {

  void Next() final override;
  void Prev() final override;
+  // 'target' does not contain timestamp, even if user timestamp feature is
+  // enabled.
  void Seek(const Slice& target) final override;
  void SeekForPrev(const Slice& target) final override;
  void SeekToFirst() final override;
@@ -221,7 +231,7 @@ class DBIter final : public Iterator {
  // entry can be found within the prefix.
  void PrevInternal(const Slice* prefix);
  bool TooManyInternalKeysSkipped(bool increment = true);
-  bool IsVisible(SequenceNumber sequence);
+  bool IsVisible(SequenceNumber sequence, const Slice& ts);

  // Temporarily pin the blocks that we encounter until ReleaseTempPinnedData()
  // is called
@@ -327,6 +337,8 @@ class DBIter final : public Iterator {
  // for diff snapshots we want the lower bound on the seqnum;
  // if this value > 0 iterator will return internal keys
  SequenceNumber start_seqnum_;
+  const Slice* const timestamp_ub_;
+  const size_t timestamp_size_;
 };

 // Return a new iterator that converts internal keys (yielded by
@@ -1873,7 +1873,7 @@ TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocksRandomized) {
  DestroyAndReopen(options);

  const int kNumKeys = 500;
-  // Small number of merge operands to make sure that DBIter::Prev() dont
+  // Small number of merge operands to make sure that DBIter::Prev() don't
  // fall back to Seek()
  const int kNumMergeOperands = 3;
  // Use value size that will make sure that every block contain 1 key
@@ -1908,7 +1908,7 @@ TEST_P(DBIteratorTest, IterPrevKeyCrossingBlocksRandomized) {
  ASSERT_OK(Flush());

  // Separate values and merge operands in different file so that we
-  // make sure that we dont merge them while flushing but actually
+  // make sure that we don't merge them while flushing but actually
  // merge them in the read path
  for (int i = 0; i < kNumKeys; i++) {
    if (rnd.PercentTrue(kNoMergeOpPercentage)) {
@@ -0,0 +1,515 @@
+// Copyright (c) 2020-present, Facebook, Inc.  All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "test_util/testharness.h"
+
+#ifdef OS_LINUX
+#include "env/io_posix.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+
+namespace ROCKSDB_NAMESPACE {
+class EnvWithCustomLogicalBlockSizeCache : public EnvWrapper {
+ public:
+  EnvWithCustomLogicalBlockSizeCache(Env* env, LogicalBlockSizeCache* cache)
+      : EnvWrapper(env), cache_(cache) {}
+
+  Status RegisterDbPaths(const std::vector<std::string>& paths) override {
+    return cache_->RefAndCacheLogicalBlockSize(paths);
+  }
+
+  Status UnregisterDbPaths(const std::vector<std::string>& paths) override {
+    cache_->UnrefAndTryRemoveCachedLogicalBlockSize(paths);
+    return Status::OK();
+  }
+
+ private:
+  LogicalBlockSizeCache* cache_;
+};
+
+class DBLogicalBlockSizeCacheTest : public testing::Test {
+ public:
+  DBLogicalBlockSizeCacheTest()
+      : dbname_(test::PerThreadDBPath("logical_block_size_cache_test")),
+        data_path_0_(dbname_ + "/data_path_0"),
+        data_path_1_(dbname_ + "/data_path_1"),
+        cf_path_0_(dbname_ + "/cf_path_0"),
+        cf_path_1_(dbname_ + "/cf_path_1") {
+    auto get_fd_block_size = [&](int fd) {
+      return fd;
+    };
+    auto get_dir_block_size = [&](const std::string& /*dir*/, size_t* size) {
+      *size = 1024;
+      return Status::OK();
+    };
+    cache_.reset(new LogicalBlockSizeCache(
+        get_fd_block_size, get_dir_block_size));
+    env_.reset(new EnvWithCustomLogicalBlockSizeCache(
+        Env::Default(), cache_.get()));
+  }
+
+ protected:
+  std::string dbname_;
+  std::string data_path_0_;
+  std::string data_path_1_;
+  std::string cf_path_0_;
+  std::string cf_path_1_;
+  std::unique_ptr<LogicalBlockSizeCache> cache_;
+  std::unique_ptr<Env> env_;
+};
+
+TEST_F(DBLogicalBlockSizeCacheTest, OpenClose) {
+  // Tests that Open will cache the logical block size for data paths,
+  // and Close will remove the cached sizes.
+  Options options;
+  options.create_if_missing = true;
+  options.env = env_.get();
+  options.db_paths = {{data_path_0_, 2048}, {data_path_1_, 2048}};
+
+  for (int i = 0; i < 2; i++) {
+    DB* db;
+    if (!i) {
+      printf("Open\n");
+      ASSERT_OK(DB::Open(options, dbname_, &db));
+    } else {
+#ifdef ROCKSDB_LITE
+      break;
+#else
+      printf("OpenForReadOnly\n");
+      ASSERT_OK(DB::OpenForReadOnly(options, dbname_, &db));
+#endif
+    }
+    ASSERT_EQ(2, cache_->Size());
+    ASSERT_TRUE(cache_->Contains(data_path_0_));
+    ASSERT_EQ(1, cache_->GetRefCount(data_path_0_));
+    ASSERT_TRUE(cache_->Contains(data_path_1_));
+    ASSERT_EQ(1, cache_->GetRefCount(data_path_1_));
+    ASSERT_OK(db->Close());
+    ASSERT_EQ(0, cache_->Size());
+    delete db;
+  }
+  ASSERT_OK(DestroyDB(dbname_, options, {}));
+}
+
+TEST_F(DBLogicalBlockSizeCacheTest, OpenDelete) {
+  // Tests that Open will cache the logical block size for data paths,
+  // and delete the db pointer will remove the cached sizes.
+  Options options;
+  options.create_if_missing = true;
+  options.env = env_.get();
+
+  for (int i = 0; i < 2; i++) {
+    DB* db;
+    if (!i) {
+      printf("Open\n");
+      ASSERT_OK(DB::Open(options, dbname_, &db));
+    } else {
+#ifdef ROCKSDB_LITE
+      break;
+#else
+      printf("OpenForReadOnly\n");
+      ASSERT_OK(DB::OpenForReadOnly(options, dbname_, &db));
+#endif
+    }
+    ASSERT_EQ(1, cache_->Size());
+    ASSERT_TRUE(cache_->Contains(dbname_));
+    ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+    delete db;
+    ASSERT_EQ(0, cache_->Size());
+  }
+  ASSERT_OK(DestroyDB(dbname_, options, {}));
+}
+
+TEST_F(DBLogicalBlockSizeCacheTest, CreateColumnFamily) {
+  // Tests that CreateColumnFamily will cache the cf_paths,
+  // drop the column family handle won't drop the cache,
+  // drop and then delete the column family handle will drop the cache.
+  Options options;
+  options.create_if_missing = true;
+  options.env = env_.get();
+  ColumnFamilyOptions cf_options;
+  cf_options.cf_paths = {{cf_path_0_, 1024}, {cf_path_1_, 2048}};
+
+  DB* db;
+  ASSERT_OK(DB::Open(options, dbname_, &db));
+  ASSERT_EQ(1, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(dbname_));
+  ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+
+  ColumnFamilyHandle* cf = nullptr;
+  ASSERT_OK(db->CreateColumnFamily(cf_options, "cf", &cf));
+  ASSERT_EQ(3, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(dbname_));
+  ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+  ASSERT_TRUE(cache_->Contains(cf_path_1_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_1_));
+
+  // Drop column family does not drop cache.
+  ASSERT_OK(db->DropColumnFamily(cf));
+  ASSERT_EQ(3, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(dbname_));
+  ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+  ASSERT_TRUE(cache_->Contains(cf_path_1_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_1_));
+
+  // Delete handle will drop cache.
+  ASSERT_OK(db->DestroyColumnFamilyHandle(cf));
+  ASSERT_TRUE(cache_->Contains(dbname_));
+  ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+
+  delete db;
+  ASSERT_EQ(0, cache_->Size());
+  ASSERT_OK(DestroyDB(dbname_, options, {{"cf", cf_options}}));
+}
+
+TEST_F(DBLogicalBlockSizeCacheTest, CreateColumnFamilies) {
+  // Tests that CreateColumnFamilies will cache the cf_paths,
+  // drop the column family handle won't drop the cache,
+  // drop and then delete the column family handle will drop the cache.
+  Options options;
+  options.create_if_missing = true;
+  options.env = env_.get();
+  ColumnFamilyOptions cf_options;
+  cf_options.cf_paths = {{cf_path_0_, 1024}};
+
+  DB* db;
+  ASSERT_OK(DB::Open(options, dbname_, &db));
+  ASSERT_EQ(1, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(dbname_));
+  ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+
+  std::vector<ColumnFamilyHandle*> cfs;
+  ASSERT_OK(db->CreateColumnFamilies(cf_options, {"cf1", "cf2"}, &cfs));
+  ASSERT_EQ(2, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(dbname_));
+  ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_));
+
+  // Drop column family does not drop cache.
+  for (ColumnFamilyHandle* cf : cfs) {
+    ASSERT_OK(db->DropColumnFamily(cf));
+    ASSERT_EQ(2, cache_->Size());
+    ASSERT_TRUE(cache_->Contains(dbname_));
+    ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+    ASSERT_TRUE(cache_->Contains(cf_path_0_));
+    ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_));
+  }
+
+  // Delete one handle will not drop cache because another handle is still
+  // referencing cf_path_0_.
+  ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[0]));
+  ASSERT_EQ(2, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(dbname_));
+  ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+
+  // Delete the last handle will drop cache.
+  ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[1]));
+  ASSERT_EQ(1, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(dbname_));
+  ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+
+  delete db;
+  ASSERT_EQ(0, cache_->Size());
+  ASSERT_OK(DestroyDB(dbname_, options,
+      {{"cf1", cf_options}, {"cf2", cf_options}}));
+}
+
+TEST_F(DBLogicalBlockSizeCacheTest, OpenWithColumnFamilies) {
+  // Tests that Open two column families with the same cf_path will cache the
+  // cf_path and have 2 references to the cached size,
+  // drop the column family handle won't drop the cache,
+  // drop and then delete the column family handle will drop the cache.
+  Options options;
+  options.create_if_missing = true;
+  options.env = env_.get();
+
+  ColumnFamilyOptions cf_options;
+  cf_options.cf_paths = {{cf_path_0_, 1024}};
+
+  for (int i = 0; i < 2; i++) {
+    DB* db;
+    ColumnFamilyHandle* cf1 = nullptr;
+    ColumnFamilyHandle* cf2 = nullptr;
+    ASSERT_OK(DB::Open(options, dbname_, &db));
+    ASSERT_OK(db->CreateColumnFamily(cf_options, "cf1", &cf1));
+    ASSERT_OK(db->CreateColumnFamily(cf_options, "cf2", &cf2));
+    ASSERT_OK(db->DestroyColumnFamilyHandle(cf1));
+    ASSERT_OK(db->DestroyColumnFamilyHandle(cf2));
+    delete db;
+    ASSERT_EQ(0, cache_->Size());
+
+    std::vector<ColumnFamilyHandle*> cfs;
+    if (!i) {
+      printf("Open\n");
+      ASSERT_OK(DB::Open(options, dbname_,
+                         {{"cf1", cf_options},
+                          {"cf2", cf_options},
+                          {"default", ColumnFamilyOptions()}},
+                         &cfs, &db));
+    } else {
+#ifdef ROCKSDB_LITE
+      break;
+#else
+      printf("OpenForReadOnly\n");
+      ASSERT_OK(DB::OpenForReadOnly(options, dbname_,
+                                    {{"cf1", cf_options},
+                                     {"cf2", cf_options},
+                                     {"default", ColumnFamilyOptions()}},
+                                    &cfs, &db));
+#endif
+    }
+
+    // Logical block sizes of dbname_ and cf_path_0_ are cached during Open.
+    ASSERT_EQ(2, cache_->Size());
+    ASSERT_TRUE(cache_->Contains(dbname_));
+    ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+    ASSERT_TRUE(cache_->Contains(cf_path_0_));
+    ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_));
+
+    // Drop handles won't drop the cache.
+    ASSERT_OK(db->DropColumnFamily(cfs[0]));
+    ASSERT_OK(db->DropColumnFamily(cfs[1]));
+    ASSERT_EQ(2, cache_->Size());
+    ASSERT_TRUE(cache_->Contains(dbname_));
+    ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+    ASSERT_TRUE(cache_->Contains(cf_path_0_));
+    ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_));
+
+    // Delete 1st handle won't drop the cache for cf_path_0_.
+    ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[0]));
+    ASSERT_EQ(2, cache_->Size());
+    ASSERT_TRUE(cache_->Contains(dbname_));
+    ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+    ASSERT_TRUE(cache_->Contains(cf_path_0_));
+    ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+
+    // Delete 2nd handle will drop the cache for cf_path_0_.
+    ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[1]));
+    ASSERT_EQ(1, cache_->Size());
+    ASSERT_TRUE(cache_->Contains(dbname_));
+    ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+
+    // Delete the default handle won't affect the cache because db still refers
+    // to the default CF.
+    ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[2]));
+    ASSERT_EQ(1, cache_->Size());
+    ASSERT_TRUE(cache_->Contains(dbname_));
+    ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+
+    delete db;
+    ASSERT_EQ(0, cache_->Size());
+  }
+  ASSERT_OK(DestroyDB(dbname_, options,
+      {{"cf1", cf_options}, {"cf2", cf_options}}));
+}
+
+TEST_F(DBLogicalBlockSizeCacheTest, DestroyColumnFamilyHandle) {
+  // Tests that destroy column family without dropping won't drop the cache,
+  // because compaction and flush might still need to get logical block size
+  // when opening new files.
+  Options options;
+  options.create_if_missing = true;
+  options.env = env_.get();
+  ColumnFamilyOptions cf_options;
+  cf_options.cf_paths = {{cf_path_0_, 1024}};
+
+  DB* db;
+  ASSERT_OK(DB::Open(options, dbname_, &db));
+  ASSERT_EQ(1, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(dbname_));
+  ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+  ColumnFamilyHandle* cf = nullptr;
+  ASSERT_OK(db->CreateColumnFamily(cf_options, "cf", &cf));
+  ASSERT_EQ(2, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(dbname_));
+  ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+
+  // Delete handle won't drop cache.
+  ASSERT_OK(db->DestroyColumnFamilyHandle(cf));
+  ASSERT_EQ(2, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(dbname_));
+  ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+
+  delete db;
+  ASSERT_EQ(0, cache_->Size());
+
+  // Open with column families.
+  std::vector<ColumnFamilyHandle*> cfs;
+  for (int i = 0; i < 2; i++) {
+    if (!i) {
+      printf("Open\n");
+      ASSERT_OK(DB::Open(
+          options, dbname_,
+          {{"cf", cf_options}, {"default", ColumnFamilyOptions()}}, &cfs, &db));
+    } else {
+#ifdef ROCKSDB_LITE
+      break;
+#else
+      printf("OpenForReadOnly\n");
+      ASSERT_OK(DB::OpenForReadOnly(
+          options, dbname_,
+          {{"cf", cf_options}, {"default", ColumnFamilyOptions()}}, &cfs, &db));
+#endif
+    }
+    // cf_path_0_ and dbname_ are cached.
+    ASSERT_EQ(2, cache_->Size());
+    ASSERT_TRUE(cache_->Contains(dbname_));
+    ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+    ASSERT_TRUE(cache_->Contains(cf_path_0_));
+    ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+
+    // Deleting handle won't drop cache.
+    ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[0]));
+    ASSERT_OK(db->DestroyColumnFamilyHandle(cfs[1]));
+    ASSERT_EQ(2, cache_->Size());
+    ASSERT_TRUE(cache_->Contains(dbname_));
+    ASSERT_EQ(1, cache_->GetRefCount(dbname_));
+    ASSERT_TRUE(cache_->Contains(cf_path_0_));
+    ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+
+    delete db;
+    ASSERT_EQ(0, cache_->Size());
+  }
+  ASSERT_OK(DestroyDB(dbname_, options, {{"cf", cf_options}}));
+}
+
+TEST_F(DBLogicalBlockSizeCacheTest, MultiDBWithDifferentPaths) {
+  // Tests the cache behavior when there are multiple DBs sharing the same env
+  // with different db_paths and cf_paths.
+  Options options;
+  options.create_if_missing = true;
+  options.env = env_.get();
+
+  ASSERT_OK(env_->CreateDirIfMissing(dbname_));
+
+  DB* db0;
+  ASSERT_OK(DB::Open(options, data_path_0_, &db0));
+  ASSERT_EQ(1, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(data_path_0_));
+
+  ColumnFamilyOptions cf_options0;
+  cf_options0.cf_paths = {{cf_path_0_, 1024}};
+  ColumnFamilyHandle* cf0;
+  db0->CreateColumnFamily(cf_options0, "cf", &cf0);
+  ASSERT_EQ(2, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(data_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(data_path_0_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+
+  DB* db1;
+  ASSERT_OK(DB::Open(options, data_path_1_, &db1));
+  ASSERT_EQ(3, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(data_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(data_path_0_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+  ASSERT_TRUE(cache_->Contains(data_path_1_));
+  ASSERT_EQ(1, cache_->GetRefCount(data_path_1_));
+
+  ColumnFamilyOptions cf_options1;
+  cf_options1.cf_paths = {{cf_path_1_, 1024}};
+  ColumnFamilyHandle* cf1;
+  db1->CreateColumnFamily(cf_options1, "cf", &cf1);
+  ASSERT_EQ(4, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(data_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(data_path_0_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+  ASSERT_TRUE(cache_->Contains(data_path_1_));
+  ASSERT_EQ(1, cache_->GetRefCount(data_path_1_));
+  ASSERT_TRUE(cache_->Contains(cf_path_1_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_1_));
+
+  db0->DestroyColumnFamilyHandle(cf0);
+  delete db0;
+  ASSERT_EQ(2, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(data_path_1_));
+  ASSERT_EQ(1, cache_->GetRefCount(data_path_1_));
+  ASSERT_TRUE(cache_->Contains(cf_path_1_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_1_));
+  ASSERT_OK(DestroyDB(data_path_0_, options, {{"cf", cf_options0}}));
+
+  db1->DestroyColumnFamilyHandle(cf1);
+  delete db1;
+  ASSERT_EQ(0, cache_->Size());
+  ASSERT_OK(DestroyDB(data_path_1_, options, {{"cf", cf_options1}}));
+}
+
+TEST_F(DBLogicalBlockSizeCacheTest, MultiDBWithSamePaths) {
+  // Tests the cache behavior when there are multiple DBs sharing the same env
+  // with the same db_paths and cf_paths.
+  Options options;
+  options.create_if_missing = true;
+  options.env = env_.get();
+  options.db_paths = {{data_path_0_, 1024}};
+  ColumnFamilyOptions cf_options;
+  cf_options.cf_paths = {{cf_path_0_, 1024}};
+
+  ASSERT_OK(env_->CreateDirIfMissing(dbname_));
+
+  DB* db0;
+  ASSERT_OK(DB::Open(options, dbname_ + "/db0", &db0));
+  ASSERT_EQ(1, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(data_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(data_path_0_));
+
+  ColumnFamilyHandle* cf0;
+  db0->CreateColumnFamily(cf_options, "cf", &cf0);
+  ASSERT_EQ(2, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(data_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(data_path_0_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+
+  DB* db1;
+  ASSERT_OK(DB::Open(options, dbname_ + "/db1", &db1));
+  ASSERT_EQ(2, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(data_path_0_));
+  ASSERT_EQ(2, cache_->GetRefCount(data_path_0_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+
+  ColumnFamilyHandle* cf1;
+  db1->CreateColumnFamily(cf_options, "cf", &cf1);
+  ASSERT_EQ(2, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(data_path_0_));
+  ASSERT_EQ(2, cache_->GetRefCount(data_path_0_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(2, cache_->GetRefCount(cf_path_0_));
+
+  db0->DestroyColumnFamilyHandle(cf0);
+  delete db0;
+  ASSERT_EQ(2, cache_->Size());
+  ASSERT_TRUE(cache_->Contains(data_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(data_path_0_));
+  ASSERT_TRUE(cache_->Contains(cf_path_0_));
+  ASSERT_EQ(1, cache_->GetRefCount(cf_path_0_));
+  ASSERT_OK(DestroyDB(dbname_ + "/db0", options, {{"cf", cf_options}}));
+
+  db1->DestroyColumnFamilyHandle(cf1);
+  delete db1;
+  ASSERT_EQ(0, cache_->Size());
+  ASSERT_OK(DestroyDB(dbname_ + "/db1", options, {{"cf", cf_options}}));
+}
+
+}  // namespace ROCKSDB_NAMESPACE
+#endif // OS_LINUX
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -261,7 +261,7 @@ TEST_F(DBMemTableTest, ConcurrentMergeWrite) {
  ReadOptions roptions;
  SequenceNumber max_covering_tombstone_seq = 0;
  LookupKey lkey("key", kMaxSequenceNumber);
-  res = mem->Get(lkey, &value, &status, &merge_context,
+  res = mem->Get(lkey, &value, /*timestamp=*/nullptr, &status, &merge_context,
                 &max_covering_tombstone_seq, roptions);
  ASSERT_TRUE(res);
  uint64_t ivalue = DecodeFixed64(Slice(value).data());
@@ -24,7 +24,7 @@
 #endif

 #include "cache/lru_cache.h"
-#include "db/blob_index.h"
+#include "db/blob/blob_index.h"
 #include "db/db_impl/db_impl.h"
 #include "db/db_test_util.h"
 #include "db/dbformat.h"
@@ -4297,7 +4297,7 @@ TEST_P(DBTestWithParam, PreShutdownManualCompaction) {
    ASSERT_EQ("1,1,1", FilesPerLevel(1));

    // Compaction range overlaps files
-    Compact(1, "p1", "p9");
+    Compact(1, "p", "q");
    ASSERT_EQ("0,0,1", FilesPerLevel(1));

    // Populate a different range
@@ -5370,6 +5370,8 @@ class DelayedMergeOperator : public MergeOperator {
  const char* Name() const override { return "DelayedMergeOperator"; }
 };

+// TODO: hangs in CircleCI's Windows env
+#ifndef OS_WIN
 TEST_F(DBTest, MergeTestTime) {
  std::string one, two, three;
  PutFixed64(&one, 1);
@@ -5417,6 +5419,7 @@ TEST_F(DBTest, MergeTestTime) {
 #endif  // ROCKSDB_USING_THREAD_STATUS
  this->env_->time_elapse_only_sleep_ = false;
 }
+#endif // OS_WIN

 #ifndef ROCKSDB_LITE
 TEST_P(DBTestWithParam, MergeCompactionTimeTest) {
@@ -1678,12 +1678,12 @@ TEST_P(PinL0IndexAndFilterBlocksTest, DisablePrefetchingNonL0IndexAndFilter) {
    ASSERT_EQ(fm + 3, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
    ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
    ASSERT_EQ(im + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-    ASSERT_EQ(ih + 2, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+    ASSERT_EQ(ih + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
  } else {
    ASSERT_EQ(fm + 3, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
    ASSERT_EQ(fh + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
    ASSERT_EQ(im + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-    ASSERT_EQ(ih + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+    ASSERT_EQ(ih + 4, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
  }

  // Bloom and index hit will happen when a Get() happens.
@@ -1692,12 +1692,12 @@ TEST_P(PinL0IndexAndFilterBlocksTest, DisablePrefetchingNonL0IndexAndFilter) {
    ASSERT_EQ(fm + 3, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
    ASSERT_EQ(fh + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
    ASSERT_EQ(im + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-    ASSERT_EQ(ih + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+    ASSERT_EQ(ih + 4, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
  } else {
    ASSERT_EQ(fm + 3, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
    ASSERT_EQ(fh + 2, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
    ASSERT_EQ(im + 3, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-    ASSERT_EQ(ih + 4, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+    ASSERT_EQ(ih + 5, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
  }
 }

@@ -1528,6 +1528,24 @@ TEST_F(DBWALTest, TruncateLastLogAfterRecoverWithoutFlush) {
  constexpr size_t kKB = 1024;
  Options options = CurrentOptions();
  options.avoid_flush_during_recovery = true;
+  // Test fallocate support of running file system.
+  // Skip this test if fallocate is not supported.
+  std::string fname_test_fallocate = dbname_ + "/preallocate_testfile";
+  int fd = -1;
+  do {
+    fd = open(fname_test_fallocate.c_str(), O_CREAT | O_RDWR | O_TRUNC, 0644);
+  } while (fd < 0 && errno == EINTR);
+  ASSERT_GT(fd, 0);
+  int alloc_status = fallocate(fd, 0, 0, 1);
+  int err_number = errno;
+  close(fd);
+  ASSERT_OK(options.env->DeleteFile(fname_test_fallocate));
+  if (err_number == ENOSYS || err_number == EOPNOTSUPP) {
+    fprintf(stderr, "Skipped preallocated space check: %s\n", strerror(err_number));
+    return;
+  }
+  ASSERT_EQ(0, alloc_status);
+
  DestroyAndReopen(options);
  size_t preallocated_size =
      dbfull()->TEST_GetWalPreallocateBlockSize(options.write_buffer_size);
@@ -0,0 +1,809 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/db_test_util.h"
+#include "port/stack_trace.h"
+#include "rocksdb/perf_context.h"
+#include "rocksdb/utilities/debug.h"
+#include "table/block_based/block_based_table_reader.h"
+#include "table/block_based/block_builder.h"
+#include "test_util/fault_injection_test_env.h"
+#if !defined(ROCKSDB_LITE)
+#include "test_util/sync_point.h"
+#endif
+
+namespace ROCKSDB_NAMESPACE {
+class DBBasicTestWithTimestampBase : public DBTestBase {
+ public:
+  explicit DBBasicTestWithTimestampBase(const std::string& dbname)
+      : DBTestBase(dbname) {}
+
+ protected:
+  static std::string Key1(uint64_t k) {
+    uint32_t x = 1;
+    const bool is_little_endian = (*reinterpret_cast<char*>(&x) != 0);
+    std::string ret;
+    if (is_little_endian) {
+      ret.assign(reinterpret_cast<char*>(&k), sizeof(k));
+    } else {
+      ret.resize(sizeof(k));
+      ret[0] = k & 0xff;
+      ret[1] = (k >> 8) & 0xff;
+      ret[2] = (k >> 16) & 0xff;
+      ret[3] = (k >> 24) & 0xff;
+      ret[4] = (k >> 32) & 0xff;
+      ret[5] = (k >> 40) & 0xff;
+      ret[6] = (k >> 48) & 0xff;
+      ret[7] = (k >> 56) & 0xff;
+    }
+    std::reverse(ret.begin(), ret.end());
+    return ret;
+  }
+
+  class TestComparator : public Comparator {
+   private:
+    const Comparator* cmp_without_ts_;
+
+   public:
+    explicit TestComparator(size_t ts_sz)
+        : Comparator(ts_sz), cmp_without_ts_(nullptr) {
+      cmp_without_ts_ = BytewiseComparator();
+    }
+
+    const char* Name() const override { return "TestComparator"; }
+
+    void FindShortSuccessor(std::string*) const override {}
+
+    void FindShortestSeparator(std::string*, const Slice&) const override {}
+
+    int Compare(const Slice& a, const Slice& b) const override {
+      int r = CompareWithoutTimestamp(a, b);
+      if (r != 0 || 0 == timestamp_size()) {
+        return r;
+      }
+      return -CompareTimestamp(
+          Slice(a.data() + a.size() - timestamp_size(), timestamp_size()),
+          Slice(b.data() + b.size() - timestamp_size(), timestamp_size()));
+    }
+
+    using Comparator::CompareWithoutTimestamp;
+    int CompareWithoutTimestamp(const Slice& a, bool a_has_ts, const Slice& b,
+                                bool b_has_ts) const override {
+      if (a_has_ts) {
+        assert(a.size() >= timestamp_size());
+      }
+      if (b_has_ts) {
+        assert(b.size() >= timestamp_size());
+      }
+      Slice lhs = a_has_ts ? StripTimestampFromUserKey(a, timestamp_size()) : a;
+      Slice rhs = b_has_ts ? StripTimestampFromUserKey(b, timestamp_size()) : b;
+      return cmp_without_ts_->Compare(lhs, rhs);
+    }
+
+    int CompareTimestamp(const Slice& ts1, const Slice& ts2) const override {
+      if (!ts1.data() && !ts2.data()) {
+        return 0;
+      } else if (ts1.data() && !ts2.data()) {
+        return 1;
+      } else if (!ts1.data() && ts2.data()) {
+        return -1;
+      }
+      assert(ts1.size() == ts2.size());
+      uint64_t low1 = 0;
+      uint64_t low2 = 0;
+      uint64_t high1 = 0;
+      uint64_t high2 = 0;
+      const size_t kSize = ts1.size();
+      std::unique_ptr<char[]> ts1_buf(new char[kSize]);
+      memcpy(ts1_buf.get(), ts1.data(), ts1.size());
+      std::unique_ptr<char[]> ts2_buf(new char[kSize]);
+      memcpy(ts2_buf.get(), ts2.data(), ts2.size());
+      Slice ts1_copy = Slice(ts1_buf.get(), kSize);
+      Slice ts2_copy = Slice(ts2_buf.get(), kSize);
+      auto* ptr1 = const_cast<Slice*>(&ts1_copy);
+      auto* ptr2 = const_cast<Slice*>(&ts2_copy);
+      if (!GetFixed64(ptr1, &low1) || !GetFixed64(ptr1, &high1) ||
+          !GetFixed64(ptr2, &low2) || !GetFixed64(ptr2, &high2)) {
+        assert(false);
+      }
+      if (high1 < high2) {
+        return -1;
+      } else if (high1 > high2) {
+        return 1;
+      }
+      if (low1 < low2) {
+        return -1;
+      } else if (low1 > low2) {
+        return 1;
+      }
+      return 0;
+    }
+  };
+
+  std::string Timestamp(uint64_t low, uint64_t high) {
+    std::string ts;
+    PutFixed64(&ts, low);
+    PutFixed64(&ts, high);
+    return ts;
+  }
+
+  void CheckIterUserEntry(const Iterator* it, const Slice& expected_key,
+                          const Slice& expected_value,
+                          const Slice& expected_ts) const {
+    ASSERT_TRUE(it->Valid());
+    ASSERT_OK(it->status());
+    ASSERT_EQ(expected_key, it->key());
+    ASSERT_EQ(expected_value, it->value());
+    ASSERT_EQ(expected_ts, it->timestamp());
+  }
+
+  void CheckIterEntry(const Iterator* it, const Slice& expected_ukey,
+                      SequenceNumber expected_seq, ValueType expected_val_type,
+                      const Slice& expected_value, const Slice& expected_ts) {
+    ASSERT_TRUE(it->Valid());
+    ASSERT_OK(it->status());
+    std::string ukey_and_ts;
+    ukey_and_ts.assign(expected_ukey.data(), expected_ukey.size());
+    ukey_and_ts.append(expected_ts.data(), expected_ts.size());
+    ParsedInternalKey parsed_ikey(ukey_and_ts, expected_seq, expected_val_type);
+    std::string ikey;
+    AppendInternalKey(&ikey, parsed_ikey);
+    ASSERT_EQ(Slice(ikey), it->key());
+    if (expected_val_type == kTypeValue) {
+      ASSERT_EQ(expected_value, it->value());
+    }
+    ASSERT_EQ(expected_ts, it->timestamp());
+  }
+};
+
+class DBBasicTestWithTimestamp : public DBBasicTestWithTimestampBase {
+ public:
+  DBBasicTestWithTimestamp()
+      : DBBasicTestWithTimestampBase("db_basic_test_with_timestamp") {}
+};
+
+TEST_F(DBBasicTestWithTimestamp, SimpleForwardIterate) {
+  const int kNumKeysPerFile = 128;
+  const uint64_t kMaxKey = 1024;
+  Options options = CurrentOptions();
+  options.env = env_;
+  // TODO(yanqin) re-enable auto compaction
+  options.disable_auto_compactions = true;
+  options.create_if_missing = true;
+  const size_t kTimestampSize = Timestamp(0, 0).size();
+  TestComparator test_cmp(kTimestampSize);
+  options.comparator = &test_cmp;
+  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+  DestroyAndReopen(options);
+  const std::vector<uint64_t> start_keys = {1, 0};
+  const std::vector<std::string> write_timestamps = {Timestamp(1, 0),
+                                                     Timestamp(3, 0)};
+  const std::vector<std::string> read_timestamps = {Timestamp(2, 0),
+                                                    Timestamp(4, 0)};
+  for (size_t i = 0; i < write_timestamps.size(); ++i) {
+    WriteOptions write_opts;
+    Slice write_ts = write_timestamps[i];
+    write_opts.timestamp = &write_ts;
+    for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) {
+      Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(i));
+      ASSERT_OK(s);
+    }
+  }
+  for (size_t i = 0; i < read_timestamps.size(); ++i) {
+    ReadOptions read_opts;
+    Slice read_ts = read_timestamps[i];
+    read_opts.timestamp = &read_ts;
+    std::unique_ptr<Iterator> it(db_->NewIterator(read_opts));
+    int count = 0;
+    uint64_t key = 0;
+    for (it->Seek(Key1(0)), key = start_keys[i]; it->Valid();
+         it->Next(), ++count, ++key) {
+      CheckIterUserEntry(it.get(), Key1(key), "value" + std::to_string(i),
+                         write_timestamps[i]);
+    }
+    size_t expected_count = kMaxKey - start_keys[i] + 1;
+    ASSERT_EQ(expected_count, count);
+
+    // SeekToFirst() with lower bound.
+    // Then iter with lower and upper bounds.
+    uint64_t l = 0;
+    uint64_t r = kMaxKey + 1;
+    while (l < r) {
+      std::string lb_str = Key1(l);
+      Slice lb = lb_str;
+      std::string ub_str = Key1(r);
+      Slice ub = ub_str;
+      read_opts.iterate_lower_bound = &lb;
+      read_opts.iterate_upper_bound = &ub;
+      it.reset(db_->NewIterator(read_opts));
+      for (it->SeekToFirst(), key = std::max(l, start_keys[i]), count = 0;
+           it->Valid(); it->Next(), ++key, ++count) {
+        CheckIterUserEntry(it.get(), Key1(key), "value" + std::to_string(i),
+                           write_timestamps[i]);
+      }
+      ASSERT_EQ(r - std::max(l, start_keys[i]), count);
+      l += (kMaxKey / 100);
+      r -= (kMaxKey / 100);
+    }
+  }
+  Close();
+}
+
+TEST_F(DBBasicTestWithTimestamp, ForwardIterateStartSeqnum) {
+  const int kNumKeysPerFile = 128;
+  const uint64_t kMaxKey = 0xffffffffffffffff;
+  const uint64_t kMinKey = kMaxKey - 1023;
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.create_if_missing = true;
+  // TODO(yanqin) re-enable auto compaction
+  options.disable_auto_compactions = true;
+  const size_t kTimestampSize = Timestamp(0, 0).size();
+  TestComparator test_cmp(kTimestampSize);
+  options.comparator = &test_cmp;
+  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+  DestroyAndReopen(options);
+  std::vector<SequenceNumber> start_seqs;
+
+  const int kNumTimestamps = 4;
+  std::vector<std::string> write_ts_list;
+  for (int t = 0; t != kNumTimestamps; ++t) {
+    write_ts_list.push_back(Timestamp(2 * t, /*do not care*/ 17));
+  }
+  WriteOptions write_opts;
+  for (size_t i = 0; i != write_ts_list.size(); ++i) {
+    Slice write_ts = write_ts_list[i];
+    write_opts.timestamp = &write_ts;
+    uint64_t k = kMinKey;
+    do {
+      Status s = db_->Put(write_opts, Key1(k), "value" + std::to_string(i));
+      ASSERT_OK(s);
+      if (k == kMaxKey) {
+        break;
+      }
+      ++k;
+    } while (k != 0);
+    start_seqs.push_back(db_->GetLatestSequenceNumber());
+  }
+  std::vector<std::string> read_ts_list;
+  for (int t = 0; t != kNumTimestamps - 1; ++t) {
+    read_ts_list.push_back(Timestamp(2 * t + 3, /*do not care*/ 17));
+  }
+  ReadOptions read_opts;
+  for (size_t i = 0; i != read_ts_list.size(); ++i) {
+    Slice read_ts = read_ts_list[i];
+    read_opts.timestamp = &read_ts;
+    read_opts.iter_start_seqnum = start_seqs[i];
+    std::unique_ptr<Iterator> iter(db_->NewIterator(read_opts));
+    SequenceNumber expected_seq = start_seqs[i] + 1;
+    uint64_t key = kMinKey;
+    for (iter->Seek(Key1(kMinKey)); iter->Valid(); iter->Next()) {
+      CheckIterEntry(iter.get(), Key1(key), expected_seq, kTypeValue,
+                     "value" + std::to_string(i + 1), write_ts_list[i + 1]);
+      ++key;
+      ++expected_seq;
+    }
+  }
+  Close();
+}
+
+TEST_F(DBBasicTestWithTimestamp, ReseekToTargetTimestamp) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.create_if_missing = true;
+  constexpr size_t kNumKeys = 16;
+  options.max_sequential_skip_in_iterations = kNumKeys / 2;
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+  // TODO(yanqin) re-enable auto compaction
+  options.disable_auto_compactions = true;
+  const size_t kTimestampSize = Timestamp(0, 0).size();
+  TestComparator test_cmp(kTimestampSize);
+  options.comparator = &test_cmp;
+  DestroyAndReopen(options);
+  // Insert kNumKeys
+  WriteOptions write_opts;
+  Status s;
+  for (size_t i = 0; i != kNumKeys; ++i) {
+    std::string ts_str = Timestamp(static_cast<uint64_t>(i + 1), 0);
+    Slice ts = ts_str;
+    write_opts.timestamp = &ts;
+    s = db_->Put(write_opts, "foo", "value" + std::to_string(i));
+    ASSERT_OK(s);
+  }
+  {
+    ReadOptions read_opts;
+    std::string ts_str = Timestamp(1, 0);
+    Slice ts = ts_str;
+    read_opts.timestamp = &ts;
+    std::unique_ptr<Iterator> iter(db_->NewIterator(read_opts));
+    iter->SeekToFirst();
+    CheckIterUserEntry(iter.get(), "foo", "value0", ts_str);
+    ASSERT_EQ(
+        1, options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION));
+  }
+  Close();
+}
+
+TEST_F(DBBasicTestWithTimestamp, ReseekToNextUserKey) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.create_if_missing = true;
+  constexpr size_t kNumKeys = 16;
+  options.max_sequential_skip_in_iterations = kNumKeys / 2;
+  options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
+  // TODO(yanqin) re-enable auto compaction
+  options.disable_auto_compactions = true;
+  const size_t kTimestampSize = Timestamp(0, 0).size();
+  TestComparator test_cmp(kTimestampSize);
+  options.comparator = &test_cmp;
+  DestroyAndReopen(options);
+  // Write kNumKeys + 1 keys
+  WriteOptions write_opts;
+  Status s;
+  for (size_t i = 0; i != kNumKeys; ++i) {
+    std::string ts_str = Timestamp(static_cast<uint64_t>(i + 1), 0);
+    Slice ts = ts_str;
+    write_opts.timestamp = &ts;
+    s = db_->Put(write_opts, "a", "value" + std::to_string(i));
+    ASSERT_OK(s);
+  }
+  {
+    std::string ts_str = Timestamp(static_cast<uint64_t>(kNumKeys + 1), 0);
+    WriteBatch batch(0, 0, kTimestampSize);
+    batch.Put("a", "new_value");
+    batch.Put("b", "new_value");
+    s = batch.AssignTimestamp(ts_str);
+    ASSERT_OK(s);
+    s = db_->Write(write_opts, &batch);
+    ASSERT_OK(s);
+  }
+  {
+    ReadOptions read_opts;
+    std::string ts_str = Timestamp(static_cast<uint64_t>(kNumKeys + 1), 0);
+    Slice ts = ts_str;
+    read_opts.timestamp = &ts;
+    std::unique_ptr<Iterator> iter(db_->NewIterator(read_opts));
+    iter->Seek("a");
+    iter->Next();
+    CheckIterUserEntry(iter.get(), "b", "new_value", ts_str);
+    ASSERT_EQ(
+        1, options.statistics->getTickerCount(NUMBER_OF_RESEEKS_IN_ITERATION));
+  }
+  Close();
+}
+
+TEST_F(DBBasicTestWithTimestamp, MaxKeysSkipped) {
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.create_if_missing = true;
+  const size_t kTimestampSize = Timestamp(0, 0).size();
+  TestComparator test_cmp(kTimestampSize);
+  options.comparator = &test_cmp;
+  DestroyAndReopen(options);
+  constexpr size_t max_skippable_internal_keys = 2;
+  const size_t kNumKeys = max_skippable_internal_keys + 2;
+  WriteOptions write_opts;
+  Status s;
+  {
+    std::string ts_str = Timestamp(1, 0);
+    Slice ts = ts_str;
+    write_opts.timestamp = &ts;
+    ASSERT_OK(db_->Put(write_opts, "a", "value"));
+  }
+  for (size_t i = 0; i < kNumKeys; ++i) {
+    std::string ts_str = Timestamp(static_cast<uint64_t>(i + 1), 0);
+    Slice ts = ts_str;
+    write_opts.timestamp = &ts;
+    s = db_->Put(write_opts, "b", "value" + std::to_string(i));
+    ASSERT_OK(s);
+  }
+  {
+    ReadOptions read_opts;
+    read_opts.max_skippable_internal_keys = max_skippable_internal_keys;
+    std::string ts_str = Timestamp(1, 0);
+    Slice ts = ts_str;
+    read_opts.timestamp = &ts;
+    std::unique_ptr<Iterator> iter(db_->NewIterator(read_opts));
+    iter->SeekToFirst();
+    iter->Next();
+    ASSERT_TRUE(iter->status().IsIncomplete());
+  }
+  Close();
+}
+
+class DBBasicTestWithTimestampCompressionSettings
+    : public DBBasicTestWithTimestampBase,
+      public testing::WithParamInterface<std::tuple<
+          std::shared_ptr<const FilterPolicy>, CompressionType, uint32_t>> {
+ public:
+  DBBasicTestWithTimestampCompressionSettings()
+      : DBBasicTestWithTimestampBase(
+            "db_basic_test_with_timestamp_compression") {}
+};
+
+TEST_P(DBBasicTestWithTimestampCompressionSettings, PutAndGet) {
+  const int kNumKeysPerFile = 1024;
+  const size_t kNumTimestamps = 4;
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.env = env_;
+  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+  size_t ts_sz = Timestamp(0, 0).size();
+  TestComparator test_cmp(ts_sz);
+  options.comparator = &test_cmp;
+  BlockBasedTableOptions bbto;
+  bbto.filter_policy = std::get<0>(GetParam());
+  bbto.whole_key_filtering = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+
+  const CompressionType comp_type = std::get<1>(GetParam());
+#if LZ4_VERSION_NUMBER < 10400  // r124+
+  if (comp_type == kLZ4Compression || comp_type == kLZ4HCCompression) {
+    return;
+  }
+#endif  // LZ4_VERSION_NUMBER >= 10400
+  if (!ZSTD_Supported() && comp_type == kZSTD) {
+    return;
+  }
+  if (!Zlib_Supported() && comp_type == kZlibCompression) {
+    return;
+  }
+
+  options.compression = comp_type;
+  options.compression_opts.max_dict_bytes = std::get<2>(GetParam());
+  if (comp_type == kZSTD) {
+    options.compression_opts.zstd_max_train_bytes = std::get<2>(GetParam());
+  }
+  options.target_file_size_base = 1 << 26;  // 64MB
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({"pikachu"}, options);
+  size_t num_cfs = handles_.size();
+  ASSERT_EQ(2, num_cfs);
+  std::vector<std::string> write_ts_list;
+  std::vector<std::string> read_ts_list;
+
+  for (size_t i = 0; i != kNumTimestamps; ++i) {
+    write_ts_list.push_back(Timestamp(i * 2, 0));
+    read_ts_list.push_back(Timestamp(1 + i * 2, 0));
+    const Slice write_ts = write_ts_list.back();
+    WriteOptions wopts;
+    wopts.timestamp = &write_ts;
+    for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
+      for (size_t j = 0; j != (kNumKeysPerFile - 1) / kNumTimestamps; ++j) {
+        ASSERT_OK(Put(cf, Key1(j),
+                      "value_" + std::to_string(j) + "_" + std::to_string(i),
+                      wopts));
+      }
+    }
+  }
+  const auto& verify_db_func = [&]() {
+    for (size_t i = 0; i != kNumTimestamps; ++i) {
+      ReadOptions ropts;
+      const Slice read_ts = read_ts_list[i];
+      ropts.timestamp = &read_ts;
+      for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
+        ColumnFamilyHandle* cfh = handles_[cf];
+        for (size_t j = 0; j != (kNumKeysPerFile - 1) / kNumTimestamps; ++j) {
+          std::string value;
+          ASSERT_OK(db_->Get(ropts, cfh, Key1(j), &value));
+          ASSERT_EQ("value_" + std::to_string(j) + "_" + std::to_string(i),
+                    value);
+        }
+      }
+    }
+  };
+  verify_db_func();
+  Close();
+}
+
+#ifndef ROCKSDB_LITE
+// A class which remembers the name of each flushed file.
+class FlushedFileCollector : public EventListener {
+ public:
+  FlushedFileCollector() {}
+  ~FlushedFileCollector() override {}
+
+  void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
+    InstrumentedMutexLock lock(&mutex_);
+    flushed_files_.push_back(info.file_path);
+  }
+
+  std::vector<std::string> GetFlushedFiles() {
+    std::vector<std::string> result;
+    {
+      InstrumentedMutexLock lock(&mutex_);
+      result = flushed_files_;
+    }
+    return result;
+  }
+
+  void ClearFlushedFiles() {
+    InstrumentedMutexLock lock(&mutex_);
+    flushed_files_.clear();
+  }
+
+ private:
+  std::vector<std::string> flushed_files_;
+  InstrumentedMutex mutex_;
+};
+
+TEST_P(DBBasicTestWithTimestampCompressionSettings, PutAndGetWithCompaction) {
+  const int kNumKeysPerFile = 1024;
+  const size_t kNumTimestamps = 2;
+  const size_t kNumKeysPerTimestamp = (kNumKeysPerFile - 1) / kNumTimestamps;
+  const size_t kSplitPosBase = kNumKeysPerTimestamp / 2;
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.env = env_;
+  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+
+  FlushedFileCollector* collector = new FlushedFileCollector();
+  options.listeners.emplace_back(collector);
+
+  size_t ts_sz = Timestamp(0, 0).size();
+  TestComparator test_cmp(ts_sz);
+  options.comparator = &test_cmp;
+  BlockBasedTableOptions bbto;
+  bbto.filter_policy = std::get<0>(GetParam());
+  bbto.whole_key_filtering = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+
+  const CompressionType comp_type = std::get<1>(GetParam());
+#if LZ4_VERSION_NUMBER < 10400  // r124+
+  if (comp_type == kLZ4Compression || comp_type == kLZ4HCCompression) {
+    return;
+  }
+#endif  // LZ4_VERSION_NUMBER >= 10400
+  if (!ZSTD_Supported() && comp_type == kZSTD) {
+    return;
+  }
+  if (!Zlib_Supported() && comp_type == kZlibCompression) {
+    return;
+  }
+
+  options.compression = comp_type;
+  options.compression_opts.max_dict_bytes = std::get<2>(GetParam());
+  if (comp_type == kZSTD) {
+    options.compression_opts.zstd_max_train_bytes = std::get<2>(GetParam());
+  }
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  size_t num_cfs = handles_.size();
+  ASSERT_EQ(2, num_cfs);
+  std::vector<std::string> write_ts_list;
+  std::vector<std::string> read_ts_list;
+
+  const auto& verify_record_func = [&](size_t i, size_t k,
+                                       ColumnFamilyHandle* cfh) {
+    std::string value;
+    std::string timestamp;
+
+    ReadOptions ropts;
+    const Slice read_ts = read_ts_list[i];
+    ropts.timestamp = &read_ts;
+    std::string expected_timestamp =
+        std::string(write_ts_list[i].data(), write_ts_list[i].size());
+
+    ASSERT_OK(db_->Get(ropts, cfh, Key1(k), &value, &timestamp));
+    ASSERT_EQ("value_" + std::to_string(k) + "_" + std::to_string(i), value);
+    ASSERT_EQ(expected_timestamp, timestamp);
+  };
+
+  for (size_t i = 0; i != kNumTimestamps; ++i) {
+    write_ts_list.push_back(Timestamp(i * 2, 0));
+    read_ts_list.push_back(Timestamp(1 + i * 2, 0));
+    const Slice write_ts = write_ts_list.back();
+    WriteOptions wopts;
+    wopts.timestamp = &write_ts;
+    for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
+      size_t memtable_get_start = 0;
+      for (size_t j = 0; j != kNumKeysPerTimestamp; ++j) {
+        ASSERT_OK(Put(cf, Key1(j),
+                      "value_" + std::to_string(j) + "_" + std::to_string(i),
+                      wopts));
+        if (j == kSplitPosBase + i || j == kNumKeysPerTimestamp - 1) {
+          for (size_t k = memtable_get_start; k <= j; ++k) {
+            verify_record_func(i, k, handles_[cf]);
+          }
+          memtable_get_start = j + 1;
+
+          // flush all keys with the same timestamp to two sst files, split at
+          // incremental positions such that lowerlevel[1].smallest.userkey ==
+          // higherlevel[0].largest.userkey
+          ASSERT_OK(Flush(cf));
+
+          // compact files (2 at each level) to a lower level such that all keys
+          // with the same timestamp is at one level, with newer versions at
+          // higher levels.
+          CompactionOptions compact_opt;
+          compact_opt.compression = kNoCompression;
+          db_->CompactFiles(compact_opt, handles_[cf],
+                            collector->GetFlushedFiles(),
+                            static_cast<int>(kNumTimestamps - i));
+          collector->ClearFlushedFiles();
+        }
+      }
+    }
+  }
+  const auto& verify_db_func = [&]() {
+    for (size_t i = 0; i != kNumTimestamps; ++i) {
+      ReadOptions ropts;
+      const Slice read_ts = read_ts_list[i];
+      ropts.timestamp = &read_ts;
+      std::string expected_timestamp(write_ts_list[i].data(),
+                                     write_ts_list[i].size());
+      for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
+        ColumnFamilyHandle* cfh = handles_[cf];
+        for (size_t j = 0; j != kNumKeysPerTimestamp; ++j) {
+          verify_record_func(i, j, cfh);
+        }
+      }
+    }
+  };
+  verify_db_func();
+  Close();
+}
+#endif  // !ROCKSDB_LITE
+
+INSTANTIATE_TEST_CASE_P(
+    Timestamp, DBBasicTestWithTimestampCompressionSettings,
+    ::testing::Combine(
+        ::testing::Values(std::shared_ptr<const FilterPolicy>(nullptr),
+                          std::shared_ptr<const FilterPolicy>(
+                              NewBloomFilterPolicy(10, false))),
+        ::testing::Values(kNoCompression, kZlibCompression, kLZ4Compression,
+                          kLZ4HCCompression, kZSTD),
+        ::testing::Values(0, 1 << 14)));
+
+class DBBasicTestWithTimestampPrefixSeek
+    : public DBBasicTestWithTimestampBase,
+      public testing::WithParamInterface<
+          std::tuple<std::shared_ptr<const SliceTransform>,
+                     std::shared_ptr<const FilterPolicy>, bool>> {
+ public:
+  DBBasicTestWithTimestampPrefixSeek()
+      : DBBasicTestWithTimestampBase(
+            "/db_basic_test_with_timestamp_prefix_seek") {}
+};
+
+TEST_P(DBBasicTestWithTimestampPrefixSeek, ForwardIterateWithPrefix) {
+  const size_t kNumKeysPerFile = 128;
+  Options options = CurrentOptions();
+  options.env = env_;
+  options.create_if_missing = true;
+  // TODO(yanqin): re-enable auto compactions
+  options.disable_auto_compactions = true;
+  const size_t kTimestampSize = Timestamp(0, 0).size();
+  TestComparator test_cmp(kTimestampSize);
+  options.comparator = &test_cmp;
+  options.prefix_extractor = std::get<0>(GetParam());
+  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+  BlockBasedTableOptions bbto;
+  bbto.filter_policy = std::get<1>(GetParam());
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  DestroyAndReopen(options);
+
+  const uint64_t kMaxKey = 0xffffffffffffffff;
+  const uint64_t kMinKey = 0xfffffffffffff000;
+  const std::vector<std::string> write_ts_list = {Timestamp(3, 0xffffffff),
+                                                  Timestamp(6, 0xffffffff)};
+  WriteOptions write_opts;
+  {
+    for (size_t i = 0; i != write_ts_list.size(); ++i) {
+      Slice write_ts = write_ts_list[i];
+      write_opts.timestamp = &write_ts;
+      uint64_t key = kMinKey;
+      do {
+        Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(i));
+        ASSERT_OK(s);
+        if (key == kMaxKey) {
+          break;
+        }
+        ++key;
+      } while (true);
+    }
+  }
+  const std::vector<std::string> read_ts_list = {Timestamp(5, 0xffffffff),
+                                                 Timestamp(9, 0xffffffff)};
+  {
+    ReadOptions read_opts;
+    read_opts.total_order_seek = false;
+    read_opts.prefix_same_as_start = std::get<2>(GetParam());
+    fprintf(stdout, "%s %s %d\n", options.prefix_extractor->Name(),
+            bbto.filter_policy ? bbto.filter_policy->Name() : "null",
+            static_cast<int>(read_opts.prefix_same_as_start));
+    for (size_t i = 0; i != read_ts_list.size(); ++i) {
+      Slice read_ts = read_ts_list[i];
+      read_opts.timestamp = &read_ts;
+      std::unique_ptr<Iterator> iter(db_->NewIterator(read_opts));
+
+      // Seek to kMaxKey
+      iter->Seek(Key1(kMaxKey));
+      CheckIterUserEntry(iter.get(), Key1(kMaxKey), "value" + std::to_string(i),
+                         write_ts_list[i]);
+      iter->Next();
+      ASSERT_FALSE(iter->Valid());
+    }
+    const std::vector<uint64_t> targets = {kMinKey, kMinKey + 0x10,
+                                           kMinKey + 0x100, kMaxKey};
+    const SliceTransform* const pe = options.prefix_extractor.get();
+    ASSERT_NE(nullptr, pe);
+    const size_t kPrefixShift =
+        8 * (Key1(0).size() - pe->Transform(Key1(0)).size());
+    const uint64_t kPrefixMask =
+        ~((static_cast<uint64_t>(1) << kPrefixShift) - 1);
+    const uint64_t kNumKeysWithinPrefix =
+        (static_cast<uint64_t>(1) << kPrefixShift);
+    for (size_t i = 0; i != read_ts_list.size(); ++i) {
+      Slice read_ts = read_ts_list[i];
+      read_opts.timestamp = &read_ts;
+      std::unique_ptr<Iterator> it(db_->NewIterator(read_opts));
+      for (size_t j = 0; j != targets.size(); ++j) {
+        std::string start_key = Key1(targets[j]);
+        uint64_t expected_ub =
+            (targets[j] & kPrefixMask) - 1 + kNumKeysWithinPrefix;
+        uint64_t expected_key = targets[j];
+        size_t count = 0;
+        it->Seek(Key1(targets[j]));
+        while (it->Valid()) {
+          std::string saved_prev_key;
+          saved_prev_key.assign(it->key().data(), it->key().size());
+
+          // Out of prefix
+          if (!read_opts.prefix_same_as_start &&
+              pe->Transform(saved_prev_key) != pe->Transform(start_key)) {
+            break;
+          }
+          CheckIterUserEntry(it.get(), Key1(expected_key),
+                             "value" + std::to_string(i), write_ts_list[i]);
+          ++count;
+          ++expected_key;
+          it->Next();
+        }
+        ASSERT_EQ(expected_ub - targets[j] + 1, count);
+      }
+    }
+  }
+  Close();
+}
+
+// TODO(yanqin): consider handling non-fixed-length prefix extractors, e.g.
+// NoopTransform.
+INSTANTIATE_TEST_CASE_P(
+    Timestamp, DBBasicTestWithTimestampPrefixSeek,
+    ::testing::Combine(
+        ::testing::Values(
+            std::shared_ptr<const SliceTransform>(NewFixedPrefixTransform(4)),
+            std::shared_ptr<const SliceTransform>(NewFixedPrefixTransform(7)),
+            std::shared_ptr<const SliceTransform>(NewFixedPrefixTransform(8))),
+        ::testing::Values(std::shared_ptr<const FilterPolicy>(nullptr),
+                          std::shared_ptr<const FilterPolicy>(
+                              NewBloomFilterPolicy(10 /*bits_per_key*/, false)),
+                          std::shared_ptr<const FilterPolicy>(
+                              NewBloomFilterPolicy(20 /*bits_per_key*/,
+                                                   false))),
+        ::testing::Bool()));
+
+}  // namespace ROCKSDB_NAMESPACE
+
+#ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
+extern "C" {
+void RegisterCustomObjects(int argc, char** argv);
+}
+#else
+void RegisterCustomObjects(int /*argc*/, char** /*argv*/) {}
+#endif  // !ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS
+
+int main(int argc, char** argv) {
+  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
+  ::testing::InitGoogleTest(&argc, argv);
+  RegisterCustomObjects(argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -75,6 +75,15 @@ void AppendInternalKey(std::string* result, const ParsedInternalKey& key) {
  PutFixed64(result, PackSequenceAndType(key.sequence, key.type));
 }

+void AppendInternalKeyWithDifferentTimestamp(std::string* result,
+                                             const ParsedInternalKey& key,
+                                             const Slice& ts) {
+  assert(key.user_key.size() >= ts.size());
+  result->append(key.user_key.data(), key.user_key.size() - ts.size());
+  result->append(ts.data(), ts.size());
+  PutFixed64(result, PackSequenceAndType(key.sequence, key.type));
+}
+
 void AppendInternalKeyFooter(std::string* result, SequenceNumber s,
                             ValueType t) {
  PutFixed64(result, PackSequenceAndType(s, t));
@@ -104,6 +104,7 @@ struct ParsedInternalKey {
  ParsedInternalKey()
      : sequence(kMaxSequenceNumber)  // Make code analyzer happy
  {}  // Intentionally left uninitialized (for speed)
+  // u contains timestamp if user timestamp feature is enabled.
  ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
      : user_key(u), sequence(seq), type(t) {}
  std::string DebugString(bool hex = false) const;
@@ -132,6 +133,12 @@ EntryType GetEntryType(ValueType value_type);
 // Append the serialization of "key" to *result.
 extern void AppendInternalKey(std::string* result,
                              const ParsedInternalKey& key);
+
+// Append the serialization of "key" to *result, replacing the original
+// timestamp with argument ts.
+extern void AppendInternalKeyWithDifferentTimestamp(
+    std::string* result, const ParsedInternalKey& key, const Slice& ts);
+
 // Serialized internal key consists of user key followed by footer.
 // This function appends the footer to *result, assuming that *result already
 // contains the user key at the end.
@@ -162,6 +169,11 @@ inline Slice StripTimestampFromUserKey(const Slice& user_key, size_t ts_sz) {
  return Slice(user_key.data(), user_key.size() - ts_sz);
 }

+inline Slice ExtractTimestampFromUserKey(const Slice& user_key, size_t ts_sz) {
+  assert(user_key.size() >= ts_sz);
+  return Slice(user_key.data() + user_key.size() - ts_sz, ts_sz);
+}
+
 inline uint64_t ExtractInternalKeyFooter(const Slice& internal_key) {
  assert(internal_key.size() >= 8);
  const size_t n = internal_key.size();
@@ -187,7 +199,8 @@ class InternalKeyComparator

 public:
  explicit InternalKeyComparator(const Comparator* c)
-      : user_comparator_(c),
+      : Comparator(c->timestamp_size()),
+        user_comparator_(c),
        name_("rocksdb.InternalKeyComparator:" +
              std::string(user_comparator_.Name())) {}
  virtual ~InternalKeyComparator() {}
@@ -434,24 +447,31 @@ class IterKey {

  void SetInternalKey(const Slice& key_prefix, const Slice& user_key,
                      SequenceNumber s,
-                      ValueType value_type = kValueTypeForSeek) {
+                      ValueType value_type = kValueTypeForSeek,
+                      const Slice* ts = nullptr) {
    size_t psize = key_prefix.size();
    size_t usize = user_key.size();
-    EnlargeBufferIfNeeded(psize + usize + sizeof(uint64_t));
+    size_t ts_sz = (ts != nullptr ? ts->size() : 0);
+    EnlargeBufferIfNeeded(psize + usize + sizeof(uint64_t) + ts_sz);
    if (psize > 0) {
      memcpy(buf_, key_prefix.data(), psize);
    }
    memcpy(buf_ + psize, user_key.data(), usize);
-    EncodeFixed64(buf_ + usize + psize, PackSequenceAndType(s, value_type));
+    if (ts) {
+      memcpy(buf_ + psize + usize, ts->data(), ts_sz);
+    }
+    EncodeFixed64(buf_ + usize + psize + ts_sz,
+                  PackSequenceAndType(s, value_type));

    key_ = buf_;
-    key_size_ = psize + usize + sizeof(uint64_t);
+    key_size_ = psize + usize + sizeof(uint64_t) + ts_sz;
    is_user_key_ = false;
  }

  void SetInternalKey(const Slice& user_key, SequenceNumber s,
-                      ValueType value_type = kValueTypeForSeek) {
-    SetInternalKey(Slice(), user_key, s, value_type);
+                      ValueType value_type = kValueTypeForSeek,
+                      const Slice* ts = nullptr) {
+    SetInternalKey(Slice(), user_key, s, value_type, ts);
  }

  void Reserve(size_t size) {
@@ -10,18 +10,20 @@

 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
+#include "rocksdb/io_status.h"
 #include "rocksdb/perf_context.h"
 #include "rocksdb/sst_file_manager.h"
 #include "test_util/fault_injection_test_env.h"
+#include "test_util/fault_injection_test_fs.h"
 #if !defined(ROCKSDB_LITE)
 #include "test_util/sync_point.h"
 #endif

 namespace ROCKSDB_NAMESPACE {

-class DBErrorHandlingTest : public DBTestBase {
+class DBErrorHandlingFSTest : public DBTestBase {
 public:
-  DBErrorHandlingTest() : DBTestBase("/db_error_handling_test") {}
+  DBErrorHandlingFSTest() : DBTestBase("/db_error_handling_fs_test") {}

  std::string GetManifestNameFromLiveFiles() {
    std::vector<std::string> live_files;
@@ -39,21 +41,24 @@ class DBErrorHandlingTest : public DBTestBase {
  }
 };

-class DBErrorHandlingEnv : public EnvWrapper {
-  public:
-    DBErrorHandlingEnv() : EnvWrapper(Env::Default()),
-      trig_no_space(false), trig_io_error(false) {}
+class DBErrorHandlingFS : public FileSystemWrapper {
+ public:
+  DBErrorHandlingFS()
+      : FileSystemWrapper(FileSystem::Default().get()),
+        trig_no_space(false),
+        trig_io_error(false) {}

-    void SetTrigNoSpace() {trig_no_space = true;}
-    void SetTrigIoError() {trig_io_error = true;}
-  private:
-    bool trig_no_space;
-    bool trig_io_error;
+  void SetTrigNoSpace() { trig_no_space = true; }
+  void SetTrigIoError() { trig_io_error = true; }
+
+ private:
+  bool trig_no_space;
+  bool trig_io_error;
 };

-class ErrorHandlerListener : public EventListener {
+class ErrorHandlerFSListener : public EventListener {
 public:
-  ErrorHandlerListener()
+  ErrorHandlerFSListener()
      : mutex_(),
        cv_(&mutex_),
        no_auto_recovery_(false),
@@ -61,7 +66,7 @@ class ErrorHandlerListener : public EventListener {
        file_creation_started_(false),
        override_bg_error_(false),
        file_count_(0),
-        fault_env_(nullptr) {}
+        fault_fs_(nullptr) {}

  void OnTableFileCreationStarted(
      const TableFileCreationBriefInfo& /*ti*/) override {
@@ -69,16 +74,15 @@ class ErrorHandlerListener : public EventListener {
    file_creation_started_ = true;
    if (file_count_ > 0) {
      if (--file_count_ == 0) {
-        fault_env_->SetFilesystemActive(false, file_creation_error_);
-        file_creation_error_ = Status::OK();
+        fault_fs_->SetFilesystemActive(false, file_creation_error_);
+        file_creation_error_ = IOStatus::OK();
      }
    }
    cv_.SignalAll();
  }

  void OnErrorRecoveryBegin(BackgroundErrorReason /*reason*/,
-                            Status /*bg_error*/,
-                            bool* auto_recovery) override {
+                            Status /*bg_error*/, bool* auto_recovery) override {
    if (*auto_recovery && no_auto_recovery_) {
      *auto_recovery = false;
    }
@@ -125,11 +129,11 @@ class ErrorHandlerListener : public EventListener {
    override_bg_error_ = true;
  }

-  void InjectFileCreationError(FaultInjectionTestEnv* env, int file_count,
-                               Status s) {
-    fault_env_ = env;
+  void InjectFileCreationError(FaultInjectionTestFS* fs, int file_count,
+                               IOStatus io_s) {
+    fault_fs_ = fs;
    file_count_ = file_count;
-    file_creation_error_ = s;
+    file_creation_error_ = io_s;
  }

 private:
@@ -140,18 +144,19 @@ class ErrorHandlerListener : public EventListener {
  bool file_creation_started_;
  bool override_bg_error_;
  int file_count_;
-  Status file_creation_error_;
+  IOStatus file_creation_error_;
  Status bg_error_;
-  FaultInjectionTestEnv* fault_env_;
+  FaultInjectionTestFS* fault_fs_;
 };

-TEST_F(DBErrorHandlingTest, FLushWriteError) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_env(
-      new FaultInjectionTestEnv(Env::Default()));
-  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+TEST_F(DBErrorHandlingFSTest, FLushWriteError) {
+  FaultInjectionTestFS* fault_fs =
+      new FaultInjectionTestFS(FileSystem::Default().get());
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
  Options options = GetDefaultOptions();
+  options.file_system.reset(fault_fs);
  options.create_if_missing = true;
-  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;

@@ -159,15 +164,14 @@ TEST_F(DBErrorHandlingTest, FLushWriteError) {
  DestroyAndReopen(options);

  Put(Key(0), "val");
-  SyncPoint::GetInstance()->SetCallBack(
-      "FlushJob::Start", [&](void *) {
-    fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
+  SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) {
+    fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space"));
  });
  SyncPoint::GetInstance()->EnableProcessing();
  s = Flush();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
  SyncPoint::GetInstance()->DisableProcessing();
-  fault_env->SetFilesystemActive(true);
+  fault_fs->SetFilesystemActive(true);
  s = dbfull()->Resume();
  ASSERT_EQ(s, Status::OK());

@@ -176,13 +180,14 @@ TEST_F(DBErrorHandlingTest, FLushWriteError) {
  Destroy(options);
 }

-TEST_F(DBErrorHandlingTest, ManifestWriteError) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_env(
-      new FaultInjectionTestEnv(Env::Default()));
-  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+TEST_F(DBErrorHandlingFSTest, ManifestWriteError) {
+  FaultInjectionTestFS* fault_fs =
+      new FaultInjectionTestFS(FileSystem::Default().get());
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
  Options options = GetDefaultOptions();
+  options.file_system.reset(fault_fs);
  options.create_if_missing = true;
-  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;
  std::string old_manifest;
@@ -196,15 +201,15 @@ TEST_F(DBErrorHandlingTest, ManifestWriteError) {
  Flush();
  Put(Key(1), "val");
  SyncPoint::GetInstance()->SetCallBack(
-      "VersionSet::LogAndApply:WriteManifest", [&](void *) {
-    fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
-  });
+      "VersionSet::LogAndApply:WriteManifest", [&](void*) {
+        fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space"));
+      });
  SyncPoint::GetInstance()->EnableProcessing();
  s = Flush();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
  SyncPoint::GetInstance()->ClearAllCallBacks();
  SyncPoint::GetInstance()->DisableProcessing();
-  fault_env->SetFilesystemActive(true);
+  fault_fs->SetFilesystemActive(true);
  s = dbfull()->Resume();
  ASSERT_EQ(s, Status::OK());

@@ -217,13 +222,14 @@ TEST_F(DBErrorHandlingTest, ManifestWriteError) {
  Close();
 }

-TEST_F(DBErrorHandlingTest, DoubleManifestWriteError) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_env(
-      new FaultInjectionTestEnv(Env::Default()));
-  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+TEST_F(DBErrorHandlingFSTest, DoubleManifestWriteError) {
+  FaultInjectionTestFS* fault_fs =
+      new FaultInjectionTestFS(FileSystem::Default().get());
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
  Options options = GetDefaultOptions();
+  options.file_system.reset(fault_fs);
  options.create_if_missing = true;
-  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;
  std::string old_manifest;
@@ -237,18 +243,18 @@ TEST_F(DBErrorHandlingTest, DoubleManifestWriteError) {
  Flush();
  Put(Key(1), "val");
  SyncPoint::GetInstance()->SetCallBack(
-      "VersionSet::LogAndApply:WriteManifest", [&](void *) {
-    fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
-  });
+      "VersionSet::LogAndApply:WriteManifest", [&](void*) {
+        fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space"));
+      });
  SyncPoint::GetInstance()->EnableProcessing();
  s = Flush();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
-  fault_env->SetFilesystemActive(true);
+  fault_fs->SetFilesystemActive(true);

  // This Resume() will attempt to create a new manifest file and fail again
  s = dbfull()->Resume();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
-  fault_env->SetFilesystemActive(true);
+  fault_fs->SetFilesystemActive(true);
  SyncPoint::GetInstance()->ClearAllCallBacks();
  SyncPoint::GetInstance()->DisableProcessing();

@@ -265,15 +271,16 @@ TEST_F(DBErrorHandlingTest, DoubleManifestWriteError) {
  Close();
 }

-TEST_F(DBErrorHandlingTest, CompactionManifestWriteError) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_env(
-      new FaultInjectionTestEnv(Env::Default()));
-  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+TEST_F(DBErrorHandlingFSTest, CompactionManifestWriteError) {
+  FaultInjectionTestFS* fault_fs =
+      new FaultInjectionTestFS(FileSystem::Default().get());
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
  Options options = GetDefaultOptions();
+  options.file_system.reset(fault_fs);
  options.create_if_missing = true;
  options.level0_file_num_compaction_trigger = 2;
  options.listeners.emplace_back(listener);
-  options.env = fault_env.get();
  Status s;
  std::string old_manifest;
  std::string new_manifest;
@@ -304,8 +311,8 @@ TEST_F(DBErrorHandlingTest, CompactionManifestWriteError) {
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "VersionSet::LogAndApply:WriteManifest", [&](void*) {
        if (fail_manifest.load()) {
-          fault_env->SetFilesystemActive(false,
-                                         Status::NoSpace("Out of space"));
+          fault_fs->SetFilesystemActive(false,
+                                        IOStatus::NoSpace("Out of space"));
        }
      });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
@@ -318,7 +325,7 @@ TEST_F(DBErrorHandlingTest, CompactionManifestWriteError) {

  TEST_SYNC_POINT("CompactionManifestWriteError:0");
  // Clear all errors so when the compaction is retried, it will succeed
-  fault_env->SetFilesystemActive(true);
+  fault_fs->SetFilesystemActive(true);
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
  TEST_SYNC_POINT("CompactionManifestWriteError:1");
  TEST_SYNC_POINT("CompactionManifestWriteError:2");
@@ -336,15 +343,16 @@ TEST_F(DBErrorHandlingTest, CompactionManifestWriteError) {
  Close();
 }

-TEST_F(DBErrorHandlingTest, CompactionWriteError) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_env(
-      new FaultInjectionTestEnv(Env::Default()));
-  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+TEST_F(DBErrorHandlingFSTest, CompactionWriteError) {
+  FaultInjectionTestFS* fault_fs =
+      new FaultInjectionTestFS(FileSystem::Default().get());
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
  Options options = GetDefaultOptions();
+  options.file_system.reset(fault_fs);
  options.create_if_missing = true;
  options.level0_file_num_compaction_trigger = 2;
  options.listeners.emplace_back(listener);
-  options.env = fault_env.get();
  Status s;
  DestroyAndReopen(options);

@@ -354,15 +362,14 @@ TEST_F(DBErrorHandlingTest, CompactionWriteError) {
  ASSERT_EQ(s, Status::OK());

  listener->OverrideBGError(
-      Status(Status::NoSpace(), Status::Severity::kHardError)
-      );
+      Status(Status::NoSpace(), Status::Severity::kHardError));
  listener->EnableAutoRecovery(false);
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
      {{"DBImpl::FlushMemTable:FlushMemTableFinished",
        "BackgroundCallCompaction:0"}});
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "BackgroundCallCompaction:0", [&](void*) {
-        fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
+        fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space"));
      });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();

@@ -373,19 +380,19 @@ TEST_F(DBErrorHandlingTest, CompactionWriteError) {
  s = dbfull()->TEST_WaitForCompact();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);

-  fault_env->SetFilesystemActive(true);
+  fault_fs->SetFilesystemActive(true);
  s = dbfull()->Resume();
  ASSERT_EQ(s, Status::OK());
  Destroy(options);
 }

-TEST_F(DBErrorHandlingTest, CorruptionError) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_env(
-      new FaultInjectionTestEnv(Env::Default()));
+TEST_F(DBErrorHandlingFSTest, CorruptionError) {
+  FaultInjectionTestFS* fault_fs =
+      new FaultInjectionTestFS(FileSystem::Default().get());
  Options options = GetDefaultOptions();
+  options.file_system.reset(fault_fs);
  options.create_if_missing = true;
  options.level0_file_num_compaction_trigger = 2;
-  options.env = fault_env.get();
  Status s;
  DestroyAndReopen(options);

@@ -399,7 +406,8 @@ TEST_F(DBErrorHandlingTest, CorruptionError) {
        "BackgroundCallCompaction:0"}});
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
      "BackgroundCallCompaction:0", [&](void*) {
-        fault_env->SetFilesystemActive(false, Status::Corruption("Corruption"));
+        fault_fs->SetFilesystemActive(false,
+                                      IOStatus::Corruption("Corruption"));
      });
  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();

@@ -411,19 +419,20 @@ TEST_F(DBErrorHandlingTest, CorruptionError) {
  ASSERT_EQ(s.severity(),
            ROCKSDB_NAMESPACE::Status::Severity::kUnrecoverableError);

-  fault_env->SetFilesystemActive(true);
+  fault_fs->SetFilesystemActive(true);
  s = dbfull()->Resume();
  ASSERT_NE(s, Status::OK());
  Destroy(options);
 }

-TEST_F(DBErrorHandlingTest, AutoRecoverFlushError) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_env(
-      new FaultInjectionTestEnv(Env::Default()));
-  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+TEST_F(DBErrorHandlingFSTest, AutoRecoverFlushError) {
+  FaultInjectionTestFS* fault_fs =
+      new FaultInjectionTestFS(FileSystem::Default().get());
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
  Options options = GetDefaultOptions();
+  options.file_system.reset(fault_fs);
  options.create_if_missing = true;
-  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;

@@ -432,13 +441,13 @@ TEST_F(DBErrorHandlingTest, AutoRecoverFlushError) {

  Put(Key(0), "val");
  SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) {
-    fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
+    fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space"));
  });
  SyncPoint::GetInstance()->EnableProcessing();
  s = Flush();
  ASSERT_EQ(s.severity(), ROCKSDB_NAMESPACE::Status::Severity::kHardError);
  SyncPoint::GetInstance()->DisableProcessing();
-  fault_env->SetFilesystemActive(true);
+  fault_fs->SetFilesystemActive(true);
  ASSERT_EQ(listener->WaitForRecovery(5000000), true);

  s = Put(Key(1), "val");
@@ -450,13 +459,14 @@ TEST_F(DBErrorHandlingTest, AutoRecoverFlushError) {
  Destroy(options);
 }

-TEST_F(DBErrorHandlingTest, FailRecoverFlushError) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_env(
-      new FaultInjectionTestEnv(Env::Default()));
-  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+TEST_F(DBErrorHandlingFSTest, FailRecoverFlushError) {
+  FaultInjectionTestFS* fault_fs =
+      new FaultInjectionTestFS(FileSystem::Default().get());
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
  Options options = GetDefaultOptions();
+  options.file_system.reset(fault_fs);
  options.create_if_missing = true;
-  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;

@@ -465,7 +475,7 @@ TEST_F(DBErrorHandlingTest, FailRecoverFlushError) {

  Put(Key(0), "val");
  SyncPoint::GetInstance()->SetCallBack("FlushJob::Start", [&](void*) {
-    fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
+    fault_fs->SetFilesystemActive(false, IOStatus::NoSpace("Out of space"));
  });
  SyncPoint::GetInstance()->EnableProcessing();
  s = Flush();
@@ -476,14 +486,15 @@ TEST_F(DBErrorHandlingTest, FailRecoverFlushError) {
  DestroyDB(dbname_, options);
 }

-TEST_F(DBErrorHandlingTest, WALWriteError) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_env(
-      new FaultInjectionTestEnv(Env::Default()));
-  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+TEST_F(DBErrorHandlingFSTest, WALWriteError) {
+  FaultInjectionTestFS* fault_fs =
+      new FaultInjectionTestFS(FileSystem::Default().get());
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
  Options options = GetDefaultOptions();
+  options.file_system.reset(fault_fs);
  options.create_if_missing = true;
  options.writable_file_max_buffer_size = 32768;
-  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;
  Random rnd(301);
@@ -494,7 +505,7 @@ TEST_F(DBErrorHandlingTest, WALWriteError) {
  {
    WriteBatch batch;

-    for (auto i = 0; i<100; ++i) {
+    for (auto i = 0; i < 100; ++i) {
      batch.Put(Key(i), RandomString(&rnd, 1024));
    }

@@ -507,16 +518,18 @@ TEST_F(DBErrorHandlingTest, WALWriteError) {
    WriteBatch batch;
    int write_error = 0;

-    for (auto i = 100; i<199; ++i) {
+    for (auto i = 100; i < 199; ++i) {
      batch.Put(Key(i), RandomString(&rnd, 1024));
    }

-    SyncPoint::GetInstance()->SetCallBack("WritableFileWriter::Append:BeforePrepareWrite", [&](void*) {
-      write_error++;
-      if (write_error > 2) {
-        fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
-      }
-    });
+    SyncPoint::GetInstance()->SetCallBack(
+        "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) {
+          write_error++;
+          if (write_error > 2) {
+            fault_fs->SetFilesystemActive(false,
+                                          IOStatus::NoSpace("Out of space"));
+          }
+        });
    SyncPoint::GetInstance()->EnableProcessing();
    WriteOptions wopts;
    wopts.sync = true;
@@ -524,9 +537,9 @@ TEST_F(DBErrorHandlingTest, WALWriteError) {
    ASSERT_EQ(s, s.NoSpace());
  }
  SyncPoint::GetInstance()->DisableProcessing();
-  fault_env->SetFilesystemActive(true);
+  fault_fs->SetFilesystemActive(true);
  ASSERT_EQ(listener->WaitForRecovery(5000000), true);
-  for (auto i=0; i<199; ++i) {
+  for (auto i = 0; i < 199; ++i) {
    if (i < 100) {
      ASSERT_NE(Get(Key(i)), "NOT_FOUND");
    } else {
@@ -534,7 +547,7 @@ TEST_F(DBErrorHandlingTest, WALWriteError) {
    }
  }
  Reopen(options);
-  for (auto i=0; i<199; ++i) {
+  for (auto i = 0; i < 199; ++i) {
    if (i < 100) {
      ASSERT_NE(Get(Key(i)), "NOT_FOUND");
    } else {
@@ -544,14 +557,15 @@ TEST_F(DBErrorHandlingTest, WALWriteError) {
  Close();
 }

-TEST_F(DBErrorHandlingTest, MultiCFWALWriteError) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_env(
-      new FaultInjectionTestEnv(Env::Default()));
-  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+TEST_F(DBErrorHandlingFSTest, MultiCFWALWriteError) {
+  FaultInjectionTestFS* fault_fs =
+      new FaultInjectionTestFS(FileSystem::Default().get());
+  std::shared_ptr<ErrorHandlerFSListener> listener(
+      new ErrorHandlerFSListener());
  Options options = GetDefaultOptions();
+  options.file_system.reset(fault_fs);
  options.create_if_missing = true;
  options.writable_file_max_buffer_size = 32768;
-  options.env = fault_env.get();
  options.listeners.emplace_back(listener);
  Status s;
  Random rnd(301);
@@ -586,8 +600,8 @@ TEST_F(DBErrorHandlingTest, MultiCFWALWriteError) {
        "WritableFileWriter::Append:BeforePrepareWrite", [&](void*) {
          write_error++;
          if (write_error > 2) {
-            fault_env->SetFilesystemActive(false,
-                                           Status::NoSpace("Out of space"));
+            fault_fs->SetFilesystemActive(false,
+                                          IOStatus::NoSpace("Out of space"));
          }
        });
    SyncPoint::GetInstance()->EnableProcessing();
@@ -597,7 +611,7 @@ TEST_F(DBErrorHandlingTest, MultiCFWALWriteError) {
    ASSERT_EQ(s, s.NoSpace());
  }
  SyncPoint::GetInstance()->DisableProcessing();
-  fault_env->SetFilesystemActive(true);
+  fault_fs->SetFilesystemActive(true);
  ASSERT_EQ(listener->WaitForRecovery(5000000), true);

  for (auto i = 1; i < 4; ++i) {
@@ -627,24 +641,25 @@ TEST_F(DBErrorHandlingTest, MultiCFWALWriteError) {
  Close();
 }

-TEST_F(DBErrorHandlingTest, MultiDBCompactionError) {
+TEST_F(DBErrorHandlingFSTest, MultiDBCompactionError) {
  FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(Env::Default());
-  std::vector<std::unique_ptr<FaultInjectionTestEnv>> fault_env;
+  std::vector<FaultInjectionTestFS*> fault_fs;
  std::vector<Options> options;
-  std::vector<std::shared_ptr<ErrorHandlerListener>> listener;
+  std::vector<std::shared_ptr<ErrorHandlerFSListener>> listener;
  std::vector<DB*> db;
  std::shared_ptr<SstFileManager> sfm(NewSstFileManager(def_env));
  int kNumDbInstances = 3;
  Random rnd(301);

  for (auto i = 0; i < kNumDbInstances; ++i) {
-    listener.emplace_back(new ErrorHandlerListener());
+    listener.emplace_back(new ErrorHandlerFSListener());
    options.emplace_back(GetDefaultOptions());
-    fault_env.emplace_back(new FaultInjectionTestEnv(Env::Default()));
+    fault_fs.emplace_back(
+        new FaultInjectionTestFS(FileSystem::Default().get()));
    options[i].create_if_missing = true;
    options[i].level0_file_num_compaction_trigger = 2;
    options[i].writable_file_max_buffer_size = 32768;
-    options[i].env = fault_env[i].get();
+    options[i].file_system.reset(fault_fs[i]);
    options[i].listeners.emplace_back(listener[i]);
    options[i].sst_file_manager = sfm;
    DB* dbptr;
@@ -652,8 +667,8 @@ TEST_F(DBErrorHandlingTest, MultiDBCompactionError) {

    listener[i]->EnableAutoRecovery();
    // Setup for returning error for the 3rd SST, which would be level 1
-    listener[i]->InjectFileCreationError(fault_env[i].get(), 3,
-                                         Status::NoSpace("Out of space"));
+    listener[i]->InjectFileCreationError(fault_fs[i], 3,
+                                         IOStatus::NoSpace("Out of space"));
    snprintf(buf, sizeof(buf), "_%d", i);
    DestroyDB(dbname_ + std::string(buf), options[i]);
    ASSERT_EQ(DB::Open(options[i], dbname_ + std::string(buf), &dbptr),
@@ -692,7 +707,7 @@ TEST_F(DBErrorHandlingTest, MultiDBCompactionError) {
  for (auto i = 0; i < kNumDbInstances; ++i) {
    Status s = static_cast<DBImpl*>(db[i])->TEST_WaitForCompact(true);
    ASSERT_EQ(s.severity(), Status::Severity::kSoftError);
-    fault_env[i]->SetFilesystemActive(true);
+    fault_fs[i]->SetFilesystemActive(true);
  }

  def_env->SetFilesystemActive(true);
@@ -713,7 +728,7 @@ TEST_F(DBErrorHandlingTest, MultiDBCompactionError) {
    char buf[16];
    snprintf(buf, sizeof(buf), "_%d", i);
    delete db[i];
-    fault_env[i]->SetFilesystemActive(true);
+    fault_fs[i]->SetFilesystemActive(true);
    if (getenv("KEEP_DB")) {
      printf("DB is still at %s%s\n", dbname_.c_str(), buf);
    } else {
@@ -725,24 +740,25 @@ TEST_F(DBErrorHandlingTest, MultiDBCompactionError) {
  delete def_env;
 }

-TEST_F(DBErrorHandlingTest, MultiDBVariousErrors) {
+TEST_F(DBErrorHandlingFSTest, MultiDBVariousErrors) {
  FaultInjectionTestEnv* def_env = new FaultInjectionTestEnv(Env::Default());
-  std::vector<std::unique_ptr<FaultInjectionTestEnv>> fault_env;
+  std::vector<FaultInjectionTestFS*> fault_fs;
  std::vector<Options> options;
-  std::vector<std::shared_ptr<ErrorHandlerListener>> listener;
+  std::vector<std::shared_ptr<ErrorHandlerFSListener>> listener;
  std::vector<DB*> db;
  std::shared_ptr<SstFileManager> sfm(NewSstFileManager(def_env));
  int kNumDbInstances = 3;
  Random rnd(301);

  for (auto i = 0; i < kNumDbInstances; ++i) {
-    listener.emplace_back(new ErrorHandlerListener());
+    listener.emplace_back(new ErrorHandlerFSListener());
    options.emplace_back(GetDefaultOptions());
-    fault_env.emplace_back(new FaultInjectionTestEnv(Env::Default()));
+    fault_fs.emplace_back(
+        new FaultInjectionTestFS(FileSystem::Default().get()));
    options[i].create_if_missing = true;
    options[i].level0_file_num_compaction_trigger = 2;
    options[i].writable_file_max_buffer_size = 32768;
-    options[i].env = fault_env[i].get();
+    options[i].file_system.reset(fault_fs[i]);
    options[i].listeners.emplace_back(listener[i]);
    options[i].sst_file_manager = sfm;
    DB* dbptr;
@@ -752,14 +768,14 @@ TEST_F(DBErrorHandlingTest, MultiDBVariousErrors) {
    switch (i) {
      case 0:
        // Setup for returning error for the 3rd SST, which would be level 1
-        listener[i]->InjectFileCreationError(fault_env[i].get(), 3,
-                                             Status::NoSpace("Out of space"));
+        listener[i]->InjectFileCreationError(fault_fs[i], 3,
+                                             IOStatus::NoSpace("Out of space"));
        break;
      case 1:
        // Setup for returning error after the 1st SST, which would result
        // in a hard error
-        listener[i]->InjectFileCreationError(fault_env[i].get(), 2,
-                                             Status::NoSpace("Out of space"));
+        listener[i]->InjectFileCreationError(fault_fs[i], 2,
+                                             IOStatus::NoSpace("Out of space"));
        break;
      default:
        break;
@@ -816,7 +832,7 @@ TEST_F(DBErrorHandlingTest, MultiDBVariousErrors) {
        ASSERT_EQ(s, Status::OK());
        break;
    }
-    fault_env[i]->SetFilesystemActive(true);
+    fault_fs[i]->SetFilesystemActive(true);
  }

  def_env->SetFilesystemActive(true);
@@ -840,7 +856,7 @@ TEST_F(DBErrorHandlingTest, MultiDBVariousErrors) {
  for (auto i = 0; i < kNumDbInstances; ++i) {
    char buf[16];
    snprintf(buf, sizeof(buf), "_%d", i);
-    fault_env[i]->SetFilesystemActive(true);
+    fault_fs[i]->SetFilesystemActive(true);
    delete db[i];
    if (getenv("KEEP_DB")) {
      printf("DB is still at %s%s\n", dbname_.c_str(), buf);
@@ -851,7 +867,6 @@ TEST_F(DBErrorHandlingTest, MultiDBVariousErrors) {
  options.clear();
  delete def_env;
 }
-
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -796,6 +796,45 @@ TEST_F(ExternalSSTFileBasicTest, VerifyChecksumReadahead) {
  Destroy(options);
 }

+TEST_F(ExternalSSTFileBasicTest, IngestRangeDeletionTombstoneWithGlobalSeqno) {
+  for (int i = 5; i < 25; i++) {
+    ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), Key(i),
+                       Key(i) + "_val"));
+  }
+
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  Reopen(options);
+  SstFileWriter sst_file_writer(EnvOptions(), options);
+
+  // file.sst (delete 0 => 30)
+  std::string file = sst_files_dir_ + "file.sst";
+  ASSERT_OK(sst_file_writer.Open(file));
+  ASSERT_OK(sst_file_writer.DeleteRange(Key(0), Key(30)));
+  ExternalSstFileInfo file_info;
+  ASSERT_OK(sst_file_writer.Finish(&file_info));
+  ASSERT_EQ(file_info.file_path, file);
+  ASSERT_EQ(file_info.num_entries, 0);
+  ASSERT_EQ(file_info.smallest_key, "");
+  ASSERT_EQ(file_info.largest_key, "");
+  ASSERT_EQ(file_info.num_range_del_entries, 1);
+  ASSERT_EQ(file_info.smallest_range_del_key, Key(0));
+  ASSERT_EQ(file_info.largest_range_del_key, Key(30));
+
+  IngestExternalFileOptions ifo;
+  ifo.move_files = true;
+  ifo.snapshot_consistency = true;
+  ifo.allow_global_seqno = true;
+  ifo.write_global_seqno = true;
+  ifo.verify_checksums_before_ingest = false;
+  ASSERT_OK(db_->IngestExternalFile({file}, ifo));
+
+  for (int i = 5; i < 25; i++) {
+    std::string res;
+    ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &res).IsNotFound());
+  }
+}
+
 TEST_P(ExternalSSTFileBasicTest, IngestionWithRangeDeletions) {
  int kNumLevels = 7;
  Options options = CurrentOptions();
@@ -46,7 +46,7 @@ Status ExternalSstFileIngestionJob::Prepare(
            TablePropertiesCollectorFactory::Context::kUnknownColumnFamily &&
        f.cf_id != cfd_->GetID()) {
      return Status::InvalidArgument(
-          "External file column family id dont match");
+          "External file column family id don't match");
    }
  }

@@ -55,7 +55,7 @@ Status ExternalSstFileIngestionJob::Prepare(
  if (num_files == 0) {
    return Status::InvalidArgument("The list of files is empty");
  } else if (num_files > 1) {
-    // Verify that passed files dont have overlapping ranges
+    // Verify that passed files don't have overlapping ranges
    autovector<const IngestedFileInfo*> sorted_files;
    for (size_t i = 0; i < num_files; i++) {
      sorted_files.push_back(&files_to_ingest_[i]);
@@ -148,7 +148,7 @@ Status ExternalSstFileIngestionJob::Prepare(
  TEST_SYNC_POINT("ExternalSstFileIngestionJob::BeforeSyncDir");
  if (status.ok()) {
    for (auto path_id : ingestion_path_ids) {
-      status = directories_->GetDataDir(path_id)->Fsync();
+      status = directories_->GetDataDir(path_id)->Fsync(IOOptions(), nullptr);
      if (!status.ok()) {
        ROCKS_LOG_WARN(db_options_.info_log,
                       "Failed to sync directory %" ROCKSDB_PRIszt
@@ -212,7 +212,7 @@ Status ExternalSstFileIngestionJob::Run() {

  if (ingestion_options_.snapshot_consistency && !db_snapshots_->empty()) {
    // We need to assign a global sequence number to all the files even
-    // if the dont overlap with any ranges since we have snapshots
+    // if the don't overlap with any ranges since we have snapshots
    force_global_seqno = true;
  }
  // It is safe to use this instead of LastAllocatedSequence since we are
@@ -588,7 +588,7 @@ Status ExternalSstFileIngestionJob::AssignLevelAndSeqnoForIngestedFile(
      continue;
    }

-    // We dont overlap with any keys in this level, but we still need to check
+    // We don't overlap with any keys in this level, but we still need to check
    // if our file can fit in it
    if (IngestedFileFitInLevel(file_to_ingest, lvl)) {
      target_level = lvl;
@@ -646,7 +646,7 @@ Status ExternalSstFileIngestionJob::AssignGlobalSeqnoForIngestedFile(
    return Status::InvalidArgument("Global seqno is required, but disabled");
  } else if (file_to_ingest->global_seqno_offset == 0) {
    return Status::InvalidArgument(
-        "Trying to set global seqno for a file that dont have a global seqno "
+        "Trying to set global seqno for a file that don't have a global seqno "
        "field");
  }

@@ -476,17 +476,17 @@ TEST_F(ExternalSSTFileTest, Basic) {
    }
    ASSERT_NE(db_->GetLatestSequenceNumber(), 0U);

-    // Key range of file5 (400 => 499) dont overlap with any keys in DB
+    // Key range of file5 (400 => 499) don't overlap with any keys in DB
    ASSERT_OK(DeprecatedAddFile({file5}));

    // This file has overlapping values with the existing data
    s = DeprecatedAddFile({file6});
    ASSERT_FALSE(s.ok()) << s.ToString();

-    // Key range of file7 (500 => 598) dont overlap with any keys in DB
+    // Key range of file7 (500 => 598) don't overlap with any keys in DB
    ASSERT_OK(DeprecatedAddFile({file7}));

-    // Key range of file7 (600 => 700) dont overlap with any keys in DB
+    // Key range of file7 (600 => 700) don't overlap with any keys in DB
    ASSERT_OK(DeprecatedAddFile({file8}));

    // Make sure values are correct before and after flush/compaction
@@ -1609,15 +1609,15 @@ TEST_F(ExternalSSTFileTest, AddExternalSstFileWithCustomCompartor) {
              generated_files[7]};
  ASSERT_NOK(DeprecatedAddFile(in_files));

-  // These 2 files dont overlap with each other
+  // These 2 files don't overlap with each other
  in_files = {generated_files[0], generated_files[2]};
  ASSERT_OK(DeprecatedAddFile(in_files));

-  // These 2 files dont overlap with each other but overlap with keys in DB
+  // These 2 files don't overlap with each other but overlap with keys in DB
  in_files = {generated_files[3], generated_files[7]};
  ASSERT_NOK(DeprecatedAddFile(in_files));

-  // Files dont overlap and dont overlap with DB key range
+  // Files don't overlap and don't overlap with DB key range
  in_files = {generated_files[4], generated_files[6], generated_files[8]};
  ASSERT_OK(DeprecatedAddFile(in_files));

@@ -1797,7 +1797,7 @@ TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) {
      options, file_data, -1, true, write_global_seqno,
      verify_checksums_before_ingest, false, false, &true_data));

-  // This file dont overlap with anything in the DB, will go to L4
+  // This file don't overlap with anything in the DB, will go to L4
  ASSERT_EQ("0,0,0,0,1", FilesPerLevel());

  // Insert 80 -> 130 using AddFile
@@ -1822,7 +1822,7 @@ TEST_P(ExternalSSTFileTest, IngestFileWithGlobalSeqnoAssignedLevel) {
      options, file_data, -1, true, write_global_seqno,
      verify_checksums_before_ingest, false, false, &true_data));

-  // This file dont overlap with anything in the DB and fit in L4 as well
+  // This file don't overlap with anything in the DB and fit in L4 as well
  ASSERT_EQ("2,0,0,0,2", FilesPerLevel());

  // Insert 10 -> 40 using AddFile
@@ -2059,16 +2059,16 @@ TEST_F(ExternalSSTFileTest, FileWithCFInfo) {

  IngestExternalFileOptions ifo;

-  // SST CF dont match
+  // SST CF don't match
  ASSERT_NOK(db_->IngestExternalFile(handles_[0], {cf1_sst}, ifo));
-  // SST CF dont match
+  // SST CF don't match
  ASSERT_NOK(db_->IngestExternalFile(handles_[2], {cf1_sst}, ifo));
  // SST CF match
  ASSERT_OK(db_->IngestExternalFile(handles_[1], {cf1_sst}, ifo));

-  // SST CF dont match
+  // SST CF don't match
  ASSERT_NOK(db_->IngestExternalFile(handles_[1], {cf_default_sst}, ifo));
-  // SST CF dont match
+  // SST CF don't match
  ASSERT_NOK(db_->IngestExternalFile(handles_[2], {cf_default_sst}, ifo));
  // SST CF match
  ASSERT_OK(db_->IngestExternalFile(handles_[0], {cf_default_sst}, ifo));
@@ -92,8 +92,8 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
                   std::vector<SequenceNumber> existing_snapshots,
                   SequenceNumber earliest_write_conflict_snapshot,
                   SnapshotChecker* snapshot_checker, JobContext* job_context,
-                   LogBuffer* log_buffer, Directory* db_directory,
-                   Directory* output_file_directory,
+                   LogBuffer* log_buffer, FSDirectory* db_directory,
+                   FSDirectory* output_file_directory,
                   CompressionType output_compression, Statistics* stats,
                   EventLogger* event_logger, bool measure_io_stats,
                   const bool sync_output_directory, const bool write_manifest,
@@ -371,11 +371,6 @@ Status FlushJob::WriteLevel0Table() {
      meta_.oldest_ancester_time = std::min(current_time, oldest_key_time);
      meta_.file_creation_time = current_time;

-      uint64_t creation_time = (cfd_->ioptions()->compaction_style ==
-                                CompactionStyle::kCompactionStyleFIFO)
-                                   ? current_time
-                                   : meta_.oldest_ancester_time;
-
      s = BuildTable(
          dbname_, db_options_.env, db_options_.fs.get(), *cfd_->ioptions(),
          mutable_cf_options_, file_options_, cfd_->table_cache(), iter.get(),
@@ -388,7 +383,8 @@ Status FlushJob::WriteLevel0Table() {
          mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
          TableFileCreationReason::kFlush, event_logger_, job_context_->job_id,
          Env::IO_HIGH, &table_properties_, 0 /* level */,
-          creation_time, oldest_key_time, write_hint, current_time);
+          meta_.oldest_ancester_time, oldest_key_time, write_hint,
+          current_time);
      LogFlush(db_options_.info_log);
    }
    ROCKS_LOG_INFO(db_options_.info_log,
@@ -401,7 +397,7 @@ Status FlushJob::WriteLevel0Table() {
                   meta_.marked_for_compaction ? " (needs compaction)" : "");

    if (s.ok() && output_file_directory_ != nullptr && sync_output_directory_) {
-      s = output_file_directory_->Fsync();
+      s = output_file_directory_->Fsync(IOOptions(), nullptr);
    }
    TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table", &mems_);
    db_mutex_->Lock();
@@ -67,9 +67,10 @@ class FlushJob {
           std::vector<SequenceNumber> existing_snapshots,
           SequenceNumber earliest_write_conflict_snapshot,
           SnapshotChecker* snapshot_checker, JobContext* job_context,
-           LogBuffer* log_buffer, Directory* db_directory,
-           Directory* output_file_directory, CompressionType output_compression,
-           Statistics* stats, EventLogger* event_logger, bool measure_io_stats,
+           LogBuffer* log_buffer, FSDirectory* db_directory,
+           FSDirectory* output_file_directory,
+           CompressionType output_compression, Statistics* stats,
+           EventLogger* event_logger, bool measure_io_stats,
           const bool sync_output_directory, const bool write_manifest,
           Env::Priority thread_pri);

@@ -117,8 +118,8 @@ class FlushJob {
  SnapshotChecker* snapshot_checker_;
  JobContext* job_context_;
  LogBuffer* log_buffer_;
-  Directory* db_directory_;
-  Directory* output_file_directory_;
+  FSDirectory* db_directory_;
+  FSDirectory* output_file_directory_;
  CompressionType output_compression_;
  Statistics* stats_;
  EventLogger* event_logger_;
@@ -8,7 +8,7 @@
 #include <map>
 #include <string>

-#include "db/blob_index.h"
+#include "db/blob/blob_index.h"
 #include "db/column_family.h"
 #include "db/db_impl/db_impl.h"
 #include "db/flush_job.h"
@@ -3,7 +3,7 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

-#include "db/blob_index.h"
+#include "db/blob/blob_index.h"
 #include "db/db_impl/db_impl.h"
 #include "db/db_test_util.h"
 #include "db/dbformat.h"
@@ -4,8 +4,6 @@
 //  (found in the LICENSE.Apache file in the root directory).
 //
 // Test for issue 178: a manual compaction causes deleted data to reappear.
-#include <iostream>
-#include <sstream>
 #include <cstdlib>

 #include "port/port.h"
@@ -40,8 +38,8 @@ class ManualCompactionTest : public testing::Test {
 public:
  ManualCompactionTest() {
    // Get rid of any state from an old run.
-    dbname_ = ROCKSDB_NAMESPACE::test::PerThreadDBPath("rocksdb_cbug_test");
-    DestroyDB(dbname_, ROCKSDB_NAMESPACE::Options());
+    dbname_ = test::PerThreadDBPath("rocksdb_manual_compaction_test");
+    DestroyDB(dbname_, Options());
  }

  std::string dbname_;
@@ -60,6 +58,33 @@ class DestroyAllCompactionFilter : public CompactionFilter {
  const char* Name() const override { return "DestroyAllCompactionFilter"; }
 };

+class LogCompactionFilter : public CompactionFilter {
+ public:
+  const char* Name() const override { return "LogCompactionFilter"; }
+
+  bool Filter(int level, const Slice& key, const Slice& /*existing_value*/,
+              std::string* /*new_value*/,
+              bool* /*value_changed*/) const override {
+    key_level_[key.ToString()] = level;
+    return false;
+  }
+
+  void Reset() { key_level_.clear(); }
+
+  size_t NumKeys() const { return key_level_.size(); }
+
+  int KeyLevel(const Slice& key) {
+    auto it = key_level_.find(key.ToString());
+    if (it == key_level_.end()) {
+      return -1;
+    }
+    return it->second;
+  }
+
+ private:
+  mutable std::map<std::string, int> key_level_;
+};
+
 TEST_F(ManualCompactionTest, CompactTouchesAllKeys) {
  for (int iter = 0; iter < 2; ++iter) {
    DB* db;
@@ -71,7 +96,7 @@ TEST_F(ManualCompactionTest, CompactTouchesAllKeys) {
      options.compaction_style = kCompactionStyleUniversal;
    }
    options.create_if_missing = true;
-    options.compression = ROCKSDB_NAMESPACE::kNoCompression;
+    options.compression = kNoCompression;
    options.compaction_filter = new DestroyAllCompactionFilter();
    ASSERT_OK(DB::Open(options, dbname_, &db));

@@ -100,46 +125,45 @@ TEST_F(ManualCompactionTest, Test) {
  // Open database.  Disable compression since it affects the creation
  // of layers and the code below is trying to test against a very
  // specific scenario.
-  ROCKSDB_NAMESPACE::DB* db;
-  ROCKSDB_NAMESPACE::Options db_options;
+  DB* db;
+  Options db_options;
  db_options.write_buffer_size = 1024;
  db_options.create_if_missing = true;
-  db_options.compression = ROCKSDB_NAMESPACE::kNoCompression;
-  ASSERT_OK(ROCKSDB_NAMESPACE::DB::Open(db_options, dbname_, &db));
+  db_options.compression = kNoCompression;
+  ASSERT_OK(DB::Open(db_options, dbname_, &db));

  // create first key range
-  ROCKSDB_NAMESPACE::WriteBatch batch;
+  WriteBatch batch;
  for (int i = 0; i < kNumKeys; i++) {
    batch.Put(Key1(i), "value for range 1 key");
  }
-  ASSERT_OK(db->Write(ROCKSDB_NAMESPACE::WriteOptions(), &batch));
+  ASSERT_OK(db->Write(WriteOptions(), &batch));

  // create second key range
  batch.Clear();
  for (int i = 0; i < kNumKeys; i++) {
    batch.Put(Key2(i), "value for range 2 key");
  }
-  ASSERT_OK(db->Write(ROCKSDB_NAMESPACE::WriteOptions(), &batch));
+  ASSERT_OK(db->Write(WriteOptions(), &batch));

  // delete second key range
  batch.Clear();
  for (int i = 0; i < kNumKeys; i++) {
    batch.Delete(Key2(i));
  }
-  ASSERT_OK(db->Write(ROCKSDB_NAMESPACE::WriteOptions(), &batch));
+  ASSERT_OK(db->Write(WriteOptions(), &batch));

  // compact database
  std::string start_key = Key1(0);
  std::string end_key = Key1(kNumKeys - 1);
-  ROCKSDB_NAMESPACE::Slice least(start_key.data(), start_key.size());
-  ROCKSDB_NAMESPACE::Slice greatest(end_key.data(), end_key.size());
+  Slice least(start_key.data(), start_key.size());
+  Slice greatest(end_key.data(), end_key.size());

  // commenting out the line below causes the example to work correctly
  db->CompactRange(CompactRangeOptions(), &least, &greatest);

  // count the keys
-  ROCKSDB_NAMESPACE::Iterator* iter =
-      db->NewIterator(ROCKSDB_NAMESPACE::ReadOptions());
+  Iterator* iter = db->NewIterator(ReadOptions());
  int num_keys = 0;
  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
    num_keys++;
@@ -149,7 +173,119 @@ TEST_F(ManualCompactionTest, Test) {

  // close database
  delete db;
-  DestroyDB(dbname_, ROCKSDB_NAMESPACE::Options());
+  DestroyDB(dbname_, Options());
+}
+
+TEST_F(ManualCompactionTest, SkipLevel) {
+  DB* db;
+  Options options;
+  options.num_levels = 3;
+  // Initially, flushed L0 files won't exceed 100.
+  options.level0_file_num_compaction_trigger = 100;
+  options.compaction_style = kCompactionStyleLevel;
+  options.create_if_missing = true;
+  options.compression = kNoCompression;
+  LogCompactionFilter* filter = new LogCompactionFilter();
+  options.compaction_filter = filter;
+  ASSERT_OK(DB::Open(options, dbname_, &db));
+
+  WriteOptions wo;
+  FlushOptions fo;
+  ASSERT_OK(db->Put(wo, "1", ""));
+  ASSERT_OK(db->Flush(fo));
+  ASSERT_OK(db->Put(wo, "2", ""));
+  ASSERT_OK(db->Flush(fo));
+  ASSERT_OK(db->Put(wo, "4", ""));
+  ASSERT_OK(db->Put(wo, "8", ""));
+  ASSERT_OK(db->Flush(fo));
+
+  {
+    // L0: 1, 2, [4, 8]
+    // no file has keys in range [5, 7]
+    Slice start("5");
+    Slice end("7");
+    filter->Reset();
+    db->CompactRange(CompactRangeOptions(), &start, &end);
+    ASSERT_EQ(0, filter->NumKeys());
+  }
+
+  {
+    // L0: 1, 2, [4, 8]
+    // [3, 7] overlaps with 4 in L0
+    Slice start("3");
+    Slice end("7");
+    filter->Reset();
+    db->CompactRange(CompactRangeOptions(), &start, &end);
+    ASSERT_EQ(2, filter->NumKeys());
+    ASSERT_EQ(0, filter->KeyLevel("4"));
+    ASSERT_EQ(0, filter->KeyLevel("8"));
+  }
+
+  {
+    // L0: 1, 2
+    // L1: [4, 8]
+    // no file has keys in range (-inf, 0]
+    Slice end("0");
+    filter->Reset();
+    db->CompactRange(CompactRangeOptions(), nullptr, &end);
+    ASSERT_EQ(0, filter->NumKeys());
+  }
+
+  {
+    // L0: 1, 2
+    // L1: [4, 8]
+    // no file has keys in range [9, inf)
+    Slice start("9");
+    filter->Reset();
+    db->CompactRange(CompactRangeOptions(), &start, nullptr);
+    ASSERT_EQ(0, filter->NumKeys());
+  }
+
+  {
+    // L0: 1, 2
+    // L1: [4, 8]
+    // [2, 2] overlaps with 2 in L0
+    Slice start("2");
+    Slice end("2");
+    filter->Reset();
+    db->CompactRange(CompactRangeOptions(), &start, &end);
+    ASSERT_EQ(1, filter->NumKeys());
+    ASSERT_EQ(0, filter->KeyLevel("2"));
+  }
+
+  {
+    // L0: 1
+    // L1: 2, [4, 8]
+    // [2, 5] overlaps with 2 and [4, 8) in L1, skip L0
+    Slice start("2");
+    Slice end("5");
+    filter->Reset();
+    db->CompactRange(CompactRangeOptions(), &start, &end);
+    ASSERT_EQ(3, filter->NumKeys());
+    ASSERT_EQ(1, filter->KeyLevel("2"));
+    ASSERT_EQ(1, filter->KeyLevel("4"));
+    ASSERT_EQ(1, filter->KeyLevel("8"));
+  }
+
+  {
+    // L0: 1
+    // L1: [2, 4, 8]
+    // [0, inf) overlaps all files
+    Slice start("0");
+    filter->Reset();
+    db->CompactRange(CompactRangeOptions(), &start, nullptr);
+    ASSERT_EQ(4, filter->NumKeys());
+    // 1 is first compacted to L1 and then further compacted into [2, 4, 8],
+    // so finally the logged level for 1 is L1.
+    ASSERT_EQ(1, filter->KeyLevel("1"));
+    ASSERT_EQ(1, filter->KeyLevel("2"));
+    ASSERT_EQ(1, filter->KeyLevel("4"));
+    ASSERT_EQ(1, filter->KeyLevel("8"));
+  }
+
+  delete filter;
+  delete db;
+  DestroyDB(dbname_, options);
 }

 }  // anonymous namespace
@@ -600,6 +600,7 @@ struct Saver {
  bool* merge_in_progress;
  std::string* value;
  SequenceNumber seq;
+  std::string* timestamp;
  const MergeOperator* merge_operator;
  // the merge operations encountered;
  MergeContext* merge_context;
@@ -643,9 +644,11 @@ static bool SaveValue(void* arg, const char* entry) {
  uint32_t key_length;
  const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
  Slice user_key_slice = Slice(key_ptr, key_length - 8);
-  if (s->mem->GetInternalKeyComparator()
-          .user_comparator()
-          ->CompareWithoutTimestamp(user_key_slice, s->key->user_key()) == 0) {
+  const Comparator* user_comparator =
+      s->mem->GetInternalKeyComparator().user_comparator();
+  size_t ts_sz = user_comparator->timestamp_size();
+  if (user_comparator->CompareWithoutTimestamp(user_key_slice,
+                                               s->key->user_key()) == 0) {
    // Correct user key
    const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
    ValueType type;
@@ -713,6 +716,11 @@ static bool SaveValue(void* arg, const char* entry) {
        if (s->is_blob_index != nullptr) {
          *(s->is_blob_index) = (type == kTypeBlobIndex);
        }
+
+        if (ts_sz > 0 && s->timestamp != nullptr) {
+          Slice ts = ExtractTimestampFromUserKey(user_key_slice, ts_sz);
+          s->timestamp->assign(ts.data(), ts.size());
+        }
        return false;
      }
      case kTypeDeletion:
@@ -767,7 +775,8 @@ static bool SaveValue(void* arg, const char* entry) {
  return false;
 }

-bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
+bool MemTable::Get(const LookupKey& key, std::string* value,
+                   std::string* timestamp, Status* s,
                   MergeContext* merge_context,
                   SequenceNumber* max_covering_tombstone_seq,
                   SequenceNumber* seq, const ReadOptions& read_opts,
@@ -816,7 +825,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
      PERF_COUNTER_ADD(bloom_memtable_hit_count, 1);
    }
    GetFromTable(key, *max_covering_tombstone_seq, do_merge, callback,
-                 is_blob_index, value, s, merge_context, seq,
+                 is_blob_index, value, timestamp, s, merge_context, seq,
                 &found_final_value, &merge_in_progress);
  }

@@ -831,7 +840,8 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
 void MemTable::GetFromTable(const LookupKey& key,
                            SequenceNumber max_covering_tombstone_seq,
                            bool do_merge, ReadCallback* callback,
-                            bool* is_blob_index, std::string* value, Status* s,
+                            bool* is_blob_index, std::string* value,
+                            std::string* timestamp, Status* s,
                            MergeContext* merge_context, SequenceNumber* seq,
                            bool* found_final_value, bool* merge_in_progress) {
  Saver saver;
@@ -840,6 +850,7 @@ void MemTable::GetFromTable(const LookupKey& key,
  saver.merge_in_progress = merge_in_progress;
  saver.key = &key;
  saver.value = value;
+  saver.timestamp = timestamp;
  saver.seq = kMaxSequenceNumber;
  saver.mem = this;
  saver.merge_context = merge_context;
@@ -908,9 +919,9 @@ void MemTable::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
          range_del_iter->MaxCoveringTombstoneSeqnum(iter->lkey->user_key()));
    }
    GetFromTable(*(iter->lkey), iter->max_covering_tombstone_seq, true,
-                 callback, is_blob, iter->value->GetSelf(), iter->s,
-                 &(iter->merge_context), &seq, &found_final_value,
-                 &merge_in_progress);
+                 callback, is_blob, iter->value->GetSelf(),
+                 /*timestamp=*/nullptr, iter->s, &(iter->merge_context), &seq,
+                 &found_final_value, &merge_in_progress);

    if (!found_final_value && merge_in_progress) {
      *(iter->s) = Status::MergeInProgress();
@@ -212,16 +212,27 @@ class MemTable {
           MergeContext* merge_context,
           SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq,
           const ReadOptions& read_opts, ReadCallback* callback = nullptr,
+           bool* is_blob_index = nullptr, bool do_merge = true) {
+    return Get(key, value, /*timestamp=*/nullptr, s, merge_context,
+               max_covering_tombstone_seq, seq, read_opts, callback,
+               is_blob_index, do_merge);
+  }
+
+  bool Get(const LookupKey& key, std::string* value, std::string* timestamp,
+           Status* s, MergeContext* merge_context,
+           SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq,
+           const ReadOptions& read_opts, ReadCallback* callback = nullptr,
           bool* is_blob_index = nullptr, bool do_merge = true);

-  bool Get(const LookupKey& key, std::string* value, Status* s,
-           MergeContext* merge_context,
+  bool Get(const LookupKey& key, std::string* value, std::string* timestamp,
+           Status* s, MergeContext* merge_context,
           SequenceNumber* max_covering_tombstone_seq,
           const ReadOptions& read_opts, ReadCallback* callback = nullptr,
           bool* is_blob_index = nullptr, bool do_merge = true) {
    SequenceNumber seq;
-    return Get(key, value, s, merge_context, max_covering_tombstone_seq, &seq,
-               read_opts, callback, is_blob_index, do_merge);
+    return Get(key, value, timestamp, s, merge_context,
+               max_covering_tombstone_seq, &seq, read_opts, callback,
+               is_blob_index, do_merge);
  }

  void MultiGet(const ReadOptions& read_options, MultiGetRange* range,
@@ -532,9 +543,9 @@ class MemTable {
  void GetFromTable(const LookupKey& key,
                    SequenceNumber max_covering_tombstone_seq, bool do_merge,
                    ReadCallback* callback, bool* is_blob_index,
-                    std::string* value, Status* s, MergeContext* merge_context,
-                    SequenceNumber* seq, bool* found_final_value,
-                    bool* merge_in_progress);
+                    std::string* value, std::string* timestamp, Status* s,
+                    MergeContext* merge_context, SequenceNumber* seq,
+                    bool* found_final_value, bool* merge_in_progress);
 };

 extern const char* EncodeKey(std::string* scratch, const Slice& target);
@@ -104,11 +104,12 @@ int MemTableList::NumFlushed() const {
 // Return the most recent value found, if any.
 // Operands stores the list of merge operations to apply, so far.
 bool MemTableListVersion::Get(const LookupKey& key, std::string* value,
-                              Status* s, MergeContext* merge_context,
+                              std::string* timestamp, Status* s,
+                              MergeContext* merge_context,
                              SequenceNumber* max_covering_tombstone_seq,
                              SequenceNumber* seq, const ReadOptions& read_opts,
                              ReadCallback* callback, bool* is_blob_index) {
-  return GetFromList(&memlist_, key, value, s, merge_context,
+  return GetFromList(&memlist_, key, value, timestamp, s, merge_context,
                     max_covering_tombstone_seq, seq, read_opts, callback,
                     is_blob_index);
 }
@@ -128,9 +129,9 @@ bool MemTableListVersion::GetMergeOperands(
    const LookupKey& key, Status* s, MergeContext* merge_context,
    SequenceNumber* max_covering_tombstone_seq, const ReadOptions& read_opts) {
  for (MemTable* memtable : memlist_) {
-    bool done = memtable->Get(key, nullptr, s, merge_context,
-                              max_covering_tombstone_seq, read_opts, nullptr,
-                              nullptr, false);
+    bool done = memtable->Get(key, /*value*/ nullptr, /*timestamp*/ nullptr, s,
+                              merge_context, max_covering_tombstone_seq,
+                              read_opts, nullptr, nullptr, false);
    if (done) {
      return true;
    }
@@ -139,17 +140,17 @@ bool MemTableListVersion::GetMergeOperands(
 }

 bool MemTableListVersion::GetFromHistory(
-    const LookupKey& key, std::string* value, Status* s,
+    const LookupKey& key, std::string* value, std::string* timestamp, Status* s,
    MergeContext* merge_context, SequenceNumber* max_covering_tombstone_seq,
    SequenceNumber* seq, const ReadOptions& read_opts, bool* is_blob_index) {
-  return GetFromList(&memlist_history_, key, value, s, merge_context,
+  return GetFromList(&memlist_history_, key, value, timestamp, s, merge_context,
                     max_covering_tombstone_seq, seq, read_opts,
                     nullptr /*read_callback*/, is_blob_index);
 }

 bool MemTableListVersion::GetFromList(
    std::list<MemTable*>* list, const LookupKey& key, std::string* value,
-    Status* s, MergeContext* merge_context,
+    std::string* timestamp, Status* s, MergeContext* merge_context,
    SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq,
    const ReadOptions& read_opts, ReadCallback* callback, bool* is_blob_index) {
  *seq = kMaxSequenceNumber;
@@ -157,9 +158,9 @@ bool MemTableListVersion::GetFromList(
  for (auto& memtable : *list) {
    SequenceNumber current_seq = kMaxSequenceNumber;

-    bool done =
-        memtable->Get(key, value, s, merge_context, max_covering_tombstone_seq,
-                      &current_seq, read_opts, callback, is_blob_index);
+    bool done = memtable->Get(key, value, timestamp, s, merge_context,
+                              max_covering_tombstone_seq, &current_seq,
+                              read_opts, callback, is_blob_index);
    if (*seq == kMaxSequenceNumber) {
      // Store the most recent sequence number of any operation on this key.
      // Since we only care about the most recent change, we only need to
@@ -387,7 +388,7 @@ Status MemTableList::TryInstallMemtableFlushResults(
    ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
    const autovector<MemTable*>& mems, LogsWithPrepTracker* prep_tracker,
    VersionSet* vset, InstrumentedMutex* mu, uint64_t file_number,
-    autovector<MemTable*>* to_delete, Directory* db_directory,
+    autovector<MemTable*>* to_delete, FSDirectory* db_directory,
    LogBuffer* log_buffer,
    std::list<std::unique_ptr<FlushJobInfo>>* committed_flush_jobs_info) {
  AutoThreadOperationStageUpdater stage_updater(
@@ -641,7 +642,7 @@ Status InstallMemtableAtomicFlushResults(
    const autovector<const MutableCFOptions*>& mutable_cf_options_list,
    const autovector<const autovector<MemTable*>*>& mems_list, VersionSet* vset,
    InstrumentedMutex* mu, const autovector<FileMetaData*>& file_metas,
-    autovector<MemTable*>* to_delete, Directory* db_directory,
+    autovector<MemTable*>* to_delete, FSDirectory* db_directory,
    LogBuffer* log_buffer) {
  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_MEMTABLE_INSTALL_FLUSH_RESULTS);
@@ -58,20 +58,21 @@ class MemTableListVersion {
  // If any operation was found for this key, its most recent sequence number
  // will be stored in *seq on success (regardless of whether true/false is
  // returned).  Otherwise, *seq will be set to kMaxSequenceNumber.
-  bool Get(const LookupKey& key, std::string* value, Status* s,
-           MergeContext* merge_context,
+  bool Get(const LookupKey& key, std::string* value, std::string* timestamp,
+           Status* s, MergeContext* merge_context,
           SequenceNumber* max_covering_tombstone_seq, SequenceNumber* seq,
           const ReadOptions& read_opts, ReadCallback* callback = nullptr,
           bool* is_blob_index = nullptr);

-  bool Get(const LookupKey& key, std::string* value, Status* s,
-           MergeContext* merge_context,
+  bool Get(const LookupKey& key, std::string* value, std::string* timestamp,
+           Status* s, MergeContext* merge_context,
           SequenceNumber* max_covering_tombstone_seq,
           const ReadOptions& read_opts, ReadCallback* callback = nullptr,
           bool* is_blob_index = nullptr) {
    SequenceNumber seq;
-    return Get(key, value, s, merge_context, max_covering_tombstone_seq, &seq,
-               read_opts, callback, is_blob_index);
+    return Get(key, value, timestamp, s, merge_context,
+               max_covering_tombstone_seq, &seq, read_opts, callback,
+               is_blob_index);
  }

  void MultiGet(const ReadOptions& read_options, MultiGetRange* range,
@@ -88,18 +89,20 @@ class MemTableListVersion {
  // have already been flushed.  Should only be used from in-memory only
  // queries (such as Transaction validation) as the history may contain
  // writes that are also present in the SST files.
-  bool GetFromHistory(const LookupKey& key, std::string* value, Status* s,
+  bool GetFromHistory(const LookupKey& key, std::string* value,
+                      std::string* timestamp, Status* s,
                      MergeContext* merge_context,
                      SequenceNumber* max_covering_tombstone_seq,
                      SequenceNumber* seq, const ReadOptions& read_opts,
                      bool* is_blob_index = nullptr);
-  bool GetFromHistory(const LookupKey& key, std::string* value, Status* s,
+  bool GetFromHistory(const LookupKey& key, std::string* value,
+                      std::string* timestamp, Status* s,
                      MergeContext* merge_context,
                      SequenceNumber* max_covering_tombstone_seq,
                      const ReadOptions& read_opts,
                      bool* is_blob_index = nullptr) {
    SequenceNumber seq;
-    return GetFromHistory(key, value, s, merge_context,
+    return GetFromHistory(key, value, timestamp, s, merge_context,
                          max_covering_tombstone_seq, &seq, read_opts,
                          is_blob_index);
  }
@@ -137,7 +140,7 @@ class MemTableListVersion {
      const autovector<const autovector<MemTable*>*>& mems_list,
      VersionSet* vset, InstrumentedMutex* mu,
      const autovector<FileMetaData*>& file_meta,
-      autovector<MemTable*>* to_delete, Directory* db_directory,
+      autovector<MemTable*>* to_delete, FSDirectory* db_directory,
      LogBuffer* log_buffer);

  // REQUIRE: m is an immutable memtable
@@ -148,7 +151,8 @@ class MemTableListVersion {
  void TrimHistory(autovector<MemTable*>* to_delete, size_t usage);

  bool GetFromList(std::list<MemTable*>* list, const LookupKey& key,
-                   std::string* value, Status* s, MergeContext* merge_context,
+                   std::string* value, std::string* timestamp, Status* s,
+                   MergeContext* merge_context,
                   SequenceNumber* max_covering_tombstone_seq,
                   SequenceNumber* seq, const ReadOptions& read_opts,
                   ReadCallback* callback = nullptr,
@@ -260,7 +264,7 @@ class MemTableList {
      ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
      const autovector<MemTable*>& m, LogsWithPrepTracker* prep_tracker,
      VersionSet* vset, InstrumentedMutex* mu, uint64_t file_number,
-      autovector<MemTable*>* to_delete, Directory* db_directory,
+      autovector<MemTable*>* to_delete, FSDirectory* db_directory,
      LogBuffer* log_buffer,
      std::list<std::unique_ptr<FlushJobInfo>>* committed_flush_jobs_info);

@@ -375,7 +379,7 @@ class MemTableList {
      const autovector<const autovector<MemTable*>*>& mems_list,
      VersionSet* vset, InstrumentedMutex* mu,
      const autovector<FileMetaData*>& file_meta,
-      autovector<MemTable*>* to_delete, Directory* db_directory,
+      autovector<MemTable*>* to_delete, FSDirectory* db_directory,
      LogBuffer* log_buffer);

  // DB mutex held
@@ -417,6 +421,6 @@ extern Status InstallMemtableAtomicFlushResults(
    const autovector<const MutableCFOptions*>& mutable_cf_options_list,
    const autovector<const autovector<MemTable*>*>& mems_list, VersionSet* vset,
    InstrumentedMutex* mu, const autovector<FileMetaData*>& file_meta,
-    autovector<MemTable*>* to_delete, Directory* db_directory,
+    autovector<MemTable*>* to_delete, FSDirectory* db_directory,
    LogBuffer* log_buffer);
 }  // namespace ROCKSDB_NAMESPACE
@@ -221,8 +221,9 @@ TEST_F(MemTableListTest, GetTest) {
  autovector<MemTable*> to_delete;

  LookupKey lkey("key1", seq);
-  bool found = list.current()->Get(lkey, &value, &s, &merge_context,
-                                   &max_covering_tombstone_seq, ReadOptions());
+  bool found = list.current()->Get(
+      lkey, &value, /*timestamp*/nullptr, &s, &merge_context,
+      &max_covering_tombstone_seq, ReadOptions());
  ASSERT_FALSE(found);

  // Create a MemTable
@@ -244,19 +245,22 @@ TEST_F(MemTableListTest, GetTest) {

  // Fetch the newly written keys
  merge_context.Clear();
-  found = mem->Get(LookupKey("key1", seq), &value, &s, &merge_context,
+  found = mem->Get(LookupKey("key1", seq), &value,
+                   /*timestamp*/nullptr, &s, &merge_context,
                   &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(s.ok() && found);
  ASSERT_EQ(value, "value1");

  merge_context.Clear();
-  found = mem->Get(LookupKey("key1", 2), &value, &s, &merge_context,
+  found = mem->Get(LookupKey("key1", 2), &value,
+                   /*timestamp*/nullptr, &s, &merge_context,
                   &max_covering_tombstone_seq, ReadOptions());
  // MemTable found out that this key is *not* found (at this sequence#)
  ASSERT_TRUE(found && s.IsNotFound());

  merge_context.Clear();
-  found = mem->Get(LookupKey("key2", seq), &value, &s, &merge_context,
+  found = mem->Get(LookupKey("key2", seq), &value,
+                   /*timestamp*/nullptr, &s, &merge_context,
                   &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(s.ok() && found);
  ASSERT_EQ(value, "value2.2");
@@ -283,28 +287,29 @@ TEST_F(MemTableListTest, GetTest) {

  // Fetch keys via MemTableList
  merge_context.Clear();
-  found =
-      list.current()->Get(LookupKey("key1", seq), &value, &s, &merge_context,
-                          &max_covering_tombstone_seq, ReadOptions());
+  found = list.current()->Get(
+      LookupKey("key1", seq), &value, /*timestamp*/nullptr, &s,
+      &merge_context, &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(found && s.IsNotFound());

  merge_context.Clear();
-  found = list.current()->Get(LookupKey("key1", saved_seq), &value, &s,
-                              &merge_context, &max_covering_tombstone_seq,
-                              ReadOptions());
+  found = list.current()->Get(
+      LookupKey("key1", saved_seq), &value, /*timestamp*/nullptr,
+      &s, &merge_context, &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(s.ok() && found);
  ASSERT_EQ("value1", value);

  merge_context.Clear();
-  found =
-      list.current()->Get(LookupKey("key2", seq), &value, &s, &merge_context,
-                          &max_covering_tombstone_seq, ReadOptions());
+  found = list.current()->Get(
+      LookupKey("key2", seq), &value, /*timestamp*/nullptr, &s,
+      &merge_context, &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(s.ok() && found);
  ASSERT_EQ(value, "value2.3");

  merge_context.Clear();
-  found = list.current()->Get(LookupKey("key2", 1), &value, &s, &merge_context,
-                              &max_covering_tombstone_seq, ReadOptions());
+  found = list.current()->Get(
+      LookupKey("key2", 1), &value, /*timestamp*/nullptr, &s,
+      &merge_context, &max_covering_tombstone_seq, ReadOptions());
  ASSERT_FALSE(found);

  ASSERT_EQ(2, list.NumNotFlushed());
@@ -333,8 +338,9 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {
  autovector<MemTable*> to_delete;

  LookupKey lkey("key1", seq);
-  bool found = list.current()->Get(lkey, &value, &s, &merge_context,
-                                   &max_covering_tombstone_seq, ReadOptions());
+  bool found = list.current()->Get(
+      lkey, &value, /*timestamp*/nullptr, &s, &merge_context,
+      &max_covering_tombstone_seq, ReadOptions());
  ASSERT_FALSE(found);

  // Create a MemTable
@@ -355,13 +361,15 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {

  // Fetch the newly written keys
  merge_context.Clear();
-  found = mem->Get(LookupKey("key1", seq), &value, &s, &merge_context,
+  found = mem->Get(LookupKey("key1", seq), &value,
+                   /*timestamp*/nullptr, &s, &merge_context,
                   &max_covering_tombstone_seq, ReadOptions());
  // MemTable found out that this key is *not* found (at this sequence#)
  ASSERT_TRUE(found && s.IsNotFound());

  merge_context.Clear();
-  found = mem->Get(LookupKey("key2", seq), &value, &s, &merge_context,
+  found = mem->Get(LookupKey("key2", seq), &value,
+                   /*timestamp*/nullptr, &s, &merge_context,
                   &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(s.ok() && found);
  ASSERT_EQ(value, "value2.2");
@@ -372,15 +380,15 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {

  // Fetch keys via MemTableList
  merge_context.Clear();
-  found =
-      list.current()->Get(LookupKey("key1", seq), &value, &s, &merge_context,
-                          &max_covering_tombstone_seq, ReadOptions());
+  found = list.current()->Get(LookupKey("key1", seq), &value,
+                              /*timestamp*/nullptr, &s, &merge_context,
+                              &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(found && s.IsNotFound());

  merge_context.Clear();
-  found =
-      list.current()->Get(LookupKey("key2", seq), &value, &s, &merge_context,
-                          &max_covering_tombstone_seq, ReadOptions());
+  found = list.current()->Get(LookupKey("key2", seq), &value,
+                              /*timestamp*/nullptr, &s, &merge_context,
+                              &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(s.ok() && found);
  ASSERT_EQ("value2.2", value);

@@ -400,27 +408,27 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {

  // Verify keys are no longer in MemTableList
  merge_context.Clear();
-  found =
-      list.current()->Get(LookupKey("key1", seq), &value, &s, &merge_context,
-                          &max_covering_tombstone_seq, ReadOptions());
+  found = list.current()->Get(LookupKey("key1", seq), &value,
+                              /*timestamp*/nullptr, &s, &merge_context,
+                              &max_covering_tombstone_seq, ReadOptions());
  ASSERT_FALSE(found);

  merge_context.Clear();
-  found =
-      list.current()->Get(LookupKey("key2", seq), &value, &s, &merge_context,
-                          &max_covering_tombstone_seq, ReadOptions());
+  found = list.current()->Get(LookupKey("key2", seq), &value,
+                              /*timestamp*/nullptr, &s, &merge_context,
+                              &max_covering_tombstone_seq, ReadOptions());
  ASSERT_FALSE(found);

  // Verify keys are present in history
  merge_context.Clear();
  found = list.current()->GetFromHistory(
-      LookupKey("key1", seq), &value, &s, &merge_context,
+      LookupKey("key1", seq), &value, /*timestamp*/nullptr, &s, &merge_context,
      &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(found && s.IsNotFound());

  merge_context.Clear();
  found = list.current()->GetFromHistory(
-      LookupKey("key2", seq), &value, &s, &merge_context,
+      LookupKey("key2", seq), &value, /*timestamp*/nullptr, &s, &merge_context,
      &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(found);
  ASSERT_EQ("value2.2", value);
@@ -462,42 +470,42 @@ TEST_F(MemTableListTest, GetFromHistoryTest) {

  // Verify keys are no longer in MemTableList
  merge_context.Clear();
-  found =
-      list.current()->Get(LookupKey("key1", seq), &value, &s, &merge_context,
-                          &max_covering_tombstone_seq, ReadOptions());
+  found = list.current()->Get(LookupKey("key1", seq), &value,
+                              /*timestamp*/nullptr, &s, &merge_context,
+                              &max_covering_tombstone_seq, ReadOptions());
  ASSERT_FALSE(found);

  merge_context.Clear();
-  found =
-      list.current()->Get(LookupKey("key2", seq), &value, &s, &merge_context,
-                          &max_covering_tombstone_seq, ReadOptions());
+  found = list.current()->Get(LookupKey("key2", seq), &value,
+                              /*timestamp*/nullptr, &s, &merge_context,
+                              &max_covering_tombstone_seq, ReadOptions());
  ASSERT_FALSE(found);

  merge_context.Clear();
-  found =
-      list.current()->Get(LookupKey("key3", seq), &value, &s, &merge_context,
-                          &max_covering_tombstone_seq, ReadOptions());
+  found = list.current()->Get(LookupKey("key3", seq), &value,
+                              /*timestamp*/nullptr, &s, &merge_context,
+                              &max_covering_tombstone_seq, ReadOptions());
  ASSERT_FALSE(found);

  // Verify that the second memtable's keys are in the history
  merge_context.Clear();
  found = list.current()->GetFromHistory(
-      LookupKey("key1", seq), &value, &s, &merge_context,
+      LookupKey("key1", seq), &value, /*timestamp*/nullptr, &s, &merge_context,
      &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(found && s.IsNotFound());

  merge_context.Clear();
  found = list.current()->GetFromHistory(
-      LookupKey("key3", seq), &value, &s, &merge_context,
+      LookupKey("key3", seq), &value, /*timestamp*/nullptr, &s, &merge_context,
      &max_covering_tombstone_seq, ReadOptions());
  ASSERT_TRUE(found);
  ASSERT_EQ("value3", value);

  // Verify that key2 from the first memtable is no longer in the history
  merge_context.Clear();
-  found =
-      list.current()->Get(LookupKey("key2", seq), &value, &s, &merge_context,
-                          &max_covering_tombstone_seq, ReadOptions());
+  found = list.current()->Get(LookupKey("key2", seq), &value,
+                              /*timestamp*/nullptr, &s, &merge_context,
+                              &max_covering_tombstone_seq, ReadOptions());
  ASSERT_FALSE(found);

  // Cleanup
@@ -1275,7 +1275,7 @@ static std::string RandomString(Random* rnd, int len) {

 TEST_P(PlainTableDBTest, CompactionTrigger) {
  Options options = CurrentOptions();
-  options.write_buffer_size = 120 << 10;  // 100KB
+  options.write_buffer_size = 120 << 10;  // 120KB
  options.num_levels = 3;
  options.level0_file_num_compaction_trigger = 3;
  Reopen(&options);
@@ -1287,7 +1287,7 @@ TEST_P(PlainTableDBTest, CompactionTrigger) {
    std::vector<std::string> values;
    // Write 120KB (10 values, each 12K)
    for (int i = 0; i < 10; i++) {
-      values.push_back(RandomString(&rnd, 12000));
+      values.push_back(RandomString(&rnd, 12 << 10));
      ASSERT_OK(Put(Key(i), values[i]));
    }
    ASSERT_OK(Put(Key(999), ""));
@@ -405,7 +405,7 @@ class VersionBuilder::Rep {
    size_t max_load = port::kMaxSizet;

    if (!always_load) {
-      // If it is initial loading and not set to always laoding all the
+      // If it is initial loading and not set to always loading all the
      // files, we only load up to kInitialLoadLimit files, to limit the
      // time reopening the DB.
      const size_t kInitialLoadLimit = 16;
@@ -507,16 +507,7 @@ VersionBuilder::VersionBuilder(const FileOptions& file_options,
                               Logger* info_log)
    : rep_(new Rep(file_options, info_log, table_cache, base_vstorage)) {}

-VersionBuilder::~VersionBuilder() { delete rep_; }
-
-Status VersionBuilder::CheckConsistency(VersionStorageInfo* vstorage) {
-  return rep_->CheckConsistency(vstorage);
-}
-
-Status VersionBuilder::CheckConsistencyForDeletes(VersionEdit* edit,
-                                                  uint64_t number, int level) {
-  return rep_->CheckConsistencyForDeletes(edit, number, level);
-}
+VersionBuilder::~VersionBuilder() = default;

 bool VersionBuilder::CheckConsistencyForNumLevels() {
  return rep_->CheckConsistencyForNumLevels();
@@ -537,9 +528,4 @@ Status VersionBuilder::LoadTableHandlers(
                                 is_initial_load, prefix_extractor);
 }

-void VersionBuilder::MaybeAddFile(VersionStorageInfo* vstorage, int level,
-                                  FileMetaData* f) {
-  rep_->MaybeAddFile(vstorage, level, f);
-}
-
 }  // namespace ROCKSDB_NAMESPACE
@@ -8,6 +8,9 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 //
 #pragma once
+
+#include <memory>
+
 #include "rocksdb/file_system.h"
 #include "rocksdb/slice_transform.h"

@@ -27,9 +30,7 @@ class VersionBuilder {
  VersionBuilder(const FileOptions& file_options, TableCache* table_cache,
                 VersionStorageInfo* base_vstorage, Logger* info_log = nullptr);
  ~VersionBuilder();
-  Status CheckConsistency(VersionStorageInfo* vstorage);
-  Status CheckConsistencyForDeletes(VersionEdit* edit, uint64_t number,
-                                    int level);
+
  bool CheckConsistencyForNumLevels();
  Status Apply(VersionEdit* edit);
  Status SaveTo(VersionStorageInfo* vstorage);
@@ -37,11 +38,10 @@ class VersionBuilder {
                           bool prefetch_index_and_filter_in_cache,
                           bool is_initial_load,
                           const SliceTransform* prefix_extractor);
-  void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f);

 private:
  class Rep;
-  Rep* rep_;
+  std::unique_ptr<Rep> rep_;
 };

 extern bool NewestFirstBySeqNo(FileMetaData* a, FileMetaData* b);
@@ -9,7 +9,7 @@

 #include "db/version_edit.h"

-#include "db/blob_index.h"
+#include "db/blob/blob_index.h"
 #include "db/version_set.h"
 #include "logging/event_logger.h"
 #include "rocksdb/slice.h"
@@ -22,8 +22,8 @@ namespace ROCKSDB_NAMESPACE {
 const std::string kUnknownFileChecksum("");
 // The unknown sst file checksum function name.
 const std::string kUnknownFileChecksumFuncName("Unknown");
-// Mask for an identified tag from the future which can be safely ignored.
-const uint32_t kTagSafeIgnoreMask = 1 << 13;
+
+namespace {

 // Tag numbers for serialized VersionEdit.  These numbers are written to
 // disk and should not be changed. The number should be forward compatible so
@@ -40,8 +40,6 @@ enum Tag : uint32_t {
  // 8 was used for large value refs
  kPrevLogNumber = 9,
  kMinLogNumberToKeep = 10,
-  // Ignore-able field
-  kDbId = kTagSafeIgnoreMask + 1,

  // these are new formats divergent from open source leveldb
  kNewFile2 = 100,
@@ -53,26 +51,40 @@ enum Tag : uint32_t {
  kMaxColumnFamily = 203,

  kInAtomicGroup = 300,
+
+  // Mask for an unidentified tag from the future which can be safely ignored.
+  kTagSafeIgnoreMask = 1 << 13,
+
+  // Forward compatible (aka ignorable) records
+  kDbId,
+  kBlobFileAddition,
+  kBlobFileGarbage,
+  kStateUponManifestSwitch,
+  kManifestSwitched,
 };

-enum CustomTag : uint32_t {
+enum NewFileCustomTag : uint32_t {
  kTerminate = 1,  // The end of customized fields
  kNeedCompaction = 2,
-  // Since Manifest is not entirely currently forward-compatible, and the only
-  // forward-compatible part is the CutsomtTag of kNewFile, we currently encode
-  // kMinLogNumberToKeep as part of a CustomTag as a hack. This should be
-  // removed when manifest becomes forward-comptabile.
+  // Since Manifest is not entirely forward-compatible, we currently encode
+  // kMinLogNumberToKeep as part of NewFile as a hack. This should be removed
+  // when manifest becomes forward-comptabile.
  kMinLogNumberToKeepHack = 3,
  kOldestBlobFileNumber = 4,
  kOldestAncesterTime = 5,
  kFileCreationTime = 6,
  kFileChecksum = 7,
  kFileChecksumFuncName = 8,
-  kPathId = 65,
+
+  // If this bit for the custom tag is set, opening DB should fail if
+  // we don't know this field.
+  kCustomTagNonSafeIgnoreMask = 1 << 6,
+
+  // Forward incompatible (aka unignorable) fields
+  kPathId,
 };
-// If this bit for the custom tag is set, opening DB should fail if
-// we don't know this field.
-uint32_t kCustomTagNonSafeIgnoreMask = 1 << 6;
+
+}  // anonymous namespace

 uint64_t PackFileNumberAndPathId(uint64_t number, uint64_t path_id) {
  assert(number <= kFileNumberMask);
@@ -142,12 +154,16 @@ void VersionEdit::Clear() {
  has_last_sequence_ = false;
  deleted_files_.clear();
  new_files_.clear();
+  blob_file_additions_.clear();
+  blob_file_garbages_.clear();
  column_family_ = 0;
  is_column_family_add_ = false;
  is_column_family_drop_ = false;
  column_family_name_.clear();
  is_in_atomic_group_ = false;
  remaining_entries_ = 0;
+  state_upon_manifest_switch_ = false;
+  manifest_switched_ = false;
 }

 bool VersionEdit::EncodeTo(std::string* dst) const {
@@ -217,45 +233,45 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
    //   tag kNeedCompaction:
    //        now only can take one char value 1 indicating need-compaction
    //
-    PutVarint32(dst, CustomTag::kOldestAncesterTime);
+    PutVarint32(dst, NewFileCustomTag::kOldestAncesterTime);
    std::string varint_oldest_ancester_time;
    PutVarint64(&varint_oldest_ancester_time, f.oldest_ancester_time);
    TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintOldestAncesterTime",
                             &varint_oldest_ancester_time);
    PutLengthPrefixedSlice(dst, Slice(varint_oldest_ancester_time));

-    PutVarint32(dst, CustomTag::kFileCreationTime);
+    PutVarint32(dst, NewFileCustomTag::kFileCreationTime);
    std::string varint_file_creation_time;
    PutVarint64(&varint_file_creation_time, f.file_creation_time);
    TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:VarintFileCreationTime",
                             &varint_file_creation_time);
    PutLengthPrefixedSlice(dst, Slice(varint_file_creation_time));

-    PutVarint32(dst, CustomTag::kFileChecksum);
+    PutVarint32(dst, NewFileCustomTag::kFileChecksum);
    PutLengthPrefixedSlice(dst, Slice(f.file_checksum));

-    PutVarint32(dst, CustomTag::kFileChecksumFuncName);
+    PutVarint32(dst, NewFileCustomTag::kFileChecksumFuncName);
    PutLengthPrefixedSlice(dst, Slice(f.file_checksum_func_name));

    if (f.fd.GetPathId() != 0) {
-      PutVarint32(dst, CustomTag::kPathId);
+      PutVarint32(dst, NewFileCustomTag::kPathId);
      char p = static_cast<char>(f.fd.GetPathId());
      PutLengthPrefixedSlice(dst, Slice(&p, 1));
    }
    if (f.marked_for_compaction) {
-      PutVarint32(dst, CustomTag::kNeedCompaction);
+      PutVarint32(dst, NewFileCustomTag::kNeedCompaction);
      char p = static_cast<char>(1);
      PutLengthPrefixedSlice(dst, Slice(&p, 1));
    }
    if (has_min_log_number_to_keep_ && !min_log_num_written) {
-      PutVarint32(dst, CustomTag::kMinLogNumberToKeepHack);
+      PutVarint32(dst, NewFileCustomTag::kMinLogNumberToKeepHack);
      std::string varint_log_number;
      PutFixed64(&varint_log_number, min_log_number_to_keep_);
      PutLengthPrefixedSlice(dst, Slice(varint_log_number));
      min_log_num_written = true;
    }
    if (f.oldest_blob_file_number != kInvalidBlobFileNumber) {
-      PutVarint32(dst, CustomTag::kOldestBlobFileNumber);
+      PutVarint32(dst, NewFileCustomTag::kOldestBlobFileNumber);
      std::string oldest_blob_file_number;
      PutVarint64(&oldest_blob_file_number, f.oldest_blob_file_number);
      PutLengthPrefixedSlice(dst, Slice(oldest_blob_file_number));
@@ -263,7 +279,17 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
    TEST_SYNC_POINT_CALLBACK("VersionEdit::EncodeTo:NewFile4:CustomizeFields",
                             dst);

-    PutVarint32(dst, CustomTag::kTerminate);
+    PutVarint32(dst, NewFileCustomTag::kTerminate);
+  }
+
+  for (const auto& blob_file_addition : blob_file_additions_) {
+    PutVarint32(dst, kBlobFileAddition);
+    blob_file_addition.EncodeTo(dst);
+  }
+
+  for (const auto& blob_file_garbage : blob_file_garbages_) {
+    PutVarint32(dst, kBlobFileGarbage);
+    blob_file_garbage.EncodeTo(dst);
  }

  // 0 is default and does not need to be explicitly written
@@ -271,6 +297,14 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
    PutVarint32Varint32(dst, kColumnFamily, column_family_);
  }

+  if (state_upon_manifest_switch_) {
+    PutVarint32(dst, kStateUponManifestSwitch);
+  }
+
+  if (manifest_switched_) {
+    PutVarint32(dst, kManifestSwitched);
+  }
+
  if (is_column_family_add_) {
    PutVarint32(dst, kColumnFamilyAdd);
    PutLengthPrefixedSlice(dst, Slice(column_family_name_));
@@ -319,9 +353,6 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
  uint64_t file_size = 0;
  SequenceNumber smallest_seqno = 0;
  SequenceNumber largest_seqno = kMaxSequenceNumber;
-  // Since this is the only forward-compatible part of the code, we hack new
-  // extension into this record. When we do, we set this boolean to distinguish
-  // the record from the normal NewFile records.
  if (GetLevel(input, &level, &msg) && GetVarint64(input, &number) &&
      GetVarint64(input, &file_size) && GetInternalKey(input, &f.smallest) &&
      GetInternalKey(input, &f.largest) &&
@@ -571,6 +602,28 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
        break;
      }

+      case kBlobFileAddition: {
+        BlobFileAddition blob_file_addition;
+        const Status s = blob_file_addition.DecodeFrom(&input);
+        if (!s.ok()) {
+          return s;
+        }
+
+        blob_file_additions_.emplace_back(blob_file_addition);
+        break;
+      }
+
+      case kBlobFileGarbage: {
+        BlobFileGarbage blob_file_garbage;
+        const Status s = blob_file_garbage.DecodeFrom(&input);
+        if (!s.ok()) {
+          return s;
+        }
+
+        blob_file_garbages_.emplace_back(blob_file_garbage);
+        break;
+      }
+
      case kColumnFamily:
        if (!GetVarint32(&input, &column_family_)) {
          if (!msg) {
@@ -594,6 +647,14 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
        is_column_family_drop_ = true;
        break;

+      case kStateUponManifestSwitch:
+        state_upon_manifest_switch_ = true;
+        break;
+
+      case kManifestSwitched:
+        manifest_switched_ = true;
+        break;
+
      case kInAtomicGroup:
        is_in_atomic_group_ = true;
        if (!GetVarint32(&input, &remaining_entries_)) {
@@ -700,6 +761,17 @@ std::string VersionEdit::DebugString(bool hex_key) const {
    r.append(" file_checksum_func_name: ");
    r.append(f.file_checksum_func_name);
  }
+
+  for (const auto& blob_file_addition : blob_file_additions_) {
+    r.append("\n  BlobFileAddition: ");
+    r.append(blob_file_addition.DebugString());
+  }
+
+  for (const auto& blob_file_garbage : blob_file_garbages_) {
+    r.append("\n  BlobFileGarbage: ");
+    r.append(blob_file_garbage.DebugString());
+  }
+
  r.append("\n  ColumnFamily: ");
  AppendNumberTo(&r, column_family_);
  if (is_column_family_add_) {
@@ -782,6 +854,34 @@ std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
    jw.EndArray();
  }

+  if (!blob_file_additions_.empty()) {
+    jw << "BlobFileAdditions";
+
+    jw.StartArray();
+
+    for (const auto& blob_file_addition : blob_file_additions_) {
+      jw.StartArrayedObject();
+      jw << blob_file_addition;
+      jw.EndArrayedObject();
+    }
+
+    jw.EndArray();
+  }
+
+  if (!blob_file_garbages_.empty()) {
+    jw << "BlobFileGarbages";
+
+    jw.StartArray();
+
+    for (const auto& blob_file_garbage : blob_file_garbages_) {
+      jw.StartArrayedObject();
+      jw << blob_file_garbage;
+      jw.EndArrayedObject();
+    }
+
+    jw.EndArray();
+  }
+
  jw << "ColumnFamily" << column_family_;

  if (is_column_family_add_) {
@@ -13,6 +13,8 @@
 #include <string>
 #include <utility>
 #include <vector>
+#include "db/blob/blob_file_addition.h"
+#include "db/blob/blob_file_garbage.h"
 #include "db/dbformat.h"
 #include "memory/arena.h"
 #include "rocksdb/cache.h"
@@ -24,7 +26,6 @@ namespace ROCKSDB_NAMESPACE {
 class VersionSet;

 constexpr uint64_t kFileNumberMask = 0x3FFFFFFFFFFFFFFF;
-constexpr uint64_t kInvalidBlobFileNumber = 0;
 constexpr uint64_t kUnknownOldestAncesterTime = 0;
 constexpr uint64_t kUnknownFileCreationTime = 0;

@@ -307,16 +308,16 @@ class VersionEdit {
  bool HasLastSequence() const { return has_last_sequence_; }
  SequenceNumber GetLastSequence() const { return last_sequence_; }

-  // Delete the specified "file" from the specified "level".
+  // Delete the specified table file from the specified level.
  void DeleteFile(int level, uint64_t file) {
    deleted_files_.emplace(level, file);
  }

-  // Retrieve the files deleted as well as their associated levels.
+  // Retrieve the table files deleted as well as their associated levels.
  using DeletedFiles = std::set<std::pair<int, uint64_t>>;
  const DeletedFiles& GetDeletedFiles() const { return deleted_files_; }

-  // Add the specified file at the specified level.
+  // Add the specified table file at the specified level.
  // REQUIRES: This version has not been saved (see VersionSet::SaveTo)
  // REQUIRES: "smallest" and "largest" are smallest and largest keys in file
  // REQUIRES: "oldest_blob_file_number" is the number of the oldest blob file
@@ -342,12 +343,45 @@ class VersionEdit {
    new_files_.emplace_back(level, f);
  }

-  // Retrieve the files added as well as their associated levels.
+  // Retrieve the table files added as well as their associated levels.
  using NewFiles = std::vector<std::pair<int, FileMetaData>>;
  const NewFiles& GetNewFiles() const { return new_files_; }

+  // Add a new blob file.
+  void AddBlobFile(uint64_t blob_file_number, uint64_t total_blob_count,
+                   uint64_t total_blob_bytes, std::string checksum_method,
+                   std::string checksum_value) {
+    blob_file_additions_.emplace_back(
+        blob_file_number, total_blob_count, total_blob_bytes,
+        std::move(checksum_method), std::move(checksum_value));
+  }
+
+  // Retrieve all the blob files added.
+  using BlobFileAdditions = std::vector<BlobFileAddition>;
+  const BlobFileAdditions& GetBlobFileAdditions() const {
+    return blob_file_additions_;
+  }
+
+  // Add garbage for an existing blob file.  Note: intentionally broken English
+  // follows.
+  void AddBlobFileGarbage(uint64_t blob_file_number,
+                          uint64_t garbage_blob_count,
+                          uint64_t garbage_blob_bytes) {
+    blob_file_garbages_.emplace_back(blob_file_number, garbage_blob_count,
+                                     garbage_blob_bytes);
+  }
+
+  // Retrieve all the blob file garbage added.
+  using BlobFileGarbages = std::vector<BlobFileGarbage>;
+  const BlobFileGarbages& GetBlobFileGarbages() const {
+    return blob_file_garbages_;
+  }
+
  // Number of edits
-  size_t NumEntries() const { return new_files_.size() + deleted_files_.size(); }
+  size_t NumEntries() const {
+    return new_files_.size() + deleted_files_.size() +
+           blob_file_additions_.size() + blob_file_garbages_.size();
+  }

  void SetColumnFamily(uint32_t column_family_id) {
    column_family_ = column_family_id;
@@ -389,6 +423,19 @@ class VersionEdit {
  std::string DebugString(bool hex_key = false) const;
  std::string DebugJSON(int edit_num, bool hex_key = false) const;

+  void SetStateUponManifestSwitch(bool tag) {
+    state_upon_manifest_switch_ = tag;
+  }
+  bool GetStateUponManifestSwitch() const {
+    return state_upon_manifest_switch_;
+  }
+  void SetManifestSwitched(bool tag) {
+    manifest_switched_ = tag;
+  }
+  bool GetManifestSwitched() const {
+    return manifest_switched_;
+  }
+
 private:
  friend class ReactiveVersionSet;
  friend class VersionSet;
@@ -421,6 +468,9 @@ class VersionEdit {
  DeletedFiles deleted_files_;
  NewFiles new_files_;

+  BlobFileAdditions blob_file_additions_;
+  BlobFileGarbages blob_file_garbages_;
+
  // Each version edit record should have column_family_ set
  // If it's not set, it is default (0)
  uint32_t column_family_ = 0;
@@ -433,6 +483,13 @@ class VersionEdit {

  bool is_in_atomic_group_ = false;
  uint32_t remaining_entries_ = 0;
+  // To distinguish the version edit written by WriteCurrentStateToManifest
+  // and other regular writes. Default is false.
+  bool state_upon_manifest_switch_ = false;
+  // To indicate when WriteCurrentStateToManifest is successful. When both
+  // state_upon_manifest_switch and manifest_switch_finished are true, it can
+  // ensure the manifest switch is finished.
+  bool manifest_switched_ = false;
 };

 }  // namespace ROCKSDB_NAMESPACE
@@ -11,6 +11,7 @@
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
 #include "util/coding.h"
+#include "util/string_util.h"

 namespace ROCKSDB_NAMESPACE {

@@ -45,6 +46,7 @@ TEST_F(VersionEditTest, EncodeDecode) {
  edit.SetLogNumber(kBig + 100);
  edit.SetNextFile(kBig + 200);
  edit.SetLastSequence(kBig + 1000);
+  edit.SetStateUponManifestSwitch(true);
  TestEncodeDecode(edit);
 }

@@ -79,6 +81,7 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
  edit.SetLogNumber(kBig + 100);
  edit.SetNextFile(kBig + 200);
  edit.SetLastSequence(kBig + 1000);
+  edit.SetStateUponManifestSwitch(true);
  TestEncodeDecode(edit);

  std::string encoded, encoded2;
@@ -102,6 +105,7 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
  ASSERT_EQ(kInvalidBlobFileNumber,
            new_files[2].second.oldest_blob_file_number);
  ASSERT_EQ(1001, new_files[3].second.oldest_blob_file_number);
+  ASSERT_TRUE(parsed.GetStateUponManifestSwitch());
 }

 TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
@@ -278,6 +282,68 @@ TEST_F(VersionEditTest, DbId) {
  TestEncodeDecode(edit);
 }

+TEST_F(VersionEditTest, ManifestSwitchTag) {
+  VersionEdit edit1, decode1;
+  edit1.SetStateUponManifestSwitch(true);
+  TestEncodeDecode(edit1);
+  std::string encoded1;
+  edit1.EncodeTo(&encoded1);
+  ASSERT_OK(decode1.DecodeFrom(encoded1));
+  ASSERT_TRUE(decode1.GetStateUponManifestSwitch());
+  ASSERT_TRUE(!decode1.GetManifestSwitched());
+
+  VersionEdit edit2, decode2;
+  edit2.SetManifestSwitched(true);
+  TestEncodeDecode(edit2);
+  std::string encoded2;
+  edit2.EncodeTo(&encoded2);
+  ASSERT_OK(decode2.DecodeFrom(encoded2));
+  ASSERT_TRUE(!decode2.GetStateUponManifestSwitch());
+  ASSERT_TRUE(decode2.GetManifestSwitched());
+
+  VersionEdit edit3, decode3;
+  edit3.SetStateUponManifestSwitch(true);
+  edit3.SetManifestSwitched(true);
+  TestEncodeDecode(edit3);
+  std::string encoded3;
+  edit3.EncodeTo(&encoded3);
+  ASSERT_OK(decode3.DecodeFrom(encoded3));
+  ASSERT_TRUE(decode3.GetStateUponManifestSwitch());
+  ASSERT_TRUE(decode3.GetManifestSwitched());
+
+
+}
+
+TEST_F(VersionEditTest, BlobFileAdditionAndGarbage) {
+  VersionEdit edit;
+
+  const std::string checksum_method_prefix = "Hash";
+  const std::string checksum_value_prefix = "Value";
+
+  for (uint64_t blob_file_number = 1; blob_file_number <= 10;
+       ++blob_file_number) {
+    const uint64_t total_blob_count = blob_file_number << 10;
+    const uint64_t total_blob_bytes = blob_file_number << 20;
+
+    std::string checksum_method(checksum_method_prefix);
+    AppendNumberTo(&checksum_method, blob_file_number);
+
+    std::string checksum_value(checksum_value_prefix);
+    AppendNumberTo(&checksum_value, blob_file_number);
+
+    edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
+                     checksum_method, checksum_value);
+
+    const uint64_t garbage_blob_count = total_blob_count >> 2;
+    const uint64_t garbage_blob_bytes = total_blob_bytes >> 1;
+
+    edit.AddBlobFileGarbage(blob_file_number, garbage_blob_count,
+                            garbage_blob_bytes);
+  }
+
+  TestEncodeDecode(edit);
+}
+
 }  // namespace ROCKSDB_NAMESPACE

 int main(int argc, char** argv) {
@@ -955,8 +955,8 @@ class LevelIterator final : public InternalIterator {
  bool KeyReachedUpperBound(const Slice& internal_key) {
    return read_options_.iterate_upper_bound != nullptr &&
           user_comparator_.CompareWithoutTimestamp(
-               ExtractUserKey(internal_key),
-               *read_options_.iterate_upper_bound) >= 0;
+               ExtractUserKey(internal_key), /*a_has_ts=*/true,
+               *read_options_.iterate_upper_bound, /*b_has_ts=*/false) >= 0;
  }

  InternalIterator* NewFileIterator() {
@@ -1754,7 +1754,7 @@ Version::Version(ColumnFamilyData* column_family_data, VersionSet* vset,
      version_number_(version_number) {}

 void Version::Get(const ReadOptions& read_options, const LookupKey& k,
-                  PinnableSlice* value, Status* status,
+                  PinnableSlice* value, std::string* timestamp, Status* status,
                  MergeContext* merge_context,
                  SequenceNumber* max_covering_tombstone_seq, bool* value_found,
                  bool* key_exists, SequenceNumber* seq, ReadCallback* callback,
@@ -1778,8 +1778,8 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
  GetContext get_context(
      user_comparator(), merge_operator_, info_log_, db_statistics_,
      status->ok() ? GetContext::kNotFound : GetContext::kMerge, user_key,
-      do_merge ? value : nullptr, value_found, merge_context, do_merge,
-      max_covering_tombstone_seq, this->env_, seq,
+      do_merge ? value : nullptr, do_merge ? timestamp : nullptr, value_found,
+      merge_context, do_merge, max_covering_tombstone_seq, this->env_, seq,
      merge_operator_ ? &pinned_iters_mgr : nullptr, callback, is_blob,
      tracing_get_id);

@@ -1918,8 +1918,8 @@ void Version::MultiGet(const ReadOptions& read_options, MultiGetRange* range,
    get_ctx.emplace_back(
        user_comparator(), merge_operator_, info_log_, db_statistics_,
        iter->s->ok() ? GetContext::kNotFound : GetContext::kMerge, iter->ukey,
-        iter->value, nullptr, &(iter->merge_context), true,
-        &iter->max_covering_tombstone_seq, this->env_, nullptr,
+        iter->value, /*timestamp*/ nullptr, nullptr, &(iter->merge_context),
+        true, &iter->max_covering_tombstone_seq, this->env_, nullptr,
        merge_operator_ ? &pinned_iters_mgr : nullptr, callback, is_blob,
        tracing_mget_id);
    // MergeInProgress status, if set, has been transferred to the get_context
@@ -3600,7 +3600,7 @@ void VersionSet::AppendVersion(ColumnFamilyData* column_family_data,

 Status VersionSet::ProcessManifestWrites(
    std::deque<ManifestWriter>& writers, InstrumentedMutex* mu,
-    Directory* db_directory, bool new_descriptor_log,
+    FSDirectory* db_directory, bool new_descriptor_log,
    const ColumnFamilyOptions* new_cf_options) {
  assert(!writers.empty());
  ManifestWriter& first_writer = writers.front();
@@ -4026,7 +4026,7 @@ Status VersionSet::LogAndApply(
    const autovector<ColumnFamilyData*>& column_family_datas,
    const autovector<const MutableCFOptions*>& mutable_cf_options_list,
    const autovector<autovector<VersionEdit*>>& edit_lists,
-    InstrumentedMutex* mu, Directory* db_directory, bool new_descriptor_log,
+    InstrumentedMutex* mu, FSDirectory* db_directory, bool new_descriptor_log,
    const ColumnFamilyOptions* new_cf_options) {
  mu->AssertHeld();
  int num_edits = 0;
@@ -5003,6 +5003,7 @@ Status VersionSet::WriteCurrentStateToManifest(
    VersionEdit edit_for_db_id;
    assert(!db_id_.empty());
    edit_for_db_id.SetDBId(db_id_);
+    edit_for_db_id.SetStateUponManifestSwitch(true);
    std::string db_id_record;
    if (!edit_for_db_id.EncodeTo(&db_id_record)) {
      return Status::Corruption("Unable to Encode VersionEdit:" +
@@ -5028,6 +5029,7 @@ Status VersionSet::WriteCurrentStateToManifest(
        edit.AddColumnFamily(cfd->GetName());
        edit.SetColumnFamily(cfd->GetID());
      }
+      edit.SetStateUponManifestSwitch(true);
      edit.SetComparatorName(
          cfd->internal_comparator().user_comparator()->Name());
      std::string record;
@@ -5045,6 +5047,7 @@ Status VersionSet::WriteCurrentStateToManifest(
      // Save files
      VersionEdit edit;
      edit.SetColumnFamily(cfd->GetID());
+      edit.SetStateUponManifestSwitch(true);

      for (int level = 0; level < cfd->NumberLevels(); level++) {
        for (const auto& f :
@@ -5072,7 +5075,16 @@ Status VersionSet::WriteCurrentStateToManifest(
      }
    }
  }
-  return Status::OK();
+  VersionEdit end_flag;
+  end_flag.SetStateUponManifestSwitch(true);
+  end_flag.SetManifestSwitched(true);
+  std::string end_record;
+  if (!end_flag.EncodeTo(&end_record)) {
+    return Status::Corruption("Unable to Encode VersionEdit:" +
+                              end_flag.DebugString(true));
+  }
+  Status s_end_record = log->AddRecord(end_record);
+  return  s_end_record;
 }

 // TODO(aekmekji): in CompactionJob::GenSubcompactionBoundaries(), this
@@ -595,7 +595,7 @@ class Version {
  //    merge_context.operands_list and don't merge the operands
  // REQUIRES: lock is not held
  void Get(const ReadOptions&, const LookupKey& key, PinnableSlice* value,
-           Status* status, MergeContext* merge_context,
+           std::string* timestamp, Status* status, MergeContext* merge_context,
           SequenceNumber* max_covering_tombstone_seq,
           bool* value_found = nullptr, bool* key_exists = nullptr,
           SequenceNumber* seq = nullptr, ReadCallback* callback = nullptr,
@@ -822,7 +822,7 @@ class VersionSet {
  Status LogAndApply(
      ColumnFamilyData* column_family_data,
      const MutableCFOptions& mutable_cf_options, VersionEdit* edit,
-      InstrumentedMutex* mu, Directory* db_directory = nullptr,
+      InstrumentedMutex* mu, FSDirectory* db_directory = nullptr,
      bool new_descriptor_log = false,
      const ColumnFamilyOptions* column_family_options = nullptr) {
    autovector<ColumnFamilyData*> cfds;
@@ -842,7 +842,7 @@ class VersionSet {
      ColumnFamilyData* column_family_data,
      const MutableCFOptions& mutable_cf_options,
      const autovector<VersionEdit*>& edit_list, InstrumentedMutex* mu,
-      Directory* db_directory = nullptr, bool new_descriptor_log = false,
+      FSDirectory* db_directory = nullptr, bool new_descriptor_log = false,
      const ColumnFamilyOptions* column_family_options = nullptr) {
    autovector<ColumnFamilyData*> cfds;
    cfds.emplace_back(column_family_data);
@@ -861,7 +861,7 @@ class VersionSet {
      const autovector<ColumnFamilyData*>& cfds,
      const autovector<const MutableCFOptions*>& mutable_cf_options_list,
      const autovector<autovector<VersionEdit*>>& edit_lists,
-      InstrumentedMutex* mu, Directory* db_directory = nullptr,
+      InstrumentedMutex* mu, FSDirectory* db_directory = nullptr,
      bool new_descriptor_log = false,
      const ColumnFamilyOptions* new_cf_options = nullptr);

@@ -1170,7 +1170,7 @@ class VersionSet {
 private:
  // REQUIRES db mutex at beginning. may release and re-acquire db mutex
  Status ProcessManifestWrites(std::deque<ManifestWriter>& writers,
-                               InstrumentedMutex* mu, Directory* db_directory,
+                               InstrumentedMutex* mu, FSDirectory* db_directory,
                               bool new_descriptor_log,
                               const ColumnFamilyOptions* new_cf_options);

@@ -1237,7 +1237,7 @@ class ReactiveVersionSet : public VersionSet {
      const autovector<ColumnFamilyData*>& /*cfds*/,
      const autovector<const MutableCFOptions*>& /*mutable_cf_options_list*/,
      const autovector<autovector<VersionEdit*>>& /*edit_lists*/,
-      InstrumentedMutex* /*mu*/, Directory* /*db_directory*/,
+      InstrumentedMutex* /*mu*/, FSDirectory* /*db_directory*/,
      bool /*new_descriptor_log*/,
      const ColumnFamilyOptions* /*new_cf_option*/) override {
    return Status::NotSupported("not supported in reactive mode");
@@ -464,7 +464,7 @@ size_t WriteThread::EnterAsBatchGroupLeader(Writer* leader,
    }

    if (w->callback != nullptr && !w->callback->AllowWriteBatching()) {
-      // dont batch writes that don't want to be batched
+      // don't batch writes that don't want to be batched
      break;
    }

@@ -128,7 +128,9 @@ DECLARE_int32(universal_min_merge_width);
 DECLARE_int32(universal_max_merge_width);
 DECLARE_int32(universal_max_size_amplification_percent);
 DECLARE_int32(clear_column_family_one_in);
-DECLARE_int32(get_live_files_and_wal_files_one_in);
+DECLARE_int32(get_live_files_one_in);
+DECLARE_int32(get_sorted_wal_files_one_in);
+DECLARE_int32(get_current_wal_file_one_in);
 DECLARE_int32(set_options_one_in);
 DECLARE_int32(set_in_place_one_in);
 DECLARE_int64(cache_size);
@@ -167,6 +169,8 @@ DECLARE_double(max_bytes_for_level_multiplier);
 DECLARE_int32(range_deletion_width);
 DECLARE_uint64(rate_limiter_bytes_per_sec);
 DECLARE_bool(rate_limit_bg_reads);
+DECLARE_uint64(sst_file_manager_bytes_per_sec);
+DECLARE_uint64(sst_file_manager_bytes_per_truncate);
 DECLARE_bool(use_txn);
 DECLARE_uint64(txn_write_policy);
 DECLARE_bool(unordered_write);
@@ -256,10 +256,21 @@ DEFINE_int32(clear_column_family_one_in, 1000000,
             "it again. If N == 0, never drop/create column families. "
             "When test_batches_snapshots is true, this flag has no effect");

-DEFINE_int32(get_live_files_and_wal_files_one_in, 1000000,
-             "With a chance of 1/N, call GetLiveFiles, GetSortedWalFiles "
-             "and GetCurrentWalFile to verify if it returns correctly. If "
-             "N == 0, never call the three interfaces.");
+DEFINE_int32(get_live_files_one_in, 1000000,
+             "With a chance of 1/N, call GetLiveFiles to verify if it returns "
+             "correctly. If N == 0, do not call the interface.");
+
+DEFINE_int32(
+    get_sorted_wal_files_one_in, 1000000,
+    "With a chance of 1/N, call GetSortedWalFiles to verify if it returns "
+    "correctly. (Note that this API may legitimately return an error.) If N == "
+    "0, do not call the interface.");
+
+DEFINE_int32(
+    get_current_wal_file_one_in, 1000000,
+    "With a chance of 1/N, call GetCurrentWalFile to verify if it returns "
+    "correctly. (Note that this API may legitimately return an error.) If N == "
+    "0, do not call the interface.");

 DEFINE_int32(set_options_one_in, 0,
             "With a chance of 1/N, change some random options");
@@ -430,6 +441,14 @@ DEFINE_uint64(rate_limiter_bytes_per_sec, 0, "Set options.rate_limiter value.");
 DEFINE_bool(rate_limit_bg_reads, false,
            "Use options.rate_limiter on compaction reads");

+DEFINE_uint64(sst_file_manager_bytes_per_sec, 0,
+              "Set `Options::sst_file_manager` to delete at this rate. By "
+              "default the deletion rate is unbounded.");
+
+DEFINE_uint64(sst_file_manager_bytes_per_truncate, 0,
+              "Set `Options::sst_file_manager` to delete in chunks of this "
+              "many bytes. By default whole files will be deleted.");
+
 DEFINE_bool(use_txn, false,
            "Use TransactionDB. Currently the default write policy is "
            "TxnDBWritePolicy::WRITE_PREPARED");
@@ -12,6 +12,7 @@
 #include "db_stress_tool/db_stress_common.h"
 #include "db_stress_tool/db_stress_driver.h"
 #include "rocksdb/convenience.h"
+#include "rocksdb/sst_file_manager.h"

 namespace ROCKSDB_NAMESPACE {
 StressTest::StressTest()
@@ -40,6 +41,7 @@ StressTest::StressTest()
    }

    Options options;
+    options.env = db_stress_env;
    // Remove files without preserving manfiest files
 #ifndef ROCKSDB_LITE
    const Status s = !FLAGS_use_blob_db
@@ -591,13 +593,28 @@ void StressTest::OperateDb(ThreadState* thread) {
      }

 #ifndef ROCKSDB_LITE
-      // Every 1 in N verify the one of the following: 1) GetLiveFiles
-      // 2) GetSortedWalFiles 3) GetCurrentWalFile. Each time, randomly select
-      // one of them to run the test.
-      if (thread->rand.OneInOpt(FLAGS_get_live_files_and_wal_files_one_in)) {
-        Status status = VerifyGetLiveAndWalFiles(thread);
+      // Verify GetLiveFiles with a 1 in N chance.
+      if (thread->rand.OneInOpt(FLAGS_get_live_files_one_in)) {
+        Status status = VerifyGetLiveFiles();
        if (!status.ok()) {
-          VerificationAbort(shared, "VerifyGetLiveAndWalFiles status not OK",
+          VerificationAbort(shared, "VerifyGetLiveFiles status not OK", status);
+        }
+      }
+
+      // Verify GetSortedWalFiles with a 1 in N chance.
+      if (thread->rand.OneInOpt(FLAGS_get_sorted_wal_files_one_in)) {
+        Status status = VerifyGetSortedWalFiles();
+        if (!status.ok()) {
+          VerificationAbort(shared, "VerifyGetSortedWalFiles status not OK",
+                            status);
+        }
+      }
+
+      // Verify GetCurrentWalFile with a 1 in N chance.
+      if (thread->rand.OneInOpt(FLAGS_get_current_wal_file_one_in)) {
+        Status status = VerifyGetCurrentWalFile();
+        if (!status.ok()) {
+          VerificationAbort(shared, "VerifyGetCurrentWalFile status not OK",
                            status);
        }
      }
@@ -976,28 +993,23 @@ Status StressTest::TestIterate(ThreadState* thread,
 }

 #ifndef ROCKSDB_LITE
-// Test the return status of GetLiveFiles, GetSortedWalFiles, and
-// GetCurrentWalFile. Each time, randomly select one of them to run
-// and return the status.
-Status StressTest::VerifyGetLiveAndWalFiles(ThreadState* thread) {
-  int case_num = thread->rand.Uniform(3);
-  if (case_num == 0) {
-    std::vector<std::string> live_file;
-    uint64_t manifest_size;
-    return db_->GetLiveFiles(live_file, &manifest_size);
-  }
+// Test the return status of GetLiveFiles.
+Status StressTest::VerifyGetLiveFiles() const {
+  std::vector<std::string> live_file;
+  uint64_t manifest_size = 0;
+  return db_->GetLiveFiles(live_file, &manifest_size);
+}

-  if (case_num == 1) {
-    VectorLogPtr log_ptr;
-    return db_->GetSortedWalFiles(log_ptr);
-  }
+// Test the return status of GetSortedWalFiles.
+Status StressTest::VerifyGetSortedWalFiles() const {
+  VectorLogPtr log_ptr;
+  return db_->GetSortedWalFiles(log_ptr);
+}

-  if (case_num == 2) {
-    std::unique_ptr<LogFile> cur_wal_file;
-    return db_->GetCurrentWalFile(&cur_wal_file);
-  }
-  assert(false);
-  return Status::Corruption("Undefined case happens!");
+// Test the return status of GetCurrentWalFile.
+Status StressTest::VerifyGetCurrentWalFile() const {
+  std::unique_ptr<LogFile> cur_wal_file;
+  return db_->GetCurrentWalFile(&cur_wal_file);
 }
 #endif  // !ROCKSDB_LITE

@@ -1839,6 +1851,21 @@ void StressTest::Open() {
      options_.new_table_reader_for_compaction_inputs = true;
    }
  }
+  if (FLAGS_sst_file_manager_bytes_per_sec > 0 ||
+      FLAGS_sst_file_manager_bytes_per_truncate > 0) {
+    Status status;
+    options_.sst_file_manager.reset(NewSstFileManager(
+        db_stress_env, options_.info_log, "" /* trash_dir */,
+        static_cast<int64_t>(FLAGS_sst_file_manager_bytes_per_sec),
+        true /* delete_existing_trash */, &status,
+        0.25 /* max_trash_db_ratio */,
+        FLAGS_sst_file_manager_bytes_per_truncate));
+    if (!status.ok()) {
+      fprintf(stderr, "SstFileManager creation failed: %s\n",
+              status.ToString().c_str());
+      exit(1);
+    }
+  }

  if (FLAGS_prefix_size == 0 && FLAGS_rep_factory == kHashSkipList) {
    fprintf(stderr,
@@ -184,7 +184,10 @@ class StressTest {

  Status MaybeReleaseSnapshots(ThreadState* thread, uint64_t i);
 #ifndef ROCKSDB_LITE
-  Status VerifyGetLiveAndWalFiles(ThreadState* thread);
+  Status VerifyGetLiveFiles() const;
+  Status VerifyGetSortedWalFiles() const;
+  Status VerifyGetCurrentWalFile() const;
+
  virtual Status TestApproximateSize(
      ThreadState* thread, uint64_t iteration,
      const std::vector<int>& rand_column_families,
@@ -35,8 +35,11 @@ def test_binary(
        external_deps = rocksdb_external_deps,
    )

+    binary_path = "$(location :{})".format(test_bin)
+
    custom_unittest(
        name = test_name,
-        command = [TEST_RUNNER, "$(location :{})".format(test_bin)],
+        command = [TEST_RUNNER, binary_path],
        type = ttype,
+        env = {"BUCK_BASE_BINARY": binary_path},
    )
@@ -300,6 +300,13 @@ class CompositeEnvWrapper : public Env {

  FileSystem* fs_env_target() const { return fs_env_target_; }

+  Status RegisterDbPaths(const std::vector<std::string>& paths) override {
+    return fs_env_target_->RegisterDbPaths(paths);
+  }
+  Status UnregisterDbPaths(const std::vector<std::string>& paths) override {
+    return fs_env_target_->UnregisterDbPaths(paths);
+  }
+
  // The following text is boilerplate that forwards all methods to target()
  Status NewSequentialFile(const std::string& f,
                           std::unique_ptr<SequentialFile>* r,
@@ -38,6 +38,32 @@ class ChrootEnv : public EnvWrapper {
 #endif
  }

+  Status RegisterDbPaths(const std::vector<std::string>& paths) override {
+    std::vector<std::string> encoded_paths;
+    encoded_paths.reserve(paths.size());
+    for (auto& path : paths) {
+      auto status_and_enc_path = EncodePathWithNewBasename(path);
+      if (!status_and_enc_path.first.ok()) {
+        return status_and_enc_path.first;
+      }
+      encoded_paths.emplace_back(status_and_enc_path.second);
+    }
+    return EnvWrapper::Env::RegisterDbPaths(encoded_paths);
+  }
+
+  Status UnregisterDbPaths(const std::vector<std::string>& paths) override {
+    std::vector<std::string> encoded_paths;
+    encoded_paths.reserve(paths.size());
+    for (auto& path : paths) {
+      auto status_and_enc_path = EncodePathWithNewBasename(path);
+      if (!status_and_enc_path.first.ok()) {
+        return status_and_enc_path.first;
+      }
+      encoded_paths.emplace_back(status_and_enc_path.second);
+    }
+    return EnvWrapper::Env::UnregisterDbPaths(encoded_paths);
+  }
+
  Status NewSequentialFile(const std::string& fname,
                           std::unique_ptr<SequentialFile>* result,
                           const EnvOptions& options) override {
@@ -1180,6 +1180,99 @@ TEST_P(EnvPosixTestWithParam, MultiRead) {
  }
 }

+TEST_F(EnvPosixTest, MultiReadNonAlignedLargeNum) {
+  // In this test we don't do aligned read, wo it doesn't work for
+  // direct I/O case.
+  EnvOptions soptions;
+  soptions.use_direct_reads = soptions.use_direct_writes = false;
+  std::string fname = test::PerThreadDBPath(env_, "testfile");
+
+  const size_t kTotalSize = 81920;
+  std::string expected_data;
+  Random rnd(301);
+  test::RandomString(&rnd, kTotalSize, &expected_data);
+
+  // Create file.
+  {
+    std::unique_ptr<WritableFile> wfile;
+    ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions));
+    ASSERT_OK(wfile->Append(expected_data));
+    ASSERT_OK(wfile->Close());
+  }
+
+  // More attempts to simulate more partial result sequences.
+  for (uint32_t attempt = 0; attempt < 25; attempt++) {
+    // Right now kIoUringDepth is hard coded as 256, so we need very large
+    // number of keys to cover the case of multiple rounds of submissions.
+    // Right now the test latency is still acceptable. If it ends up with
+    // too long, we can modify the io uring depth with SyncPoint here.
+    const int num_reads = rnd.Uniform(512) + 1;
+
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
+        "PosixRandomAccessFile::MultiRead:io_uring_result", [&](void* arg) {
+          if (attempt > 5) {
+            // Improve partial result rates in second half of the run to
+            // cover the case of repeated partial results.
+            int odd = (attempt < 15) ? num_reads / 2 : 4;
+            // No failure in first several attempts.
+            size_t& bytes_read = *static_cast<size_t*>(arg);
+            if (rnd.OneIn(odd)) {
+              bytes_read = 0;
+            } else if (rnd.OneIn(odd / 2)) {
+              bytes_read = static_cast<size_t>(
+                  rnd.Uniform(static_cast<int>(bytes_read)));
+            }
+          }
+        });
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+    // Generate (offset, len) pairs
+    std::set<int> start_offsets;
+    for (int i = 0; i < num_reads; i++) {
+      int rnd_off;
+      // No repeat offsets.
+      while (start_offsets.find(rnd_off = rnd.Uniform(81920)) != start_offsets.end()) {}
+      start_offsets.insert(rnd_off);
+    }
+    std::vector<size_t> offsets;
+    std::vector<size_t> lens;
+    // std::set already sorted the offsets.
+    for (int so: start_offsets) {
+      offsets.push_back(so);
+    }
+    for (size_t i = 0; i < offsets.size() - 1; i++) {
+      lens.push_back(static_cast<size_t>(rnd.Uniform(static_cast<int>(offsets[i + 1] - offsets[i])) + 1));
+    }
+    lens.push_back(static_cast<size_t>(rnd.Uniform(static_cast<int>(kTotalSize - offsets.back())) + 1));
+    ASSERT_EQ(num_reads, lens.size());
+
+    // Create requests
+    std::vector<std::string> scratches;
+    scratches.reserve(num_reads);
+    std::vector<ReadRequest> reqs(num_reads);
+    for (size_t i = 0; i < reqs.size(); ++i) {
+      reqs[i].offset = offsets[i];
+      reqs[i].len = lens[i];
+      scratches.emplace_back(reqs[i].len, ' ');
+      reqs[i].scratch = const_cast<char*>(scratches.back().data());
+    }
+
+    // Query the data
+    std::unique_ptr<RandomAccessFile> file;
+    ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions));
+    ASSERT_OK(file->MultiRead(reqs.data(), reqs.size()));
+
+    // Validate results
+    for (int i = 0; i < num_reads; ++i) {
+      ASSERT_OK(reqs[i].status);
+      ASSERT_EQ(Slice(expected_data.data() + offsets[i], lens[i]).ToString(true),
+                reqs[i].result.ToString(true));
+    }
+
+    ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
+  }
+}
+  
 // Only works in linux platforms
 #ifdef OS_WIN
 TEST_P(EnvPosixTestWithParam, DISABLED_InvalidateCache) {
@@ -82,8 +82,13 @@ inline mode_t GetDBFileMode(bool allow_non_owner_access) {
 }

 // list of pathnames that are locked
-static std::set<std::string> lockedFiles;
-static port::Mutex mutex_lockedFiles;
+// Only used for error message.
+struct LockHoldingInfo {
+  int64_t acquire_time;
+  uint64_t acquiring_thread;
+};
+static std::map<std::string, LockHoldingInfo> locked_files;
+static port::Mutex mutex_locked_files;

 static int LockOrUnlock(int fd, bool lock) {
  errno = 0;
@@ -178,7 +183,8 @@ class PosixFileSystem : public FileSystem {
                       errno);
      }
    }
-    result->reset(new PosixSequentialFile(fname, file, fd, options));
+    result->reset(new PosixSequentialFile(
+        fname, file, fd, GetLogicalBlockSize(fname, fd), options));
    return IOStatus::OK();
  }

@@ -237,12 +243,13 @@ class PosixFileSystem : public FileSystem {
        }
 #endif
      }
-      result->reset(new PosixRandomAccessFile(fname, fd, options
+      result->reset(new PosixRandomAccessFile(
+          fname, fd, GetLogicalBlockSize(fname, fd), options
 #if defined(ROCKSDB_IOURING_PRESENT)
-                                              ,
-                                              thread_local_io_urings_.get()
+          ,
+          thread_local_io_urings_.get()
 #endif
-                                                  ));
+              ));
    }
    return s;
  }
@@ -323,12 +330,14 @@ class PosixFileSystem : public FileSystem {
        }
      }
 #endif
-      result->reset(new PosixWritableFile(fname, fd, options));
+      result->reset(new PosixWritableFile(
+          fname, fd, GetLogicalBlockSize(fname, fd), options));
    } else {
      // disable mmap writes
      EnvOptions no_mmap_writes_options = options;
      no_mmap_writes_options.use_mmap_writes = false;
-      result->reset(new PosixWritableFile(fname, fd, no_mmap_writes_options));
+      result->reset(new PosixWritableFile(
+          fname, fd, GetLogicalBlockSize(fname, fd), no_mmap_writes_options));
    }
    return s;
  }
@@ -423,12 +432,14 @@ class PosixFileSystem : public FileSystem {
        }
      }
 #endif
-      result->reset(new PosixWritableFile(fname, fd, options));
+      result->reset(new PosixWritableFile(
+          fname, fd, GetLogicalBlockSize(fname, fd), options));
    } else {
      // disable mmap writes
      FileOptions no_mmap_writes_options = options;
      no_mmap_writes_options.use_mmap_writes = false;
-      result->reset(new PosixWritableFile(fname, fd, no_mmap_writes_options));
+      result->reset(new PosixWritableFile(
+          fname, fd, GetLogicalBlockSize(fname, fd), no_mmap_writes_options));
    }
    return s;
  }
@@ -699,9 +710,16 @@ class PosixFileSystem : public FileSystem {
    *lock = nullptr;
    IOStatus result;

-    mutex_lockedFiles.Lock();
-    // If it already exists in the lockedFiles set, then it is already locked,
-    // and fail this lock attempt. Otherwise, insert it into lockedFiles.
+    LockHoldingInfo lhi;
+    int64_t current_time = 0;
+    // Ignore status code as the time is only used for error message.
+    Env::Default()->GetCurrentTime(&current_time);
+    lhi.acquire_time = current_time;
+    lhi.acquiring_thread = Env::Default()->GetThreadID();
+
+    mutex_locked_files.Lock();
+    // If it already exists in the locked_files set, then it is already locked,
+    // and fail this lock attempt. Otherwise, insert it into locked_files.
    // This check is needed because fcntl() does not detect lock conflict
    // if the fcntl is issued by the same thread that earlier acquired
    // this lock.
@@ -709,10 +727,18 @@ class PosixFileSystem : public FileSystem {
    // Otherwise, we will open a new file descriptor. Locks are associated with
    // a process, not a file descriptor and when *any* file descriptor is
    // closed, all locks the process holds for that *file* are released
-    if (lockedFiles.insert(fname).second == false) {
-      mutex_lockedFiles.Unlock();
+    const auto it_success = locked_files.insert({fname, lhi});
+    if (it_success.second == false) {
+      mutex_locked_files.Unlock();
      errno = ENOLCK;
-      return IOError("lock ", fname, errno);
+      LockHoldingInfo& prev_info = it_success.first->second;
+      // Note that the thread ID printed is the same one as the one in
+      // posix logger, but posix logger prints it hex format.
+      return IOError("lock hold by current process, acquire time " +
+                         ToString(prev_info.acquire_time) +
+                         " acquiring thread " +
+                         ToString(prev_info.acquiring_thread),
+                     fname, errno);
    }

    int fd;
@@ -727,7 +753,7 @@ class PosixFileSystem : public FileSystem {
    } else if (LockOrUnlock(fd, true) == -1) {
      // if there is an error in locking, then remove the pathname from
      // lockedfiles
-      lockedFiles.erase(fname);
+      locked_files.erase(fname);
      result = IOError("While lock file", fname, errno);
      close(fd);
    } else {
@@ -738,7 +764,7 @@ class PosixFileSystem : public FileSystem {
      *lock = my_lock;
    }

-    mutex_lockedFiles.Unlock();
+    mutex_locked_files.Unlock();
    return result;
  }

@@ -746,10 +772,10 @@ class PosixFileSystem : public FileSystem {
                      IODebugContext* /*dbg*/) override {
    PosixFileLock* my_lock = reinterpret_cast<PosixFileLock*>(lock);
    IOStatus result;
-    mutex_lockedFiles.Lock();
+    mutex_locked_files.Lock();
    // If we are unlocking, then verify that we had locked it earlier,
-    // it should already exist in lockedFiles. Remove it from lockedFiles.
-    if (lockedFiles.erase(my_lock->filename) != 1) {
+    // it should already exist in locked_files. Remove it from locked_files.
+    if (locked_files.erase(my_lock->filename) != 1) {
      errno = ENOLCK;
      result = IOError("unlock", my_lock->filename, errno);
    } else if (LockOrUnlock(my_lock->fd_, false) == -1) {
@@ -757,7 +783,7 @@ class PosixFileSystem : public FileSystem {
    }
    close(my_lock->fd_);
    delete my_lock;
-    mutex_lockedFiles.Unlock();
+    mutex_locked_files.Unlock();
    return result;
  }

@@ -833,7 +859,15 @@ class PosixFileSystem : public FileSystem {
    optimized.fallocate_with_keep_size = true;
    return optimized;
  }
-
+#ifdef OS_LINUX
+  Status RegisterDbPaths(const std::vector<std::string>& paths) override {
+    return logical_block_size_cache_.RefAndCacheLogicalBlockSize(paths);
+  }
+  Status UnregisterDbPaths(const std::vector<std::string>& paths) override {
+    logical_block_size_cache_.UnrefAndTryRemoveCachedLogicalBlockSize(paths);
+    return Status::OK();
+  }
+#endif
 private:
  bool checkedDiskForMmap_;
  bool forceMmapOff_;  // do we override Env options?
@@ -879,8 +913,26 @@ class PosixFileSystem : public FileSystem {
  // If true, allow non owner read access for db files. Otherwise, non-owner
  //  has no access to db files.
  bool allow_non_owner_access_;
+
+#ifdef OS_LINUX
+  static LogicalBlockSizeCache logical_block_size_cache_;
+#endif
+  static size_t GetLogicalBlockSize(const std::string& fname, int fd);
 };

+#ifdef OS_LINUX
+LogicalBlockSizeCache PosixFileSystem::logical_block_size_cache_;
+#endif
+
+size_t PosixFileSystem::GetLogicalBlockSize(const std::string& fname, int fd) {
+#ifdef OS_LINUX
+  return logical_block_size_cache_.GetLogicalBlockSize(fname, fd);
+#else
+  (void) fname;
+  return PosixHelper::GetLogicalBlockSizeOfFd(fd);
+#endif
+}
+
 PosixFileSystem::PosixFileSystem()
    : checkedDiskForMmap_(false),
      forceMmapOff_(false),
@@ -45,6 +45,35 @@

 namespace ROCKSDB_NAMESPACE {

+std::string IOErrorMsg(const std::string& context,
+                       const std::string& file_name) {
+  if (file_name.empty()) {
+    return context;
+  }
+  return context + ": " + file_name;
+}
+
+// file_name can be left empty if it is not unkown.
+IOStatus IOError(const std::string& context, const std::string& file_name,
+                 int err_number) {
+  switch (err_number) {
+    case ENOSPC: {
+      IOStatus s = IOStatus::NoSpace(IOErrorMsg(context, file_name),
+                                     strerror(err_number));
+      s.SetRetryable(true);
+      return s;
+    }
+    case ESTALE:
+      return IOStatus::IOError(IOStatus::kStaleFile);
+    case ENOENT:
+      return IOStatus::PathNotFound(IOErrorMsg(context, file_name),
+                                    strerror(err_number));
+    default:
+      return IOStatus::IOError(IOErrorMsg(context, file_name),
+                               strerror(err_number));
+  }
+}
+
 // A wrapper for fadvise, if the platform doesn't support fadvise,
 // it will simply return 0.
 int Fadvise(int fd, off_t offset, size_t len, int advice) {
@@ -112,75 +141,6 @@ bool PosixPositionedWrite(int fd, const char* buf, size_t nbyte, off_t offset) {
  return true;
 }

-size_t GetLogicalBufferSize(int __attribute__((__unused__)) fd) {
-#ifdef OS_LINUX
-  struct stat buf;
-  int result = fstat(fd, &buf);
-  if (result == -1) {
-    return kDefaultPageSize;
-  }
-  if (major(buf.st_dev) == 0) {
-    // Unnamed devices (e.g. non-device mounts), reserved as null device number.
-    // These don't have an entry in /sys/dev/block/. Return a sensible default.
-    return kDefaultPageSize;
-  }
-
-  // Reading queue/logical_block_size does not require special permissions.
-  const int kBufferSize = 100;
-  char path[kBufferSize];
-  char real_path[PATH_MAX + 1];
-  snprintf(path, kBufferSize, "/sys/dev/block/%u:%u", major(buf.st_dev),
-           minor(buf.st_dev));
-  if (realpath(path, real_path) == nullptr) {
-    return kDefaultPageSize;
-  }
-  std::string device_dir(real_path);
-  if (!device_dir.empty() && device_dir.back() == '/') {
-    device_dir.pop_back();
-  }
-  // NOTE: sda3 and nvme0n1p1 do not have a `queue/` subdir, only the parent sda
-  // and nvme0n1 have it.
-  // $ ls -al '/sys/dev/block/8:3'
-  // lrwxrwxrwx. 1 root root 0 Jun 26 01:38 /sys/dev/block/8:3 ->
-  // ../../block/sda/sda3
-  // $ ls -al '/sys/dev/block/259:4'
-  // lrwxrwxrwx 1 root root 0 Jan 31 16:04 /sys/dev/block/259:4 ->
-  // ../../devices/pci0000:17/0000:17:00.0/0000:18:00.0/nvme/nvme0/nvme0n1/nvme0n1p1
-  size_t parent_end = device_dir.rfind('/', device_dir.length() - 1);
-  if (parent_end == std::string::npos) {
-    return kDefaultPageSize;
-  }
-  size_t parent_begin = device_dir.rfind('/', parent_end - 1);
-  if (parent_begin == std::string::npos) {
-    return kDefaultPageSize;
-  }
-  std::string parent =
-      device_dir.substr(parent_begin + 1, parent_end - parent_begin - 1);
-  std::string child = device_dir.substr(parent_end + 1, std::string::npos);
-  if (parent != "block" &&
-      (child.compare(0, 4, "nvme") || child.find('p') != std::string::npos)) {
-    device_dir = device_dir.substr(0, parent_end);
-  }
-  std::string fname = device_dir + "/queue/logical_block_size";
-  FILE* fp;
-  size_t size = 0;
-  fp = fopen(fname.c_str(), "r");
-  if (fp != nullptr) {
-    char* line = nullptr;
-    size_t len = 0;
-    if (getline(&line, &len, fp) != -1) {
-      sscanf(line, "%zu", &size);
-    }
-    free(line);
-    fclose(fp);
-  }
-  if (size != 0 && (size & (size - 1)) == 0) {
-    return size;
-  }
-#endif
-  return kDefaultPageSize;
-}
-
 #ifdef ROCKSDB_RANGESYNC_PRESENT

 #if !defined(ZFS_SUPER_MAGIC)
@@ -247,12 +207,13 @@ bool IsSectorAligned(const void* ptr, size_t sector_size) {
 * PosixSequentialFile
 */
 PosixSequentialFile::PosixSequentialFile(const std::string& fname, FILE* file,
-                                         int fd, const EnvOptions& options)
+                                         int fd, size_t logical_block_size,
+                                         const EnvOptions& options)
    : filename_(fname),
      file_(file),
      fd_(fd),
      use_direct_io_(options.use_direct_reads),
-      logical_sector_size_(GetLogicalBufferSize(fd_)) {
+      logical_sector_size_(logical_block_size) {
  assert(!options.use_direct_reads || !options.use_mmap_reads);
 }

@@ -409,13 +370,178 @@ size_t PosixHelper::GetUniqueIdFromFile(int fd, char* id, size_t max_size) {
  return static_cast<size_t>(rid - id);
 }
 #endif
+
+#ifdef OS_LINUX
+std::string RemoveTrailingSlash(const std::string& path) {
+  std::string p = path;
+  if (p.size() > 1 && p.back() == '/') {
+    p.pop_back();
+  }
+  return p;
+}
+
+Status LogicalBlockSizeCache::RefAndCacheLogicalBlockSize(
+    const std::vector<std::string>& directories) {
+  std::vector<std::string> dirs;
+  dirs.reserve(directories.size());
+  for (auto& d : directories) {
+    dirs.emplace_back(RemoveTrailingSlash(d));
+  }
+
+  std::map<std::string, size_t> dir_sizes;
+  {
+    ReadLock lock(&cache_mutex_);
+    for (const auto& dir : dirs) {
+      if (cache_.find(dir) == cache_.end()) {
+        dir_sizes.emplace(dir, 0);
+      }
+    }
+  }
+
+  Status s;
+  for (auto& dir_size : dir_sizes) {
+    s = get_logical_block_size_of_directory_(dir_size.first, &dir_size.second);
+    if (!s.ok()) {
+      return s;
+    }
+  }
+
+  WriteLock lock(&cache_mutex_);
+  for (const auto& dir : dirs) {
+    auto& v = cache_[dir];
+    v.ref++;
+    auto dir_size = dir_sizes.find(dir);
+    if (dir_size != dir_sizes.end()) {
+      v.size = dir_size->second;
+    }
+  }
+  return Status::OK();
+}
+
+void LogicalBlockSizeCache::UnrefAndTryRemoveCachedLogicalBlockSize(
+    const std::vector<std::string>& directories) {
+  std::vector<std::string> dirs;
+  dirs.reserve(directories.size());
+  for (auto& dir : directories) {
+    dirs.emplace_back(RemoveTrailingSlash(dir));
+  }
+
+  WriteLock lock(&cache_mutex_);
+  for (const auto& dir : dirs) {
+    auto it = cache_.find(dir);
+    if (it != cache_.end() && !(--(it->second.ref))) {
+      cache_.erase(it);
+    }
+  }
+}
+
+size_t LogicalBlockSizeCache::GetLogicalBlockSize(const std::string& fname,
+                                                  int fd) {
+  std::string dir = fname.substr(0, fname.find_last_of("/"));
+  if (dir.empty()) {
+    dir = "/";
+  }
+  {
+    ReadLock lock(&cache_mutex_);
+    auto it = cache_.find(dir);
+    if (it != cache_.end()) {
+      return it->second.size;
+    }
+  }
+  return get_logical_block_size_of_fd_(fd);
+}
+#endif
+
+Status PosixHelper::GetLogicalBlockSizeOfDirectory(const std::string& directory,
+                                                   size_t* size) {
+  int fd = open(directory.c_str(), O_DIRECTORY | O_RDONLY);
+  if (fd == -1) {
+    close(fd);
+    return Status::IOError("Cannot open directory " + directory);
+  }
+  *size = PosixHelper::GetLogicalBlockSizeOfFd(fd);
+  close(fd);
+  return Status::OK();
+}
+
+size_t PosixHelper::GetLogicalBlockSizeOfFd(int fd) {
+#ifdef OS_LINUX
+  struct stat buf;
+  int result = fstat(fd, &buf);
+  if (result == -1) {
+    return kDefaultPageSize;
+  }
+  if (major(buf.st_dev) == 0) {
+    // Unnamed devices (e.g. non-device mounts), reserved as null device number.
+    // These don't have an entry in /sys/dev/block/. Return a sensible default.
+    return kDefaultPageSize;
+  }
+
+  // Reading queue/logical_block_size does not require special permissions.
+  const int kBufferSize = 100;
+  char path[kBufferSize];
+  char real_path[PATH_MAX + 1];
+  snprintf(path, kBufferSize, "/sys/dev/block/%u:%u", major(buf.st_dev),
+           minor(buf.st_dev));
+  if (realpath(path, real_path) == nullptr) {
+    return kDefaultPageSize;
+  }
+  std::string device_dir(real_path);
+  if (!device_dir.empty() && device_dir.back() == '/') {
+    device_dir.pop_back();
+  }
+  // NOTE: sda3 and nvme0n1p1 do not have a `queue/` subdir, only the parent sda
+  // and nvme0n1 have it.
+  // $ ls -al '/sys/dev/block/8:3'
+  // lrwxrwxrwx. 1 root root 0 Jun 26 01:38 /sys/dev/block/8:3 ->
+  // ../../block/sda/sda3
+  // $ ls -al '/sys/dev/block/259:4'
+  // lrwxrwxrwx 1 root root 0 Jan 31 16:04 /sys/dev/block/259:4 ->
+  // ../../devices/pci0000:17/0000:17:00.0/0000:18:00.0/nvme/nvme0/nvme0n1/nvme0n1p1
+  size_t parent_end = device_dir.rfind('/', device_dir.length() - 1);
+  if (parent_end == std::string::npos) {
+    return kDefaultPageSize;
+  }
+  size_t parent_begin = device_dir.rfind('/', parent_end - 1);
+  if (parent_begin == std::string::npos) {
+    return kDefaultPageSize;
+  }
+  std::string parent =
+      device_dir.substr(parent_begin + 1, parent_end - parent_begin - 1);
+  std::string child = device_dir.substr(parent_end + 1, std::string::npos);
+  if (parent != "block" &&
+      (child.compare(0, 4, "nvme") || child.find('p') != std::string::npos)) {
+    device_dir = device_dir.substr(0, parent_end);
+  }
+  std::string fname = device_dir + "/queue/logical_block_size";
+  FILE* fp;
+  size_t size = 0;
+  fp = fopen(fname.c_str(), "r");
+  if (fp != nullptr) {
+    char* line = nullptr;
+    size_t len = 0;
+    if (getline(&line, &len, fp) != -1) {
+      sscanf(line, "%zu", &size);
+    }
+    free(line);
+    fclose(fp);
+  }
+  if (size != 0 && (size & (size - 1)) == 0) {
+    return size;
+  }
+#endif
+  (void)fd;
+  return kDefaultPageSize;
+}
+
 /*
 * PosixRandomAccessFile
 *
 * pread() based random-access
 */
 PosixRandomAccessFile::PosixRandomAccessFile(
-    const std::string& fname, int fd, const EnvOptions& options
+    const std::string& fname, int fd, size_t logical_block_size,
+    const EnvOptions& options
 #if defined(ROCKSDB_IOURING_PRESENT)
    ,
    ThreadLocalPtr* thread_local_io_urings
@@ -424,7 +550,7 @@ PosixRandomAccessFile::PosixRandomAccessFile(
    : filename_(fname),
      fd_(fd),
      use_direct_io_(options.use_direct_reads),
-      logical_sector_size_(GetLogicalBufferSize(fd_))
+      logical_sector_size_(logical_block_size)
 #if defined(ROCKSDB_IOURING_PRESENT)
      ,
      thread_local_io_urings_(thread_local_io_urings)
@@ -988,13 +1114,14 @@ IOStatus PosixMmapFile::Allocate(uint64_t offset, uint64_t len,
 * Use posix write to write data to a file.
 */
 PosixWritableFile::PosixWritableFile(const std::string& fname, int fd,
+                                     size_t logical_block_size,
                                     const EnvOptions& options)
    : FSWritableFile(options),
      filename_(fname),
      use_direct_io_(options.use_direct_writes),
      fd_(fd),
      filesize_(0),
-      logical_sector_size_(GetLogicalBufferSize(fd_)) {
+      logical_sector_size_(logical_block_size) {
 #ifdef ROCKSDB_FALLOCATE_PRESENT
  allow_fallocate_ = options.allow_fallocate;
  fallocate_with_keep_size_ = options.fallocate_with_keep_size;
@@ -14,11 +14,15 @@
 #endif
 #include <unistd.h>
 #include <atomic>
+#include <functional>
+#include <map>
 #include <string>
+#include "port/port.h"
 #include "rocksdb/env.h"
-#include "util/thread_local.h"
 #include "rocksdb/file_system.h"
 #include "rocksdb/io_status.h"
+#include "util/mutexlock.h"
+#include "util/thread_local.h"

 // For non linux platform, the following macros are used only as place
 // holder.
@@ -27,44 +31,97 @@
 #define POSIX_FADV_RANDOM 1     /* [MC1] expect random page refs */
 #define POSIX_FADV_SEQUENTIAL 2 /* [MC1] expect sequential page refs */
 #define POSIX_FADV_WILLNEED 3   /* [MC1] will need these pages */
-#define POSIX_FADV_DONTNEED 4   /* [MC1] dont need these pages */
+#define POSIX_FADV_DONTNEED 4   /* [MC1] don't need these pages */
 #endif

 namespace ROCKSDB_NAMESPACE {
-static std::string IOErrorMsg(const std::string& context,
-                              const std::string& file_name) {
-  if (file_name.empty()) {
-    return context;
-  }
-  return context + ": " + file_name;
-}
-
+std::string IOErrorMsg(const std::string& context,
+                       const std::string& file_name);
 // file_name can be left empty if it is not unkown.
-static IOStatus IOError(const std::string& context,
-                        const std::string& file_name, int err_number) {
-  switch (err_number) {
-    case ENOSPC: {
-      IOStatus s = IOStatus::NoSpace(IOErrorMsg(context, file_name),
-                                     strerror(err_number));
-      s.SetRetryable(true);
-      return s;
-    }
-  case ESTALE:
-    return IOStatus::IOError(IOStatus::kStaleFile);
-  case ENOENT:
-    return IOStatus::PathNotFound(IOErrorMsg(context, file_name),
-                                  strerror(err_number));
-  default:
-    return IOStatus::IOError(IOErrorMsg(context, file_name),
-                             strerror(err_number));
-  }
-}
+IOStatus IOError(const std::string& context, const std::string& file_name,
+                 int err_number);

 class PosixHelper {
 public:
  static size_t GetUniqueIdFromFile(int fd, char* id, size_t max_size);
+  static size_t GetLogicalBlockSizeOfFd(int fd);
+  static Status GetLogicalBlockSizeOfDirectory(const std::string& directory,
+                                               size_t* size);
 };

+#ifdef OS_LINUX
+// Files under a specific directory have the same logical block size.
+// This class caches the logical block size for the specified directories to
+// save the CPU cost of computing the size.
+// Safe for concurrent access from multiple threads without any external
+// synchronization.
+class LogicalBlockSizeCache {
+ public:
+  LogicalBlockSizeCache(
+      std::function<size_t(int)> get_logical_block_size_of_fd =
+          PosixHelper::GetLogicalBlockSizeOfFd,
+      std::function<Status(const std::string&, size_t*)>
+          get_logical_block_size_of_directory =
+              PosixHelper::GetLogicalBlockSizeOfDirectory)
+      : get_logical_block_size_of_fd_(get_logical_block_size_of_fd),
+        get_logical_block_size_of_directory_(
+            get_logical_block_size_of_directory) {}
+
+  // Takes the following actions:
+  // 1. Increases reference count of the directories;
+  // 2. If the directory's logical block size is not cached,
+  //    compute the buffer size and cache the result.
+  Status RefAndCacheLogicalBlockSize(
+      const std::vector<std::string>& directories);
+
+  // Takes the following actions:
+  // 1. Decreases reference count of the directories;
+  // 2. If the reference count of a directory reaches 0, remove the directory
+  //    from the cache.
+  void UnrefAndTryRemoveCachedLogicalBlockSize(
+      const std::vector<std::string>& directories);
+
+  // Returns the logical block size for the file.
+  //
+  // If the file is under a cached directory, return the cached size.
+  // Otherwise, the size is computed.
+  size_t GetLogicalBlockSize(const std::string& fname, int fd);
+
+  int GetRefCount(const std::string& dir) {
+    ReadLock lock(&cache_mutex_);
+    auto it = cache_.find(dir);
+    if (it == cache_.end()) {
+      return 0;
+    }
+    return it->second.ref;
+  }
+
+  size_t Size() const { return cache_.size(); }
+
+  bool Contains(const std::string& dir) {
+    ReadLock lock(&cache_mutex_);
+    return cache_.find(dir) != cache_.end();
+  }
+
+ private:
+  struct CacheValue {
+    CacheValue() : size(0), ref(0) {}
+
+    // Logical block size of the directory.
+    size_t size;
+    // Reference count of the directory.
+    int ref;
+  };
+
+  std::function<size_t(int)> get_logical_block_size_of_fd_;
+  std::function<Status(const std::string&, size_t*)>
+      get_logical_block_size_of_directory_;
+
+  std::map<std::string, CacheValue> cache_;
+  port::RWMutex cache_mutex_;
+};
+#endif
+
 class PosixSequentialFile : public FSSequentialFile {
 private:
  std::string filename_;
@@ -75,6 +132,7 @@ class PosixSequentialFile : public FSSequentialFile {

 public:
  PosixSequentialFile(const std::string& fname, FILE* file, int fd,
+                      size_t logical_block_size,
                      const EnvOptions& options);
  virtual ~PosixSequentialFile();

@@ -123,6 +181,7 @@ class PosixRandomAccessFile : public FSRandomAccessFile {

 public:
  PosixRandomAccessFile(const std::string& fname, int fd,
+                        size_t logical_block_size,
                        const EnvOptions& options
 #if defined(ROCKSDB_IOURING_PRESENT)
                        ,
@@ -172,6 +231,7 @@ class PosixWritableFile : public FSWritableFile {

 public:
  explicit PosixWritableFile(const std::string& fname, int fd,
+                             size_t logical_block_size,
                             const EnvOptions& options);
  virtual ~PosixWritableFile();

@@ -0,0 +1,140 @@
+// Copyright (c) 2020-present, Facebook, Inc.  All rights reserved.
+// This source code is licensed under both the GPLv2 (found in the
+// COPYING file in the root directory) and Apache 2.0 License
+// (found in the LICENSE.Apache file in the root directory).
+
+#include "test_util/testharness.h"
+
+#ifdef ROCKSDB_LIB_IO_POSIX
+#include "env/io_posix.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+#ifdef OS_LINUX
+class LogicalBlockSizeCacheTest : public testing::Test {};
+
+// Tests the caching behavior.
+TEST_F(LogicalBlockSizeCacheTest, Cache) {
+  int ncall = 0;
+  auto get_fd_block_size = [&](int fd) {
+    ncall++;
+    return fd;
+  };
+  std::map<std::string, int> dir_fds{
+      {"/", 0},
+      {"/db", 1},
+      {"/db1", 2},
+      {"/db2", 3},
+  };
+  auto get_dir_block_size = [&](const std::string& dir, size_t* size) {
+    ncall++;
+    *size = dir_fds[dir];
+    return Status::OK();
+  };
+  LogicalBlockSizeCache cache(get_fd_block_size, get_dir_block_size);
+  ASSERT_EQ(0, ncall);
+  ASSERT_EQ(0, cache.Size());
+
+  ASSERT_EQ(6, cache.GetLogicalBlockSize("/sst", 6));
+  ASSERT_EQ(1, ncall);
+  ASSERT_EQ(7, cache.GetLogicalBlockSize("/db/sst1", 7));
+  ASSERT_EQ(2, ncall);
+  ASSERT_EQ(8, cache.GetLogicalBlockSize("/db/sst2", 8));
+  ASSERT_EQ(3, ncall);
+
+  ASSERT_OK(cache.RefAndCacheLogicalBlockSize({"/", "/db1/", "/db2"}));
+  ASSERT_EQ(3, cache.Size());
+  ASSERT_TRUE(cache.Contains("/"));
+  ASSERT_TRUE(cache.Contains("/db1"));
+  ASSERT_TRUE(cache.Contains("/db2"));
+  ASSERT_EQ(6, ncall);
+  // Block size for / is cached.
+  ASSERT_EQ(0, cache.GetLogicalBlockSize("/sst", 6));
+  ASSERT_EQ(6, ncall);
+  // No cached size for /db.
+  ASSERT_EQ(7, cache.GetLogicalBlockSize("/db/sst1", 7));
+  ASSERT_EQ(7, ncall);
+  ASSERT_EQ(8, cache.GetLogicalBlockSize("/db/sst2", 8));
+  ASSERT_EQ(8, ncall);
+  // Block size for /db1 is cached.
+  ASSERT_EQ(2, cache.GetLogicalBlockSize("/db1/sst1", 4));
+  ASSERT_EQ(8, ncall);
+  ASSERT_EQ(2, cache.GetLogicalBlockSize("/db1/sst2", 5));
+  ASSERT_EQ(8, ncall);
+  // Block size for /db2 is cached.
+  ASSERT_EQ(3, cache.GetLogicalBlockSize("/db2/sst1", 6));
+  ASSERT_EQ(8, ncall);
+  ASSERT_EQ(3, cache.GetLogicalBlockSize("/db2/sst2", 7));
+  ASSERT_EQ(8, ncall);
+
+  cache.RefAndCacheLogicalBlockSize({"/db"});
+  ASSERT_EQ(4, cache.Size());
+  ASSERT_TRUE(cache.Contains("/"));
+  ASSERT_TRUE(cache.Contains("/db1"));
+  ASSERT_TRUE(cache.Contains("/db2"));
+  ASSERT_TRUE(cache.Contains("/db"));
+
+  ASSERT_EQ(9, ncall);
+  // Block size for /db is cached.
+  ASSERT_EQ(1, cache.GetLogicalBlockSize("/db/sst1", 7));
+  ASSERT_EQ(9, ncall);
+  ASSERT_EQ(1, cache.GetLogicalBlockSize("/db/sst2", 8));
+  ASSERT_EQ(9, ncall);
+}
+
+// Tests the reference counting behavior.
+TEST_F(LogicalBlockSizeCacheTest, Ref) {
+  int ncall = 0;
+  auto get_fd_block_size = [&](int fd) {
+    ncall++;
+    return fd;
+  };
+  std::map<std::string, int> dir_fds{
+      {"/db", 0},
+  };
+  auto get_dir_block_size = [&](const std::string& dir, size_t* size) {
+    ncall++;
+    *size = dir_fds[dir];
+    return Status::OK();
+  };
+  LogicalBlockSizeCache cache(get_fd_block_size, get_dir_block_size);
+
+  ASSERT_EQ(0, ncall);
+
+  ASSERT_EQ(1, cache.GetLogicalBlockSize("/db/sst0", 1));
+  ASSERT_EQ(1, ncall);
+
+  cache.RefAndCacheLogicalBlockSize({"/db"});
+  ASSERT_EQ(2, ncall);
+  ASSERT_EQ(1, cache.GetRefCount("/db"));
+  // Block size for /db is cached. Ref count = 1.
+  ASSERT_EQ(0, cache.GetLogicalBlockSize("/db/sst1", 1));
+  ASSERT_EQ(2, ncall);
+
+  // Ref count = 2, but won't recompute the cached buffer size.
+  cache.RefAndCacheLogicalBlockSize({"/db"});
+  ASSERT_EQ(2, cache.GetRefCount("/db"));
+  ASSERT_EQ(2, ncall);
+
+  // Ref count = 1.
+  cache.UnrefAndTryRemoveCachedLogicalBlockSize({"/db"});
+  ASSERT_EQ(1, cache.GetRefCount("/db"));
+  // Block size for /db is still cached.
+  ASSERT_EQ(0, cache.GetLogicalBlockSize("/db/sst2", 1));
+  ASSERT_EQ(2, ncall);
+
+  // Ref count = 0 and cached buffer size for /db is removed.
+  cache.UnrefAndTryRemoveCachedLogicalBlockSize({"/db"});
+  ASSERT_EQ(0, cache.Size());
+  ASSERT_EQ(1, cache.GetLogicalBlockSize("/db/sst0", 1));
+  ASSERT_EQ(3, ncall);
+}
+#endif
+
+}  // namespace ROCKSDB_NAMESPACE
+#endif
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -587,6 +587,7 @@ Status MockEnv::Truncate(const std::string& fname, size_t size) {

 Status MockEnv::CreateDir(const std::string& dirname) {
  auto dn = NormalizePath(dirname);
+  MutexLock lock(&mutex_);
  if (file_map_.find(dn) == file_map_.end()) {
    MemFile* file = new MemFile(this, dn, false);
    file->Ref();
@@ -24,75 +24,80 @@ class MockEnv : public EnvWrapper {
 public:
  explicit MockEnv(Env* base_env);

-  virtual ~MockEnv();
+  ~MockEnv() override;

  // Partial implementation of the Env interface.
-  virtual Status NewSequentialFile(const std::string& fname,
-                                   std::unique_ptr<SequentialFile>* result,
-                                   const EnvOptions& soptions) override;
+  Status RegisterDbPaths(const std::vector<std::string>& /*paths*/) override {
+    return Status::OK();
+  }

-  virtual Status NewRandomAccessFile(const std::string& fname,
-                                     std::unique_ptr<RandomAccessFile>* result,
-                                     const EnvOptions& soptions) override;
+  Status UnregisterDbPaths(const std::vector<std::string>& /*paths*/) override {
+    return Status::OK();
+  }

-  virtual Status NewRandomRWFile(const std::string& fname,
-                                 std::unique_ptr<RandomRWFile>* result,
-                                 const EnvOptions& options) override;
+  Status NewSequentialFile(const std::string& fname,
+                           std::unique_ptr<SequentialFile>* result,
+                           const EnvOptions& soptions) override;

-  virtual Status ReuseWritableFile(const std::string& fname,
-                                   const std::string& old_fname,
-                                   std::unique_ptr<WritableFile>* result,
-                                   const EnvOptions& options) override;
+  Status NewRandomAccessFile(const std::string& fname,
+                             std::unique_ptr<RandomAccessFile>* result,
+                             const EnvOptions& soptions) override;

-  virtual Status NewWritableFile(const std::string& fname,
-                                 std::unique_ptr<WritableFile>* result,
-                                 const EnvOptions& env_options) override;
+  Status NewRandomRWFile(const std::string& fname,
+                         std::unique_ptr<RandomRWFile>* result,
+                         const EnvOptions& options) override;

-  virtual Status NewDirectory(const std::string& name,
-                              std::unique_ptr<Directory>* result) override;
+  Status ReuseWritableFile(const std::string& fname,
+                           const std::string& old_fname,
+                           std::unique_ptr<WritableFile>* result,
+                           const EnvOptions& options) override;

-  virtual Status FileExists(const std::string& fname) override;
+  Status NewWritableFile(const std::string& fname,
+                         std::unique_ptr<WritableFile>* result,
+                         const EnvOptions& env_options) override;

-  virtual Status GetChildren(const std::string& dir,
-                             std::vector<std::string>* result) override;
+  Status NewDirectory(const std::string& name,
+                      std::unique_ptr<Directory>* result) override;
+
+  Status FileExists(const std::string& fname) override;
+
+  Status GetChildren(const std::string& dir,
+                     std::vector<std::string>* result) override;

  void DeleteFileInternal(const std::string& fname);

-  virtual Status DeleteFile(const std::string& fname) override;
+  Status DeleteFile(const std::string& fname) override;

-  virtual Status Truncate(const std::string& fname, size_t size) override;
+  Status Truncate(const std::string& fname, size_t size) override;

-  virtual Status CreateDir(const std::string& dirname) override;
+  Status CreateDir(const std::string& dirname) override;

-  virtual Status CreateDirIfMissing(const std::string& dirname) override;
+  Status CreateDirIfMissing(const std::string& dirname) override;

-  virtual Status DeleteDir(const std::string& dirname) override;
+  Status DeleteDir(const std::string& dirname) override;

-  virtual Status GetFileSize(const std::string& fname,
-                             uint64_t* file_size) override;
+  Status GetFileSize(const std::string& fname, uint64_t* file_size) override;

-  virtual Status GetFileModificationTime(const std::string& fname,
-                                         uint64_t* time) override;
+  Status GetFileModificationTime(const std::string& fname,
+                                 uint64_t* time) override;

-  virtual Status RenameFile(const std::string& src,
-                            const std::string& target) override;
+  Status RenameFile(const std::string& src, const std::string& target) override;

-  virtual Status LinkFile(const std::string& src,
-                          const std::string& target) override;
+  Status LinkFile(const std::string& src, const std::string& target) override;

-  virtual Status NewLogger(const std::string& fname,
-                           std::shared_ptr<Logger>* result) override;
+  Status NewLogger(const std::string& fname,
+                   std::shared_ptr<Logger>* result) override;

-  virtual Status LockFile(const std::string& fname, FileLock** flock) override;
+  Status LockFile(const std::string& fname, FileLock** flock) override;

-  virtual Status UnlockFile(FileLock* flock) override;
+  Status UnlockFile(FileLock* flock) override;

-  virtual Status GetTestDirectory(std::string* path) override;
+  Status GetTestDirectory(std::string* path) override;

  // Results of these can be affected by FakeSleepForMicroseconds()
-  virtual Status GetCurrentTime(int64_t* unix_time) override;
-  virtual uint64_t NowMicros() override;
-  virtual uint64_t NowNanos() override;
+  Status GetCurrentTime(int64_t* unix_time) override;
+  uint64_t NowMicros() override;
+  uint64_t NowNanos() override;

  Status CorruptBuffer(const std::string& fname);

@@ -28,7 +28,8 @@ int main() {
  assert(s.ok());

  // close DB
-  delete cf;
+  s = db->DestroyColumnFamilyHandle(cf);
+  assert(s.ok());
  delete db;

  // open DB with two column families
@@ -64,7 +65,8 @@ int main() {

  // close db
  for (auto handle : handles) {
-    delete handle;
+    s = db->DestroyColumnFamilyHandle(handle);
+    assert(s.ok());
  }
  delete db;

@@ -216,7 +216,7 @@ void DeleteScheduler::BackgroundEmptyTrash() {
      const FileAndDir& fad = queue_.front();
      std::string path_in_trash = fad.fname;

-      // We dont need to hold the lock while deleting the file
+      // We don't need to hold the lock while deleting the file
      mu_.Unlock();
      uint64_t deleted_bytes = 0;
      bool is_complete = true;
@@ -90,7 +90,7 @@ class DeleteSchedulerTest : public testing::Test {
  }

  void NewDeleteScheduler() {
-    // Tests in this file are for DeleteScheduler component and dont create any
+    // Tests in this file are for DeleteScheduler component and don't create any
    // DBs, so we need to set max_trash_db_ratio to 100% (instead of default
    // 25%)
    std::shared_ptr<FileSystem>
@@ -306,7 +306,7 @@ TEST_F(DeleteSchedulerTest, RateLimitingMultiThreaded) {
 }

 // Disable rate limiting by setting rate_bytes_per_sec_ to 0 and make sure
-// that when DeleteScheduler delete a file it delete it immediately and dont
+// that when DeleteScheduler delete a file it delete it immediately and don't
 // move it to trash
 TEST_F(DeleteSchedulerTest, DisableRateLimiting) {
  int bg_delete_file = 0;
@@ -88,7 +88,7 @@ Status FilePrefetchBuffer::Prefetch(RandomAccessFileReader* reader,
  Slice result;
  s = reader->Read(rounddown_offset + chunk_len,
                   static_cast<size_t>(roundup_len - chunk_len), &result,
-                   buffer_.BufferStart() + chunk_len, for_compaction);
+                   buffer_.BufferStart() + chunk_len, nullptr, for_compaction);
  if (s.ok()) {
    buffer_offset_ = rounddown_offset;
    buffer_.Size(static_cast<size_t>(chunk_len) + result.size());
@@ -370,7 +370,7 @@ bool ParseFileName(const std::string& fname, uint64_t* number,

 Status SetCurrentFile(Env* env, const std::string& dbname,
                      uint64_t descriptor_number,
-                      Directory* directory_to_fsync) {
+                      FSDirectory* directory_to_fsync) {
  // Remove leading "dbname/" and add newline to manifest file name
  std::string manifest = DescriptorFileName(dbname, descriptor_number);
  Slice contents = manifest;
@@ -385,7 +385,7 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
  }
  if (s.ok()) {
    if (directory_to_fsync != nullptr) {
-      s = directory_to_fsync->Fsync();
+      s = directory_to_fsync->Fsync(IOOptions(), nullptr);
    }
  } else {
    env->DeleteFile(tmp);
@@ -17,6 +17,7 @@

 #include "options/db_options.h"
 #include "port/port.h"
+#include "rocksdb/file_system.h"
 #include "rocksdb/options.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"
@@ -164,7 +165,7 @@ extern bool ParseFileName(const std::string& filename, uint64_t* number,
 // specified number.
 extern Status SetCurrentFile(Env* env, const std::string& dbname,
                             uint64_t descriptor_number,
-                             Directory* directory_to_fsync);
+                             FSDirectory* directory_to_fsync);

 // Make the IDENTITY file for the db
 extern Status SetIdentityFile(Env* env, const std::string& dbname,
@@ -21,7 +21,10 @@

 namespace ROCKSDB_NAMESPACE {
 Status RandomAccessFileReader::Read(uint64_t offset, size_t n, Slice* result,
-                                    char* scratch, bool for_compaction) const {
+                                    char* scratch,
+                                    std::unique_ptr<const char[]>* internal_buf,
+                                    bool for_compaction) const {
+  (void) internal_buf;
  Status s;
  uint64_t elapsed = 0;
  {
@@ -77,8 +80,13 @@ Status RandomAccessFileReader::Read(uint64_t offset, size_t n, Slice* result,
      }
      size_t res_len = 0;
      if (s.ok() && offset_advance < buf.CurrentSize()) {
-        res_len = buf.Read(scratch, offset_advance,
-                           std::min(buf.CurrentSize() - offset_advance, n));
+        res_len = std::min(buf.CurrentSize() - offset_advance, n);
+        if (internal_buf == nullptr) {
+          buf.Read(scratch, offset_advance, res_len);
+        } else {
+          scratch = buf.BufferStart();
+          internal_buf->reset(buf.Release());
+        }
      }
      *result = Slice(scratch, res_len);
 #endif  // !ROCKSDB_LITE
@@ -102,7 +102,18 @@ class RandomAccessFileReader {
  RandomAccessFileReader(const RandomAccessFileReader&) = delete;
  RandomAccessFileReader& operator=(const RandomAccessFileReader&) = delete;

+  // In non-direct IO mode,
+  // 1. if using mmap, result is stored in a buffer other than scratch;
+  // 2. if not using mmap, result is stored in the buffer starting from scratch.
+  //
+  // In direct IO mode, an internal aligned buffer is allocated.
+  // 1. If internal_buf is null, then results are copied to the buffer
+  // starting from scratch;
+  // 2. Otherwise, scratch is not used and can be null, the internal_buf owns
+  // the internally allocated buffer on return, and the result refers to a
+  // region in internal_buf.
  Status Read(uint64_t offset, size_t n, Slice* result, char* scratch,
+              std::unique_ptr<const char[]>* internal_buf,
              bool for_compaction = false) const;

  Status MultiRead(FSReadRequest* reqs, size_t num_reqs) const;
@@ -645,7 +645,6 @@ struct AdvancedColumnFamilyOptions {
  bool report_bg_io_stats = false;

  // Files older than TTL will go through the compaction process.
-  // Pre-req: This needs max_open_files to be set to -1.
  // In Level: Non-bottom-level files older than TTL will go through the
  //           compation process.
  // In FIFO: Files older than TTL will be deleted.
@@ -673,7 +672,6 @@ struct AdvancedColumnFamilyOptions {
  // Supported in Level and FIFO compaction.
  // In FIFO compaction, this option has the same meaning as TTL and whichever
  // stricter will be used.
-  // Pre-req: max_open_file == -1.
  // unit: seconds. Ex: 7 days = 7 * 24 * 60 * 60
  //
  // Values:
@@ -42,6 +42,9 @@ class Comparator {
  //   < 0 iff "a" < "b",
  //   == 0 iff "a" == "b",
  //   > 0 iff "a" > "b"
+  // Note that Compare(a, b) also compares timestamp if timestamp size is
+  // non-zero. For the same user key with different timestamps, larger (newer)
+  // timestamp comes first.
  virtual int Compare(const Slice& a, const Slice& b) const = 0;

  // Compares two slices for equality. The following invariant should always
@@ -97,15 +100,27 @@ class Comparator {

  inline size_t timestamp_size() const { return timestamp_size_; }

-  virtual int CompareWithoutTimestamp(const Slice& a, const Slice& b) const {
-    return Compare(a, b);
+  int CompareWithoutTimestamp(const Slice& a, const Slice& b) const {
+    return CompareWithoutTimestamp(a, /*a_has_ts=*/true, b, /*b_has_ts=*/true);
  }

+  // For two events e1 and e2 whose timestamps are t1 and t2 respectively,
+  // Returns value:
+  // < 0  iff t1 < t2
+  // == 0 iff t1 == t2
+  // > 0  iff t1 > t2
+  // Note that an all-zero byte array will be the smallest (oldest) timestamp
+  // of the same length.
  virtual int CompareTimestamp(const Slice& /*ts1*/,
                               const Slice& /*ts2*/) const {
    return 0;
  }

+  virtual int CompareWithoutTimestamp(const Slice& a, bool /*a_has_ts*/,
+                                      const Slice& b, bool /*b_has_ts*/) const {
+    return Compare(a, b);
+  }
+
 private:
  size_t timestamp_size_;
 };
@@ -388,6 +388,9 @@ class DB {
  // If the database contains an entry for "key" store the
  // corresponding value in *value and return OK.
  //
+  // If timestamp is enabled and a non-null timestamp pointer is passed in,
+  // timestamp is returned.
+  //
  // If there is no entry for "key" leave *value unchanged and return
  // a status for which Status::IsNotFound() returns true.
  //
@@ -412,6 +415,32 @@ class DB {
    return Get(options, DefaultColumnFamily(), key, value);
  }

+  // Get() methods that return timestamp. Derived DB classes don't need to worry
+  // about this group of methods if they don't care about timestamp feature.
+  virtual inline Status Get(const ReadOptions& options,
+                            ColumnFamilyHandle* column_family, const Slice& key,
+                            std::string* value, std::string* timestamp) {
+    assert(value != nullptr);
+    PinnableSlice pinnable_val(value);
+    assert(!pinnable_val.IsPinned());
+    auto s = Get(options, column_family, key, &pinnable_val, timestamp);
+    if (s.ok() && pinnable_val.IsPinned()) {
+      value->assign(pinnable_val.data(), pinnable_val.size());
+    }  // else value is already assigned
+    return s;
+  }
+  virtual Status Get(const ReadOptions& /*options*/,
+                     ColumnFamilyHandle* /*column_family*/,
+                     const Slice& /*key*/, PinnableSlice* /*value*/,
+                     std::string* /*timestamp*/) {
+    return Status::NotSupported(
+        "Get() that returns timestamp is not implemented.");
+  }
+  virtual Status Get(const ReadOptions& options, const Slice& key,
+                     std::string* value, std::string* timestamp) {
+    return Get(options, DefaultColumnFamily(), key, value, timestamp);
+  }
+
  // Returns all the merge operands corresponding to the key. If the
  // number of merge operands in DB is greater than
  // merge_operands_options.expected_max_number_of_operands
@@ -542,17 +571,33 @@ class DB {
  virtual bool KeyMayExist(const ReadOptions& /*options*/,
                           ColumnFamilyHandle* /*column_family*/,
                           const Slice& /*key*/, std::string* /*value*/,
+                           std::string* /*timestamp*/,
                           bool* value_found = nullptr) {
    if (value_found != nullptr) {
      *value_found = false;
    }
    return true;
  }
+
+  virtual bool KeyMayExist(const ReadOptions& options,
+                           ColumnFamilyHandle* column_family, const Slice& key,
+                           std::string* value, bool* value_found = nullptr) {
+    return KeyMayExist(options, column_family, key, value,
+                       /*timestamp=*/nullptr, value_found);
+  }
+
  virtual bool KeyMayExist(const ReadOptions& options, const Slice& key,
                           std::string* value, bool* value_found = nullptr) {
    return KeyMayExist(options, DefaultColumnFamily(), key, value, value_found);
  }

+  virtual bool KeyMayExist(const ReadOptions& options, const Slice& key,
+                           std::string* value, std::string* timestamp,
+                           bool* value_found = nullptr) {
+    return KeyMayExist(options, DefaultColumnFamily(), key, value, timestamp,
+                       value_found);
+  }
+
  // Return a heap-allocated iterator over the contents of the database.
  // The result of NewIterator() is initially invalid (caller must
  // call one of the Seek methods on the iterator before using it).
@@ -163,6 +163,15 @@ class Env {
  // The result of Default() belongs to rocksdb and must never be deleted.
  static Env* Default();

+  // See FileSystem::RegisterDbPaths.
+  virtual Status RegisterDbPaths(const std::vector<std::string>& /*paths*/) {
+    return Status::OK();
+  }
+  // See FileSystem::UnregisterDbPaths.
+  virtual Status UnregisterDbPaths(const std::vector<std::string>& /*paths*/) {
+    return Status::OK();
+  }
+
  // Create a brand new sequentially-readable file with the specified name.
  // On success, stores a pointer to the new file in *result and returns OK.
  // On failure stores nullptr in *result and returns non-OK.  If the file does
@@ -1155,6 +1164,14 @@ class EnvWrapper : public Env {
  Env* target() const { return target_; }

  // The following text is boilerplate that forwards all methods to target()
+  Status RegisterDbPaths(const std::vector<std::string>& paths) override {
+    return target_->RegisterDbPaths(paths);
+  }
+
+  Status UnregisterDbPaths(const std::vector<std::string>& paths) override {
+    return target_->UnregisterDbPaths(paths);
+  }
+
  Status NewSequentialFile(const std::string& f,
                           std::unique_ptr<SequentialFile>* r,
                           const EnvOptions& options) override {
@@ -171,6 +171,35 @@ class FileSystem {
  // The result of Default() belongs to rocksdb and must never be deleted.
  static std::shared_ptr<FileSystem> Default();

+  // Handles the event when a new DB or a new ColumnFamily starts using the
+  // specified data paths.
+  //
+  // The data paths might be shared by different DBs or ColumnFamilies,
+  // so RegisterDbPaths might be called with the same data paths.
+  // For example, when CreateColumnFamily is called multiple times with the same
+  // data path, RegisterDbPaths will also be called with the same data path.
+  //
+  // If the return status is ok, then the paths must be correspondingly
+  // called in UnregisterDbPaths;
+  // otherwise this method should have no side effect, and UnregisterDbPaths
+  // do not need to be called for the paths.
+  //
+  // Different implementations may take different actions.
+  // By default, it's a no-op and returns Status::OK.
+  virtual Status RegisterDbPaths(const std::vector<std::string>& /*paths*/) {
+    return Status::OK();
+  }
+  // Handles the event a DB or a ColumnFamily stops using the specified data
+  // paths.
+  //
+  // It should be called corresponding to each successful RegisterDbPaths.
+  //
+  // Different implementations may take different actions.
+  // By default, it's a no-op and returns Status::OK.
+  virtual Status UnregisterDbPaths(const std::vector<std::string>& /*paths*/) {
+    return Status::OK();
+  }
+
  // Create a brand new sequentially-readable file with the specified name.
  // On success, stores a pointer to the new file in *result and returns OK.
  // On failure stores nullptr in *result and returns non-OK.  If the file does
@@ -209,9 +209,7 @@ inline IOStatus& IOStatus::operator=(IOStatus&& s)
    subcode_ = std::move(s.subcode_);
    s.subcode_ = kNone;
    retryable_ = s.retryable_;
-    retryable_ = false;
    data_loss_ = s.data_loss_;
-    data_loss_ = false;
    scope_ = s.scope_;
    scope_ = kIOErrorScopeFileSystem;
    delete[] state_;
@@ -45,6 +45,7 @@ class Iterator : public Cleanable {

  // Position at the last key in the source.  The iterator is
  // Valid() after this call iff the source is not empty.
+  // Currently incompatible with user timestamp.
  virtual void SeekToLast() = 0;

  // Position at the first key in the source that at or past target.
@@ -53,11 +54,13 @@ class Iterator : public Cleanable {
  // All Seek*() methods clear any error status() that the iterator had prior to
  // the call; after the seek, status() indicates only the error (if any) that
  // happened during the seek, not any past errors.
+  // Target does not contain timestamp.
  virtual void Seek(const Slice& target) = 0;

  // Position at the last key in the source that at or before target.
  // The iterator is Valid() after this call iff the source contains
  // an entry that comes at or before target.
+  // Currently incompatible with user timestamp.
  virtual void SeekForPrev(const Slice& target) = 0;

  // Moves to the next entry in the source.  After this call, Valid() is
@@ -67,6 +70,7 @@ class Iterator : public Cleanable {

  // Moves to the previous entry in the source.  After this call, Valid() is
  // true iff the iterator was not positioned at the first entry in source.
+  // Currently incompatible with user timestamp.
  // REQUIRES: Valid()
  virtual void Prev() = 0;

@@ -108,6 +112,11 @@ class Iterator : public Cleanable {
  //   Get the user-key portion of the internal key at which the iteration
  //   stopped.
  virtual Status GetProperty(std::string prop_name, std::string* prop);
+
+  virtual Slice timestamp() const {
+    assert(false);
+    return Slice();
+  }
 };

 // Return an empty iterator (yields nothing).