Print out compression type of new SST files in logging

Summary: Closes https://github.com/facebook/rocksdb/pull/3264 Differential Revision: D6552768 Pulled By: siying fbshipit-source-id: 6303110aff22f341d5cff41f8d2d4f138a53652d
NUMBER_BLOCK_COMPRESSED, etc, shouldn't be treated as timer counter
2026-11-07 03:22:34 -05:00 · 2017-12-14 10:27:43 -08:00 · 2017-12-14 10:27:43 -08:00 · 2017-12-13 11:57:29 -08:00 · 2017-12-13 07:57:44 -08:00 · 2017-12-13 07:57:43 -08:00
88 changed files with 1013 additions and 1910 deletions
@@ -169,7 +169,7 @@ if(PORTABLE)
  # MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
  # is available, it is available by default.
  if(FORCE_SSE42 AND NOT MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -mpclmul")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
  endif()
 else()
  if(MSVC)
@@ -181,18 +181,13 @@ endif()

 include(CheckCXXSourceCompiles)
 if(NOT MSVC)
-  set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul")
+  set(CMAKE_REQUIRED_FLAGS "-msse4.2")
 endif()
 CHECK_CXX_SOURCE_COMPILES("
 #include <cstdint>
 #include <nmmintrin.h>
-#include <wmmintrin.h>
 int main() {
  volatile uint32_t x = _mm_crc32_u32(0, 0);
-  const auto a = _mm_set_epi64x(0, 0);
-  const auto b = _mm_set_epi64x(0, 0);
-  const auto c = _mm_clmulepi64_si128(a, b, 0x00);
-  auto d = _mm_cvtsi128_si64(c);
 }
 " HAVE_SSE42)
 unset(CMAKE_REQUIRED_FLAGS)
@@ -490,6 +485,7 @@ set(SOURCES
        table/block_based_table_factory.cc
        table/block_based_table_reader.cc
        table/block_builder.cc
+        table/block_fetcher.cc
        table/block_prefix_index.cc
        table/bloom_block.cc
        table/cuckoo_table_builder.cc
@@ -613,7 +609,7 @@ if(HAVE_SSE42 AND NOT FORCE_SSE42)
  if(NOT MSVC)
 set_source_files_properties(
  util/crc32c.cc
-      PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
+      PROPERTIES COMPILE_FLAGS "-msse4.2")
  endif()
 endif()

@@ -1,31 +1,11 @@
 # Rocksdb Change Log
-## 5.10.5 (03/16/2018)
-### Bug fixes
-* Fix WAL corruption caused by race condition between user write thread and backup/checkpoint thread.
-
-## 5.10.4 (02/22/2018)
-### New Features
-* Follow rsync-style naming convention for BackupEngine tempfiles. This enables some optimizations when run on GlusterFS.
-* Fix regression of Java build break on Windows.
-
-## 5.10.3 (02/21/2018)
-### Bug fixes
-* Fix build break regression using gcc-7
-* Direct I/O writable file should do fsync in Close()
-
-### New Features
-* Add rocksdb.iterator.internal-key property
-
-## 5.10.1 (01/18/2018)
-### Bug Fixes
-* Fix DB::Flush() keep waiting after flush finish under certain condition.
+## Unreleased

 ## 5.10.0 (12/11/2017)
 ### Public API Change
 * When running `make` with environment variable `USE_SSE` set and `PORTABLE` unset, will use all machine features available locally. Previously this combination only compiled SSE-related features.

 ### New Features
-* CRC32C is now using the 3-way pipelined SSE algorithm `crc32c_3way` on supported platforms to improve performance. The system will choose to use this algorithm on supported platforms automatically whenever possible. If PCLMULQDQ is not supported it will fall back to the old Fast_CRC32 algorithm.
 * Provide lifetime hints when writing files on Linux. This reduces hardware write-amp on storage devices supporting multiple streams.
 * Add a DB stat, `NUMBER_ITER_SKIP`, which returns how many internal keys were skipped during iterations (e.g., due to being tombstones or duplicate versions of a key).
 * Add PerfContext counters, `key_lock_wait_count` and `key_lock_wait_time`, which measure the number of times transactions wait on key locks and total amount of time waiting.
@@ -107,41 +107,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
            * run `brew tap homebrew/versions; brew install gcc48 --use-llvm` to install gcc 4.8 (or higher).
    * run `brew install rocksdb`

-* **FreeBSD** (11.01):
-
-    * You can either install RocksDB from the Ports system using `cd /usr/ports/databases/rocksdb && make install`, or you can follow the details below to install dependencies and compile from source code:
-
-    * Install the dependencies for RocksDB:
-
-        export BATCH=YES
-        cd /usr/ports/devel/gmake && make install
-        cd /usr/ports/devel/gflags && make install
-
-        cd /usr/ports/archivers/snappy && make install
-        cd /usr/ports/archivers/bzip2 && make install
-        cd /usr/ports/archivers/liblz4 && make install
-        cd /usr/ports/archivesrs/zstd && make install
-
-        cd /usr/ports/devel/git && make install
-
-
-    * Install the dependencies for RocksJava (optional):
-
-        export BATCH=yes
-        cd /usr/ports/java/openjdk7 && make install
-
-    * Build RocksDB from source:
-        cd ~
-        git clone https://github.com/facebook/rocksdb.git
-        cd rocksdb
-        gmake static_lib
-
-    * Build RocksJava from source (optional):
-        cd rocksdb
-        export JAVA_HOME=/usr/local/openjdk7
-        gmake rocksdbjava
-
-
 * **iOS**:
  * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.

@@ -305,9 +305,6 @@ LDFLAGS += $(LUA_LIB)

 endif

-ifeq ($(NO_THREEWAY_CRC32C), 1)
-	CXXFLAGS += -DNO_THREEWAY_CRC32C
-endif

 CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
 CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers
@@ -344,8 +341,6 @@ ifeq ($(HAVE_POWER8),1)
 LIB_CC_OBJECTS = $(LIB_SOURCES:.cc=.o)
 LIBOBJECTS += $(LIB_SOURCES_C:.c=.o)
 LIBOBJECTS += $(LIB_SOURCES_ASM:.S=.o)
-else
-LIB_CC_OBJECTS = $(LIB_SOURCES:.cc=.o)
 endif

 LIBOBJECTS += $(TOOL_LIB_SOURCES:.cc=.o)
@@ -1573,7 +1568,7 @@ else
 endif
 endif
 ifeq ($(PLATFORM), OS_FREEBSD)
-	JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/freebsd
+	JAVA_INCLUDE += -I$(JAVA_HOME)/include/freebsd
 	ROCKSDBJNILIB = librocksdbjni-freebsd$(ARCH).so
 	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-freebsd$(ARCH).jar
 endif
@@ -1599,7 +1594,7 @@ libz.a:
 		exit 1; \
 	fi
 	tar xvzf zlib-$(ZLIB_VER).tar.gz
-	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && $(MAKE)
+	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && make
 	cp zlib-$(ZLIB_VER)/libz.a .

 libbz2.a:
@@ -1611,7 +1606,7 @@ libbz2.a:
 		exit 1; \
 	fi
 	tar xvzf bzip2-$(BZIP2_VER).tar.gz
-	cd bzip2-$(BZIP2_VER) && $(MAKE) CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
+	cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
 	cp bzip2-$(BZIP2_VER)/libbz2.a .

 libsnappy.a:
@@ -1624,7 +1619,7 @@ libsnappy.a:
 	fi
 	tar xvzf snappy-$(SNAPPY_VER).tar.gz
 	cd snappy-$(SNAPPY_VER) && CFLAGS='${EXTRA_CFLAGS}' CXXFLAGS='${EXTRA_CXXFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --with-pic --enable-static --disable-shared
-	cd snappy-$(SNAPPY_VER) && $(MAKE) ${SNAPPY_MAKE_TARGET}
+	cd snappy-$(SNAPPY_VER) && make ${SNAPPY_MAKE_TARGET}
 	cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a .

 liblz4.a:
@@ -1637,7 +1632,7 @@ liblz4.a:
 		exit 1; \
 	fi
 	tar xvzf lz4-$(LZ4_VER).tar.gz
-	cd lz4-$(LZ4_VER)/lib && $(MAKE) CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
+	cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
 	cp lz4-$(LZ4_VER)/lib/liblz4.a .

 libzstd.a:
@@ -1650,45 +1645,29 @@ libzstd.a:
 		exit 1; \
 	fi
 	tar xvzf zstd-$(ZSTD_VER).tar.gz
-	cd zstd-$(ZSTD_VER)/lib && DESTDIR=. PREFIX= $(MAKE) CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' install
+	cd zstd-$(ZSTD_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
 	cp zstd-$(ZSTD_VER)/lib/libzstd.a .

 # A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries
-java_static_libobjects = $(patsubst %,jls/%,$(LIB_CC_OBJECTS))
+java_static_libobjects = $(patsubst %,jls/%,$(LIBOBJECTS))
 CLEAN_FILES += jls
-java_static_all_libobjects = $(java_static_libobjects)

 ifneq ($(ROCKSDB_JAVA_NO_COMPRESSION), 1)
 JAVA_COMPRESSIONS = libz.a libbz2.a libsnappy.a liblz4.a libzstd.a
 endif

 JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4 -DZSTD
-JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib/include
-
-ifeq ($(HAVE_POWER8),1)
-JAVA_STATIC_C_LIBOBJECTS = $(patsubst %.c.o,jls/%.c.o,$(LIB_SOURCES_C:.c=.o))
-JAVA_STATIC_ASM_LIBOBJECTS = $(patsubst %.S.o,jls/%.S.o,$(LIB_SOURCES_ASM:.S=.o))
-
-java_static_ppc_libobjects = $(JAVA_STATIC_C_LIBOBJECTS) $(JAVA_STATIC_ASM_LIBOBJECTS)
-
-jls/util/crc32c_ppc.o: util/crc32c_ppc.c
-	$(AM_V_CC)$(CC) $(CFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -c $< -o $@
-
-jls/util/crc32c_ppc_asm.o: util/crc32c_ppc_asm.S
-	$(AM_V_CC)$(CC) $(CFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -c $< -o $@
-
-java_static_all_libobjects += $(java_static_ppc_libobjects)
-endif
+JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib

 $(java_static_libobjects): jls/%.o: %.cc $(JAVA_COMPRESSIONS)
 	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -fPIC -c $< -o $@ $(COVERAGEFLAGS)

-rocksdbjavastatic: $(java_static_all_libobjects)
+rocksdbjavastatic: $(java_static_libobjects)
 	cd java;$(MAKE) javalib;
 	rm -f ./java/target/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
-	  $(java_static_all_libobjects) $(COVERAGEFLAGS) \
+	  $(java_static_libobjects) $(COVERAGEFLAGS) \
 	  $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS)
 	cd java/target;strip $(STRIPFLAGS) $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
@@ -1749,7 +1728,7 @@ JAVA_C_LIBOBJECTS = $(patsubst %.c.o,jl/%.c.o,$(JAVA_C_OBJECTS))
 JAVA_ASM_LIBOBJECTS = $(patsubst %.S.o,jl/%.S.o,$(JAVA_ASM_OBJECTS))
 endif

-java_libobjects = $(patsubst %,jl/%,$(LIB_CC_OBJECTS))
+java_libobjects = $(patsubst %,jl/%,$(LIBOBJECTS))
 CLEAN_FILES += jl
 java_all_libobjects = $(java_libobjects)

@@ -157,6 +157,7 @@ cpp_library(
        "table/block_based_table_factory.cc",
        "table/block_based_table_reader.cc",
        "table/block_builder.cc",
+        "table/block_fetcher.cc",
        "table/block_prefix_index.cc",
        "table/bloom_block.cc",
        "table/cuckoo_table_builder.cc",
@@ -36,7 +36,7 @@ def parse_src_mk(repo_path):
 # get all .cc / .c files
 def get_cc_files(repo_path):
    cc_files = []
-    for root, dirnames, filenames in os.walk(repo_path):  # noqa: B007 T25377293 Grandfathered in
+    for root, dirnames, filenames in os.walk(repo_path):
        root = root[(len(repo_path) + 1):]
        if "java" in root:
            # Skip java
@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
 # Create a tmp directory for the test to use
 TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
-# shellcheck disable=SC2068
 TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"
@@ -51,13 +51,11 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
    FBCODE_BUILD="true"
    # If we're compiling with TSAN we need pic build
    PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      # we need this to build with MySQL. Don't use for other purposes.
-      source "$PWD/build_tools/fbcode_config4.8.1.sh"
-    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
+    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
      source "$PWD/build_tools/fbcode_config.sh"
    else
-      source "$PWD/build_tools/fbcode_config_platform007.sh"
+      # we need this to build with MySQL. Don't use for other purposes.
+      source "$PWD/build_tools/fbcode_config4.8.1.sh"
    fi
 fi

@@ -143,7 +141,6 @@ case "$TARGET_OS" in
        ;;
    FreeBSD)
        PLATFORM=OS_FREEBSD
-        CXX=clang++
        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
        # PORT_FILES=port/freebsd/freebsd_specific.cc
@@ -484,16 +481,13 @@ if test -z "$PORTABLE"; then
    COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
    COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^arm`"; then
-    # TODO: Handle this with approprite options.
-    COMMON_FLAGS="$COMMON_FLAGS"
  elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
    COMMON_FLAGS="$COMMON_FLAGS -march=native "
  elif test "$USE_SSE"; then
-    COMMON_FLAGS="$COMMON_FLAGS -msse4.2 -mpclmul"
+    COMMON_FLAGS="$COMMON_FLAGS -msse4.2"
  fi
 elif test "$USE_SSE"; then
-  COMMON_FLAGS="$COMMON_FLAGS -msse4.2 -mpclmul"
+  COMMON_FLAGS="$COMMON_FLAGS -msse4.2"
 fi

 $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
@@ -507,24 +501,6 @@ if [ "$?" = 0 ]; then
  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_SSE42"
 elif test "$USE_SSE"; then
  echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling"
-  exit 1
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <wmmintrin.h>
-  int main() {
-    const auto a = _mm_set_epi64x(0, 0);
-    const auto b = _mm_set_epi64x(0, 0);
-    const auto c = _mm_clmulepi64_si128(a, b, 0x00);
-    auto d = _mm_cvtsi128_si64(c);
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_PCLMUL"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use PCLMUL intrinsics, disabling"
-  exit 1
 fi

 # iOS doesn't support thread-local storage, but this check would erroneously
@@ -13,12 +13,10 @@ error=0

 function log {
  DATE=`date +%Y-%m-%d:%H:%M:%S`
-  # shellcheck disable=SC2068
  echo $DATE $@
 }

 function log_err {
-  # shellcheck disable=SC2145
  log "ERROR: $@ Error code: $error."
 }

@@ -1,4 +1,3 @@
-# shellcheck disable=SC2148
 GCC_BASE=/mnt/gvfs/third-party2/gcc/8219ec1bcedf8ad9da05e121e193364de2cc4f61/5.x/centos6-native/c447969
 CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/64d8d58e3d84f8bde7a029763d4f5baf39d0d5b9/stable/centos6-native/6aaf4de
 LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ba9be983c81de7299b59fe71950c664a84dcb5f8/5.x/gcc-5-glibc-2.23/339d858
@@ -1,4 +1,3 @@
-# shellcheck disable=SC2148
 GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
 CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
 LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
@@ -1,18 +0,0 @@
-GCC_BASE=/mnt/gvfs/third-party2/gcc/6e8e715624fd15256a7970073387793dfcf79b46/7.x/centos7-native/b2ef2b6
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/ef37e1faa1c29782abfac1ae65a291b9b7966f6d/stable/centos7-native/c9f9104
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/c67031f0f739ac61575a061518d6ef5038f99f90/7.x/platform007/5620abc
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/60d6f124a78798b73944f5ba87c2306ae3460153/2.26/platform007/f259413
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/22c2d65676fb7c23cfa797c4f6937f38b026f3cf/1.2.8/platform007/ca4da3d
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/907b498203d297947f3bb70b9466f47e100f1873/r131/platform007/ca4da3d
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/3ee276cbacfad3074e3f07bf826ac47f06970f4e/1.3.5/platform007/15a3614
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/9c910d36d6235cc40e8ff559358f1833452300ca/master/platform007/5b0f53e
-NUMA_BASE=/mnt/gvfs/third-party2/numa/9cbf2460284c669ed19c3ccb200a71f7dd7e53c7/2.0.11/platform007/ca4da3d
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/bf3d7497fe4e6d007354f0adffa16ce3003f8338/1.3/platform007/6f3e0a9
-TBB_BASE=/mnt/gvfs/third-party2/tbb/ff4e0b093534704d8abab678a4fd7f5ea7b094c7/2018_U5/platform007/ca4da3d
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/b5c4a61a5c483ba24722005ae07895971a2ac707/fb/platform007/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/92ff90349e2f43ea0a8246d8b1cf17b6869013e3/2.29.1/centos7-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/f3f697a28122e6bcd513273dd9c1ff23852fc59f/3.13.0/platform007/ca4da3d
-LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
@@ -1,157 +0,0 @@
-#!/bin/sh
-#
-# Set environment variables so that we can compile rocksdb using
-# fbcode settings.  It uses the latest g++ and clang compilers and also
-# uses jemalloc
-# Environment variables that change the behavior of this script:
-# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
-
-
-BASEDIR=`dirname $BASH_SOURCE`
-source "$BASEDIR/dependencies_platform007.sh"
-
-CFLAGS=""
-
-# libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
-
-# glibc
-GLIBC_INCLUDE="$GLIBC_BASE/include"
-GLIBC_LIBS=" -L $GLIBC_BASE/lib"
-
-# snappy
-SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
-if test -z $PIC_BUILD; then
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
-else
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
-fi
-CFLAGS+=" -DSNAPPY"
-
-if test -z $PIC_BUILD; then
-  # location of zlib headers and libraries
-  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
-  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
-  CFLAGS+=" -DZLIB"
-
-  # location of bzip headers and libraries
-  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
-  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
-  CFLAGS+=" -DBZIP2"
-
-  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
-  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
-  CFLAGS+=" -DLZ4"
-fi
-
-ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-if test -z $PIC_BUILD; then
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-else
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
-fi
-CFLAGS+=" -DZSTD"
-
-# location of gflags headers and libraries
-GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
-if test -z $PIC_BUILD; then
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
-else
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
-fi
-CFLAGS+=" -DGFLAGS=gflags"
-
-# location of jemalloc
-JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
-JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
-
-if test -z $PIC_BUILD; then
-  # location of numa
-  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
-  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
-  CFLAGS+=" -DNUMA"
-
-  # location of libunwind
-  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
-fi
-
-# location of TBB
-TBB_INCLUDE=" -isystem $TBB_BASE/include/"
-if test -z $PIC_BUILD; then
-  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
-else
-  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
-fi
-CFLAGS+=" -DTBB"
-
-# use Intel SSE support for checksum calculations
-export USE_SSE=1
-export PORTABLE=1
-
-BINUTILS="$BINUTILS_BASE/bin"
-AR="$BINUTILS/ar"
-
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
-
-STDLIBS="-L $GCC_BASE/lib64"
-
-CLANG_BIN="$CLANG_BASE/bin"
-CLANG_LIB="$CLANG_BASE/lib"
-CLANG_SRC="$CLANG_BASE/../../src"
-
-CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
-
-if [ -z "$USE_CLANG" ]; then
-  # gcc
-  CC="$GCC_BASE/bin/gcc"
-  CXX="$GCC_BASE/bin/g++"
-
-  CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  JEMALLOC=1
-else
-  # clang
-  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
-  CC="$CLANG_BIN/clang"
-  CXX="$CLANG_BIN/clang++"
-
-  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
-
-  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $CLANG_INCLUDE"
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
-  CFLAGS+=" -Wno-expansion-to-defined "
-  CXXFLAGS="-nostdinc++"
-fi
-
-CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
-CXXFLAGS+=" $CFLAGS"
-
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
-EXEC_LDFLAGS+=" -B$BINUTILS/gold"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
-EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
-# required by libtbb
-EXEC_LDFLAGS+=" -ldl"
-
-PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
-
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
-
-VALGRIND_VER="$VALGRIND_BASE/bin/"
-
-# lua not supported because it's on track for deprecation, I think
-LUA_PATH=
-LUA_LIB=
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
@@ -1,4 +1,3 @@
-# shellcheck disable=SC1113
 #/usr/bin/env bash

 set -e
@@ -29,14 +28,12 @@ function package() {
    if dpkg --get-selections | grep --quiet $1; then
      log "$1 is already installed. skipping."
    else
-      # shellcheck disable=SC2068
      apt-get install $@ -y
    fi
  elif [[ $OS = "centos" ]]; then
    if rpm -qa | grep --quiet $1; then
      log "$1 is already installed. skipping."
    else
-      # shellcheck disable=SC2068
      yum install $@ -y
    fi
  fi
@@ -55,7 +52,6 @@ function gem_install() {
  if gem list | grep --quiet $1; then
    log "$1 is already installed. skipping."
  else
-    # shellcheck disable=SC2068
    gem install $@
  fi
 }
@@ -129,5 +125,4 @@ function main() {
    include $LIB_DIR
 }

-# shellcheck disable=SC2068
 main $@
@@ -85,9 +85,8 @@ NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
 GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
 ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
-# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090.
-# using platform007 gives us gcc-8 or higher which has that bug fixed.
-TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1"
+LITE="OPT=\"-DROCKSDB_LITE -g\""
+TSAN="COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
@@ -53,45 +53,6 @@ function get_lib_base()
  log_variable $__res_var
 }

-###########################################################
-#                platform007 dependencies                 #
-###########################################################
-
-OUTPUT="$BASEDIR/dependencies_platform007.sh"
-
-rm -f "$OUTPUT"
-touch "$OUTPUT"
-
-echo "Writing dependencies to $OUTPUT"
-
-# Compilers locations
-GCC_BASE=`readlink -f $TP2_LATEST/gcc/7.x/centos7-native/*/`
-CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
-
-log_variable GCC_BASE
-log_variable CLANG_BASE
-
-# Libraries locations
-get_lib_base libgcc     7.x     platform007
-get_lib_base glibc      2.26    platform007
-get_lib_base snappy     LATEST  platform007
-get_lib_base zlib       LATEST  platform007
-get_lib_base bzip2      LATEST  platform007
-get_lib_base lz4        LATEST  platform007
-get_lib_base zstd       LATEST  platform007
-get_lib_base gflags     LATEST  platform007
-get_lib_base jemalloc   LATEST  platform007
-get_lib_base numa       LATEST  platform007
-get_lib_base libunwind  LATEST  platform007
-get_lib_base tbb        LATEST  platform007
-
-get_lib_base kernel-headers fb platform007
-get_lib_base binutils   LATEST centos7-native
-get_lib_base valgrind   LATEST platform007
-get_lib_base lua        5.3.4  platform007
-
-git diff $OUTPUT
-
 ###########################################################
 #                   5.x dependencies                      #
 ###########################################################
@@ -72,7 +72,7 @@ def display_file_coverage(per_file_coverage, total_coverage):
    header_template = \
        "%" + str(max_file_name_length) + "s\t%s\t%s"
    separator = "-" * (max_file_name_length + 10 + 20)
-    print header_template % ("Filename", "Coverage", "Lines")  # noqa: E999 T25377293 Grandfathered in
+    print header_template % ("Filename", "Coverage", "Lines")
    print separator

    # -- Print body
@@ -1388,24 +1388,23 @@ void rocksdb_writebatch_put_log_data(
  b->rep.PutLogData(Slice(blob, len));
 }

-class H : public WriteBatch::Handler {
- public:
-  void* state_;
-  void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-  void (*deleted_)(void*, const char* k, size_t klen);
-  virtual void Put(const Slice& key, const Slice& value) override {
-    (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-  }
-  virtual void Delete(const Slice& key) override {
-    (*deleted_)(state_, key.data(), key.size());
-  }
-};
-
 void rocksdb_writebatch_iterate(
    rocksdb_writebatch_t* b,
    void* state,
    void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
    void (*deleted)(void*, const char* k, size_t klen)) {
+  class H : public WriteBatch::Handler {
+   public:
+    void* state_;
+    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+    void (*deleted_)(void*, const char* k, size_t klen);
+    virtual void Put(const Slice& key, const Slice& value) override {
+      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+    }
+    virtual void Delete(const Slice& key) override {
+      (*deleted_)(state_, key.data(), key.size());
+    }
+  };
  H handler;
  handler.state_ = state;
  handler.put_ = put;
@@ -1650,6 +1649,18 @@ void rocksdb_writebatch_wi_iterate(
    void* state,
    void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
    void (*deleted)(void*, const char* k, size_t klen)) {
+  class H : public WriteBatch::Handler {
+   public:
+    void* state_;
+    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+    void (*deleted_)(void*, const char* k, size_t klen);
+    virtual void Put(const Slice& key, const Slice& value) override {
+      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+    }
+    virtual void Delete(const Slice& key) override {
+      (*deleted_)(state_, key.data(), key.size());
+    }
+  };
  H handler;
  handler.state_ = state;
  handler.put_ = put;
@@ -3093,21 +3104,20 @@ void rocksdb_slicetransform_destroy(rocksdb_slicetransform_t* st) {
  delete st;
 }

-struct Wrapper : public rocksdb_slicetransform_t {
-  const SliceTransform* rep_;
-  ~Wrapper() { delete rep_; }
-  const char* Name() const override { return rep_->Name(); }
-  Slice Transform(const Slice& src) const override {
-    return rep_->Transform(src);
-  }
-  bool InDomain(const Slice& src) const override {
-    return rep_->InDomain(src);
-  }
-  bool InRange(const Slice& src) const override { return rep_->InRange(src); }
-  static void DoNothing(void*) { }
-};
-
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t prefixLen) {
+  struct Wrapper : public rocksdb_slicetransform_t {
+    const SliceTransform* rep_;
+    ~Wrapper() { delete rep_; }
+    const char* Name() const override { return rep_->Name(); }
+    Slice Transform(const Slice& src) const override {
+      return rep_->Transform(src);
+    }
+    bool InDomain(const Slice& src) const override {
+      return rep_->InDomain(src);
+    }
+    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+    static void DoNothing(void*) { }
+  };
  Wrapper* wrapper = new Wrapper;
  wrapper->rep_ = rocksdb::NewFixedPrefixTransform(prefixLen);
  wrapper->state_ = nullptr;
@@ -3116,6 +3126,19 @@ rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t pref
 }

 rocksdb_slicetransform_t* rocksdb_slicetransform_create_noop() {
+  struct Wrapper : public rocksdb_slicetransform_t {
+    const SliceTransform* rep_;
+    ~Wrapper() { delete rep_; }
+    const char* Name() const override { return rep_->Name(); }
+    Slice Transform(const Slice& src) const override {
+      return rep_->Transform(src);
+    }
+    bool InDomain(const Slice& src) const override {
+      return rep_->InDomain(src);
+    }
+    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+    static void DoNothing(void*) { }
+  };
  Wrapper* wrapper = new Wrapper;
  wrapper->rep_ = rocksdb::NewNoopTransform();
  wrapper->state_ = nullptr;
@@ -344,13 +344,12 @@ void SuperVersionUnrefHandle(void* ptr) {
  // When latter happens, we are in ~ColumnFamilyData(), no get should happen as
  // well.
  SuperVersion* sv = static_cast<SuperVersion*>(ptr);
-  bool was_last_ref __attribute__((__unused__));
-  was_last_ref = sv->Unref();
-  // Thread-local SuperVersions can't outlive ColumnFamilyData::super_version_.
-  // This is important because we can't do SuperVersion cleanup here.
-  // That would require locking DB mutex, which would deadlock because
-  // SuperVersionUnrefHandle is called with locked ThreadLocalPtr mutex.
-  assert(!was_last_ref);
+  if (sv->Unref()) {
+    sv->db_mutex->Lock();
+    sv->Cleanup();
+    sv->db_mutex->Unlock();
+    delete sv;
+  }
 }
 }  // anonymous namespace

@@ -386,8 +385,7 @@ ColumnFamilyData::ColumnFamilyData(
      pending_flush_(false),
      pending_compaction_(false),
      prev_compaction_needed_bytes_(0),
-      allow_2pc_(db_options.allow_2pc),
-      last_memtable_id_(0) {
+      allow_2pc_(db_options.allow_2pc) {
  Ref();

  // Convert user defined table properties collector factories to internal ones.
@@ -967,12 +965,6 @@ void ColumnFamilyData::InstallSuperVersion(
      RecalculateWriteStallConditions(mutable_cf_options);

  if (old_superversion != nullptr) {
-    // Reset SuperVersions cached in thread local storage.
-    // This should be done before old_superversion->Unref(). That's to ensure
-    // that local_sv_ never holds the last reference to SuperVersion, since
-    // it has no means to safely do SuperVersion cleanup.
-    ResetThreadLocalSuperVersions();
-
    if (old_superversion->mutable_cf_options.write_buffer_size !=
        mutable_cf_options.write_buffer_size) {
      mem_->UpdateWriteBufferSize(mutable_cf_options.write_buffer_size);
@@ -988,6 +980,9 @@ void ColumnFamilyData::InstallSuperVersion(
      sv_context->superversions_to_free.push_back(old_superversion);
    }
  }
+
+  // Reset SuperVersions cached in thread local storage
+  ResetThreadLocalSuperVersions();
 }

 void ColumnFamilyData::ResetThreadLocalSuperVersions() {
@@ -999,12 +994,10 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
      continue;
    }
    auto sv = static_cast<SuperVersion*>(ptr);
-    bool was_last_ref __attribute__((__unused__));
-    was_last_ref = sv->Unref();
-    // sv couldn't have been the last reference because
-    // ResetThreadLocalSuperVersions() is called before
-    // unref'ing super_version_.
-    assert(!was_last_ref);
+    if (sv->Unref()) {
+      sv->Cleanup();
+      delete sv;
+    }
  }
 }

@@ -239,11 +239,7 @@ class ColumnFamilyData {
  void SetCurrent(Version* _current);
  uint64_t GetNumLiveVersions() const;  // REQUIRE: DB mutex held
  uint64_t GetTotalSstFilesSize() const;  // REQUIRE: DB mutex held
-  void SetMemtable(MemTable* new_mem) {
-    uint64_t memtable_id = last_memtable_id_.fetch_add(1) + 1;
-    new_mem->SetID(memtable_id);
-    mem_ = new_mem;
-  }
+  void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }

  // calculate the oldest log needed for the durability of this column family
  uint64_t OldestLogToKeep();
@@ -423,9 +419,6 @@ class ColumnFamilyData {

  // if the database was opened with 2pc enabled
  bool allow_2pc_;
-
-  // Memtable id to track flush.
-  std::atomic<uint64_t> last_memtable_id_;
 };

 // ColumnFamilySet has interesting thread-safety requirements
@@ -184,8 +184,11 @@ void CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
                                              Slice* skip_until) {
  if (compaction_filter_ != nullptr &&
      (ikey_.type == kTypeValue || ikey_.type == kTypeBlobIndex) &&
-      (visible_at_tip_ || ikey_.sequence > latest_snapshot_ ||
-       ignore_snapshots_)) {
+      (visible_at_tip_ || ignore_snapshots_ ||
+       ikey_.sequence > latest_snapshot_ ||
+       (snapshot_checker_ != nullptr &&
+        UNLIKELY(!snapshot_checker_->IsInSnapshot(ikey_.sequence,
+                                                  latest_snapshot_))))) {
    // If the user has specified a compaction filter and the sequence
    // number is greater than any external snapshot, then invoke the
    // filter. If the return value of the compaction filter is true,
@@ -411,7 +414,10 @@ void CompactionIterator::NextFromInput() {
          cmp_->Equal(ikey_.user_key, next_ikey.user_key)) {
        // Check whether the next key belongs to the same snapshot as the
        // SingleDelete.
-        if (prev_snapshot == 0 || next_ikey.sequence > prev_snapshot) {
+        if (prev_snapshot == 0 || next_ikey.sequence > prev_snapshot ||
+            (snapshot_checker_ != nullptr &&
+             UNLIKELY(!snapshot_checker_->IsInSnapshot(next_ikey.sequence,
+                                                       prev_snapshot)))) {
          if (next_ikey.type == kTypeSingleDeletion) {
            // We encountered two SingleDeletes in a row.  This could be due to
            // unexpected user input.
@@ -422,8 +428,12 @@ void CompactionIterator::NextFromInput() {
            // input_->Next().
            ++iter_stats_.num_record_drop_obsolete;
            ++iter_stats_.num_single_del_mismatch;
-          } else if ((ikey_.sequence <= earliest_write_conflict_snapshot_) ||
-                     has_outputted_key_) {
+          } else if (has_outputted_key_ ||
+                     (ikey_.sequence <= earliest_write_conflict_snapshot_ &&
+                      (snapshot_checker_ == nullptr ||
+                       LIKELY(snapshot_checker_->IsInSnapshot(
+                           ikey_.sequence,
+                           earliest_write_conflict_snapshot_))))) {
            // Found a matching value, we can drop the single delete and the
            // value.  It is safe to drop both records since we've already
            // outputted a key in this snapshot, or there is no earlier
@@ -471,6 +481,9 @@ void CompactionIterator::NextFromInput() {
        // comparison, so the value of has_current_user_key does not matter.
        has_current_user_key_ = false;
        if (compaction_ != nullptr && ikey_.sequence <= earliest_snapshot_ &&
+            (snapshot_checker_ == nullptr ||
+             LIKELY(snapshot_checker_->IsInSnapshot(ikey_.sequence,
+                                                    earliest_snapshot_))) &&
            compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
                                                       &level_ptrs_)) {
          // Key doesn't exist outside of this range.
@@ -504,6 +517,9 @@ void CompactionIterator::NextFromInput() {
      input_->Next();
    } else if (compaction_ != nullptr && ikey_.type == kTypeDeletion &&
               ikey_.sequence <= earliest_snapshot_ &&
+               (snapshot_checker_ == nullptr ||
+                LIKELY(snapshot_checker_->IsInSnapshot(ikey_.sequence,
+                                                       earliest_snapshot_))) &&
               ikeyNotNeededForIncrementalSnapshot() &&
               compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
                                                          &level_ptrs_)) {
@@ -538,6 +554,10 @@ void CompactionIterator::NextFromInput() {
      // have hit (A)
      // We encapsulate the merge related state machine in a different
      // object to minimize change to the existing flow.
+      // In case snapshot_checker is present, we can probably merge further
+      // beyond prev_snapshot, since there could be more keys with sequence
+      // smaller than prev_snapshot, but reported by snapshot_checker as not
+      // visible by prev_snapshot. But it will make the logic more complicated.
      Status s = merge_helper_->MergeUntil(input_, range_del_agg_,
                                           prev_snapshot, bottommost_level_);
      merge_out_iter_.SeekToFirst();
@@ -625,8 +625,8 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
      "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
      "files in(%d, %d) out(%d) "
      "MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
-      "write-amplify(%.1f) %s, records in: %" PRIu64
-      ", records dropped: %" PRIu64 " output_compression: %s\n",
+      "write-amplify(%.1f) %s, records in: %d, records dropped: %d "
+      "output_compression: %s\n",
      cfd->GetName().c_str(), vstorage->LevelSummary(&tmp), bytes_read_per_sec,
      bytes_written_per_sec, compact_->compaction->output_level(),
      stats.num_input_files_in_non_output_levels,
@@ -126,41 +126,6 @@ TEST_F(DBFlushTest, FlushInLowPriThreadPool) {
  ASSERT_EQ(1, num_compactions);
 }

-TEST_F(DBFlushTest, ManualFlushWithMinWriteBufferNumberToMerge) {
-  Options options = CurrentOptions();
-  options.write_buffer_size = 100;
-  options.max_write_buffer_number = 4;
-  options.min_write_buffer_number_to_merge = 3;
-  Reopen(options);
-
-  SyncPoint::GetInstance()->LoadDependency(
-      {{"DBImpl::BGWorkFlush",
-        "DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:1"},
-       {"DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:2",
-        "DBImpl::FlushMemTableToOutputFile:BeforeInstallSV"}});
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(Put("key1", "value1"));
-
-  port::Thread t([&]() {
-    // The call wait for flush to finish, i.e. with flush_options.wait = true.
-    ASSERT_OK(Flush());
-  });
-
-  // Wait for flush start.
-  TEST_SYNC_POINT("DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:1");
-  // Insert a second memtable before the manual flush finish.
-  // At the end of the manual flush job, it will check if further flush
-  // is needed, but it will not trigger flush of the second memtable because
-  // min_write_buffer_number_to_merge is not reached.
-  ASSERT_OK(Put("key2", "value2"));
-  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
-  TEST_SYNC_POINT("DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:2");
-
-  // Manual flush should return, without waiting for flush indefinitely.
-  t.join();
-}
-
 TEST_P(DBFlushDirectIOTest, DirectIO) {
  Options options;
  options.create_if_missing = true;
@@ -810,12 +810,8 @@ class DBImpl : public DB {
  Status FlushMemTable(ColumnFamilyData* cfd, const FlushOptions& options,
                       bool writes_stopped = false);

-  // Wait for memtable flushed.
-  // If flush_memtable_id is non-null, wait until the memtable with the ID
-  // gets flush. Otherwise, wait until the column family don't have any
-  // memtable pending flush.
-  Status WaitForFlushMemTable(ColumnFamilyData* cfd,
-                              const uint64_t* flush_memtable_id = nullptr);
+  // Wait for memtable flushed
+  Status WaitForFlushMemTable(ColumnFamilyData* cfd);

  // REQUIRES: mutex locked
  Status SwitchWAL(WriteContext* write_context);
@@ -134,7 +134,6 @@ Status DBImpl::FlushMemTableToOutputFile(
  }

  if (s.ok()) {
-    TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:BeforeInstallSV");
    InstallSuperVersionAndScheduleWork(cfd, &job_context->superversion_context,
                                       mutable_cf_options);
    if (made_progress) {
@@ -810,13 +809,7 @@ int DBImpl::Level0StopWriteTrigger(ColumnFamilyHandle* column_family) {
 Status DBImpl::Flush(const FlushOptions& flush_options,
                     ColumnFamilyHandle* column_family) {
  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
-  ROCKS_LOG_INFO(immutable_db_options_.info_log, "[%s] Manual flush start.",
-                 cfh->GetName().c_str());
-  Status s = FlushMemTable(cfh->cfd(), flush_options);
-  ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                 "[%s] Manual flush finished, status: %s\n",
-                 cfh->GetName().c_str(), s.ToString().c_str());
-  return s;
+  return FlushMemTable(cfh->cfd(), flush_options);
 }

 Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
@@ -951,7 +944,6 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
                             const FlushOptions& flush_options,
                             bool writes_stopped) {
  Status s;
-  uint64_t flush_memtable_id = 0;
  {
    WriteContext context;
    InstrumentedMutexLock guard_lock(&mutex_);
@@ -969,7 +961,6 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,

    // SwitchMemtable() will release and reacquire mutex during execution
    s = SwitchMemtable(cfd, &context);
-    flush_memtable_id = cfd->imm()->GetLatestMemTableID();

    if (!writes_stopped) {
      write_thread_.ExitUnbatched(&w);
@@ -984,19 +975,16 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,

  if (s.ok() && flush_options.wait) {
    // Wait until the compaction completes
-    s = WaitForFlushMemTable(cfd, &flush_memtable_id);
+    s = WaitForFlushMemTable(cfd);
  }
  return s;
 }

-Status DBImpl::WaitForFlushMemTable(ColumnFamilyData* cfd,
-                                    const uint64_t* flush_memtable_id) {
+Status DBImpl::WaitForFlushMemTable(ColumnFamilyData* cfd) {
  Status s;
  // Wait until the compaction completes
  InstrumentedMutexLock l(&mutex_);
-  while (cfd->imm()->NumNotFlushed() > 0 && bg_error_.ok() &&
-         (flush_memtable_id == nullptr ||
-          cfd->imm()->GetEarliestMemTableID() <= *flush_memtable_id)) {
+  while (cfd->imm()->NumNotFlushed() > 0 && bg_error_.ok()) {
    if (shutting_down_.load(std::memory_order_acquire)) {
      return Status::ShutdownInProgress();
    }
@@ -521,9 +521,6 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
  PERF_TIMER_GUARD(write_pre_and_post_process_time);
  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
                        true /* disable_memtable */, pre_release_callback);
-  if (write_options.disableWAL) {
-    return status;
-  }
  RecordTick(stats_, WRITE_WITH_WAL);
  StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);

@@ -580,7 +577,13 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
  // LastAllocatedSequence is increased inside WriteToWAL under
  // wal_write_mutex_ to ensure ordered events in WAL
  size_t seq_inc = seq_per_batch_ ? write_group.size : 0 /*total_count*/;
-  status = ConcurrentWriteToWAL(write_group, log_used, &last_sequence, seq_inc);
+  if (!write_options.disableWAL) {
+    status =
+        ConcurrentWriteToWAL(write_group, log_used, &last_sequence, seq_inc);
+  } else {
+    // Otherwise we inc seq number to do solely the seq allocation
+    last_sequence = versions_->FetchAddLastAllocatedSequence(seq_inc);
+  }
  auto curr_seq = last_sequence + 1;
  for (auto* writer : write_group) {
    if (writer->CallbackFailed()) {
@@ -593,6 +596,7 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
    // else seq advances only by memtable writes
  }
  if (status.ok() && write_options.sync) {
+    assert(!write_options.disableWAL);
    // Requesting sync with two_write_queues_ is expected to be very rare. We
    // hance provide a simple implementation that is not necessarily efficient.
    if (manual_wal_flush_) {
@@ -217,9 +217,6 @@ class DBIter final: public Iterator {
        *prop = "Iterator is not valid.";
      }
      return Status::OK();
-    } else if (prop_name == "rocksdb.iterator.internal-key") {
-      *prop = saved_key_.GetUserKey().ToString();
-      return Status::OK();
    }
    return Status::InvalidArgument("Undentified property.");
  }
@@ -55,7 +55,6 @@ TEST_F(DBIteratorTest, IteratorProperty) {
  Options options = CurrentOptions();
  CreateAndReopenWithCF({"pikachu"}, options);
  Put(1, "1", "2");
-  Delete(1, "2");
  ReadOptions ropt;
  ropt.pin_data = false;
  {
@@ -65,15 +64,9 @@ TEST_F(DBIteratorTest, IteratorProperty) {
    ASSERT_NOK(iter->GetProperty("non_existing.value", &prop_value));
    ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
    ASSERT_EQ("0", prop_value);
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value));
-    ASSERT_EQ("1", prop_value);
    iter->Next();
    ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
    ASSERT_EQ("Iterator is not valid.", prop_value);
-
-    // Get internal key at which the iteration stopped (tombstone in this case).
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value));
-    ASSERT_EQ("2", prop_value);
  }
  Close();
 }
@@ -2164,48 +2157,6 @@ TEST_F(DBIteratorTest, SkipStatistics) {
  ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
 }

-TEST_F(DBIteratorTest, SeekAfterHittingManyInternalKeys) {
-  Options options = CurrentOptions();
-  DestroyAndReopen(options);
-  ReadOptions ropts;
-  ropts.max_skippable_internal_keys = 2;
-
-  Put("1", "val_1");
-  // Add more tombstones than max_skippable_internal_keys so that Next() fails.
-  Delete("2");
-  Delete("3");
-  Delete("4");
-  Delete("5");
-  Put("6", "val_6");
-
-  unique_ptr<Iterator> iter(db_->NewIterator(ropts));
-  iter->SeekToFirst();
-
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().ToString(), "1");
-  ASSERT_EQ(iter->value().ToString(), "val_1");
-
-  // This should fail as incomplete due to too many non-visible internal keys on
-  // the way to the next valid user key.
-  iter->Next();
-  ASSERT_TRUE(!iter->Valid());
-  ASSERT_TRUE(iter->status().IsIncomplete());
-
-  // Get the internal key at which Next() failed.
-  std::string prop_value;
-  ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value));
-  ASSERT_EQ("4", prop_value);
-
-  // Create a new iterator to seek to the internal key.
-  unique_ptr<Iterator> iter2(db_->NewIterator(ropts));
-  iter2->Seek(prop_value);
-  ASSERT_TRUE(iter2->Valid());
-  ASSERT_OK(iter2->status());
-
-  ASSERT_EQ(iter2->key().ToString(), "6");
-  ASSERT_EQ(iter2->value().ToString(), "val_6");
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -376,6 +376,30 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }

+TEST_F(DBSSTTest, OpenDBWithExistingTrash) {
+  Options options = CurrentOptions();
+
+  options.sst_file_manager.reset(
+      NewSstFileManager(env_, nullptr, "", 1024 * 1024 /* 1 MB/sec */));
+  auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
+
+  Destroy(last_options_);
+
+  // Add some trash files to the db directory so the DB can clean them up
+  env_->CreateDirIfMissing(dbname_);
+  ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "001.sst.trash"));
+  ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "002.sst.trash"));
+  ASSERT_OK(WriteStringToFile(env_, "abc", dbname_ + "/" + "003.sst.trash"));
+
+  // Reopen the DB and verify that it deletes existing trash files
+  ASSERT_OK(TryReopen(options));
+  sfm->WaitForEmptyTrash();
+  ASSERT_NOK(env_->FileExists(dbname_ + "/" + "001.sst.trash"));
+  ASSERT_NOK(env_->FileExists(dbname_ + "/" + "002.sst.trash"));
+  ASSERT_NOK(env_->FileExists(dbname_ + "/" + "003.sst.trash"));
+}
+
+
 // Create a DB with 2 db_paths, and generate multiple files in the 2
 // db_paths using CompactRangeOptions, make sure that files that were
 // deleted from first db_path were deleted using DeleteScheduler and
@@ -4375,7 +4375,6 @@ TEST_F(DBTest, DynamicFIFOCompactionOptions) {
  ASSERT_EQ(dbfull()->GetOptions().compaction_options_fifo.allow_compaction,
            true);
 }
-#endif  // ROCKSDB_LITE

 TEST_F(DBTest, DynamicUniversalCompactionOptions) {
  Options options;
@@ -4444,6 +4443,7 @@ TEST_F(DBTest, DynamicUniversalCompactionOptions) {
      dbfull()->GetOptions().compaction_options_universal.allow_trivial_move,
      false);
 }
+#endif  // ROCKSDB_LITE

 TEST_F(DBTest, FileCreationRandomFailure) {
  Options options;
@@ -5651,50 +5651,6 @@ TEST_F(DBTest, PauseBackgroundWorkTest) {
  // now it's done
  ASSERT_TRUE(done.load());
 }
-
-// Keep spawning short-living threads that create an iterator and quit.
-// Meanwhile in another thread keep flushing memtables.
-// This used to cause a deadlock.
-TEST_F(DBTest, ThreadLocalPtrDeadlock) {
-  std::atomic<int> flushes_done{0};
-  std::atomic<int> threads_destroyed{0};
-  auto done = [&] {
-    return flushes_done.load() > 10;
-  };
-
-  std::thread flushing_thread([&] {
-    for (int i = 0; !done(); ++i) {
-      ASSERT_OK(db_->Put(WriteOptions(), Slice("hi"),
-                         Slice(std::to_string(i).c_str())));
-      ASSERT_OK(db_->Flush(FlushOptions()));
-      int cnt = ++flushes_done;
-      fprintf(stderr, "Flushed %d times\n", cnt);
-    }
-  });
-
-  std::vector<std::thread> thread_spawning_threads(10);
-  for (auto& t: thread_spawning_threads) {
-    t = std::thread([&] {
-      while (!done()) {
-        {
-          std::thread tmp_thread([&] {
-            auto it = db_->NewIterator(ReadOptions());
-            delete it;
-          });
-          tmp_thread.join();
-        }
-        ++threads_destroyed;
-      }
-    });
-  }
-
-  for (auto& t: thread_spawning_threads) {
-    t.join();
-  }
-  flushing_thread.join();
-  fprintf(stderr, "Done. Flushed %d times, destroyed %d threads\n",
-          flushes_done.load(), threads_destroyed.load());
-}
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -54,7 +54,6 @@ class Reader {
  // The Reader will start reading at the first record located at physical
  // position >= initial_offset within the file.
  Reader(std::shared_ptr<Logger> info_log,
-  // @lint-ignore TXT2 T25377293 Grandfathered in
 	 unique_ptr<SequentialFileReader>&& file,
         Reporter* reporter, bool checksum, uint64_t initial_offset,
         uint64_t log_num);
@@ -368,11 +368,6 @@ class MemTable {
    return oldest_key_time_.load(std::memory_order_relaxed);
  }

-  // REQUIRES: db_mutex held.
-  void SetID(uint64_t id) { id_ = id; }
-
-  uint64_t GetID() const { return id_; }
-
 private:
  enum FlushStateEnum { FLUSH_NOT_REQUESTED, FLUSH_REQUESTED, FLUSH_SCHEDULED };

@@ -442,9 +437,6 @@ class MemTable {
  // Timestamp of oldest key
  std::atomic<uint64_t> oldest_key_time_;

-  // Memtable id to track flush.
-  uint64_t id_ = 0;
-
  // Returns a heuristic flush decision
  bool ShouldFlushNow() const;

@@ -5,12 +5,11 @@
 //
 #pragma once

-#include <deque>
-#include <limits>
-#include <list>
-#include <set>
 #include <string>
+#include <list>
 #include <vector>
+#include <set>
+#include <deque>

 #include "db/dbformat.h"
 #include "db/memtable.h"
@@ -245,22 +244,6 @@ class MemTableList {

  uint64_t GetMinLogContainingPrepSection();

-  uint64_t GetEarliestMemTableID() const {
-    auto& memlist = current_->memlist_;
-    if (memlist.empty()) {
-      return std::numeric_limits<uint64_t>::max();
-    }
-    return memlist.back()->GetID();
-  }
-
-  uint64_t GetLatestMemTableID() const {
-    auto& memlist = current_->memlist_;
-    if (memlist.empty()) {
-      return 0;
-    }
-    return memlist.front()->GetID();
-  }
-
 private:
  // DB mutex held
  void InstallNewVersion();
@@ -1,4 +1,3 @@
-# shellcheck disable=SC2148
 export USE_HDFS=1
 export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64:/usr/lib/hadoop/lib/native

@@ -97,9 +97,6 @@ class Iterator : public Cleanable {
  // Property "rocksdb.iterator.super-version-number":
  //   LSM version used by the iterator. The same format as DB Property
  //   kCurrentSuperVersionNumber. See its comment for more information.
-  // Property "rocksdb.iterator.internal-key":
-  //   Get the user-key portion of the internal key at which the iteration
-  //   stopped.
  virtual Status GetProperty(std::string prop_name, std::string* prop);

 private:
@@ -4,7 +4,6 @@

 #pragma once
 #include <map>
-#include <memory>
 #include <string>
 #include "rocksdb/db.h"

@@ -19,20 +18,11 @@ namespace rocksdb {
 // This class contains APIs to stack rocksdb wrappers.Eg. Stack TTL over base d
 class StackableDB : public DB {
 public:
-  // StackableDB take sole ownership of the underlying db.
+  // StackableDB is the owner of db now!
  explicit StackableDB(DB* db) : db_(db) {}

-  // StackableDB take shared ownership of the underlying db.
-  explicit StackableDB(std::shared_ptr<DB> db)
-      : db_(db.get()), shared_db_ptr_(db) {}
-
  ~StackableDB() {
-    if (shared_db_ptr_ == nullptr) {
-      delete db_;
-    } else {
-      assert(shared_db_ptr_.get() == db_);
-    }
-    db_ = nullptr;
+    delete db_;
  }

  virtual DB* GetBaseDB() {
@@ -383,7 +373,6 @@ class StackableDB : public DB {

 protected:
  DB* db_;
-  std::shared_ptr<DB> shared_db_ptr_;
 };

 } //  namespace rocksdb
@@ -6,7 +6,7 @@

 #define ROCKSDB_MAJOR 5
 #define ROCKSDB_MINOR 10
-#define ROCKSDB_PATCH 5
+#define ROCKSDB_PATCH 0

 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these
@@ -7,8 +7,6 @@ set(JNI_NATIVE_SOURCES
        rocksjni/clock_cache.cc
        rocksjni/columnfamilyhandle.cc
        rocksjni/compaction_filter.cc
-        rocksjni/compaction_filter_factory.cc
-        rocksjni/compaction_filter_factory_jnicallback.cc
        rocksjni/compaction_options_fifo.cc
        rocksjni/compaction_options_universal.cc
        rocksjni/comparator.cc
@@ -19,7 +17,6 @@ set(JNI_NATIVE_SOURCES
        rocksjni/filter.cc
        rocksjni/ingest_external_file_options.cc
        rocksjni/iterator.cc
-        rocksjni/jnicallback.cc
        rocksjni/loggerjnicallback.cc
        rocksjni/lru_cache.cc
        rocksjni/memtablejni.cc
@@ -28,7 +25,6 @@ set(JNI_NATIVE_SOURCES
        rocksjni/options_util.cc
        rocksjni/ratelimiterjni.cc
        rocksjni/remove_emptyvalue_compactionfilterjni.cc
-        rocksjni/rocks_callback_object.cc
        rocksjni/cassandra_compactionfilterjni.cc
        rocksjni/cassandra_value_operator.cc
        rocksjni/restorejni.cc
@@ -1,4 +1,3 @@
-# shellcheck disable=SC2148
 PLATFORM=64
 if [ `getconf LONG_BIT` != "64" ]
 then
@@ -8,5 +7,4 @@ fi
 ROCKS_JAR=`find target -name rocksdbjni*.jar`

 echo "Running benchmark in $PLATFORM-Bit mode."
-# shellcheck disable=SC2068
 java -server -d$PLATFORM -XX:NewSize=4m -XX:+AggressiveOpts -Djava.library.path=target -cp "${ROCKS_JAR}:benchmark/target/classes" org.rocksdb.benchmark.DbBenchmark $@
@@ -34,5 +34,4 @@ void Java_org_rocksdb_AbstractCompactionFilterFactory_disposeInternal(
  auto* ptr_sptr_cff =
      reinterpret_cast<std::shared_ptr<rocksdb::CompactionFilterFactoryJniCallback> *>(jhandle);
  delete ptr_sptr_cff;
-// @lint-ignore TXT4 T25377293 Grandfathered in
 }
@@ -146,5 +146,4 @@ void Java_org_rocksdb_IngestExternalFileOptions_disposeInternal(
  auto* options =
      reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
  delete options;
-// @lint-ignore TXT4 T25377293 Grandfathered in
 }
@@ -49,5 +49,4 @@ JniCallback::~JniCallback() {

  releaseJniEnv(attached_thread);
 }
-// @lint-ignore TXT4 T25377293 Grandfathered in
 }  // namespace rocksdb
@@ -25,5 +25,4 @@ namespace rocksdb {
  };
 }

-// @lint-ignore TXT4 T25377293 Grandfathered in
 #endif  // JAVA_ROCKSJNI_JNICALLBACK_H_
@@ -24,5 +24,4 @@ void Java_org_rocksdb_RocksCallbackObject_disposeInternal(
      //   2) Comparator -> BaseComparatorJniCallback + JniCallback -> ComparatorJniCallback
      // I think this is okay, as Comparator and JniCallback both have virtual destructors...
      delete reinterpret_cast<rocksdb::JniCallback*>(handle);
-// @lint-ignore TXT4 T25377293 Grandfathered in
 }
@@ -30,5 +30,4 @@ namespace rocksdb {

    return true;
  }
-// @lint-ignore TXT4 T25377293 Grandfathered in
 };
@@ -30,5 +30,4 @@ namespace rocksdb {

 }  // namespace rocksdb

-// @lint-ignore TXT4 T25377293 Grandfathered in
 #endif  // JAVA_ROCKSJNI_STATISTICSJNI_H_
@@ -12,10 +12,6 @@ public class Environment {
    return (OS.contains("win"));
  }

-  public static boolean isFreeBSD() {
-    return (OS.contains("freebsd"));
-  }
-
  public static boolean isMac() {
    return (OS.contains("mac"));
  }
@@ -58,8 +54,6 @@ public class Environment {
      }
    } else if (isMac()) {
      return String.format("%sjni-osx", name);
-    } else if (isFreeBSD()) {
-      return String.format("%sjni-freebsd%s", name, is64Bit() ? "64" : "32");
    } else if (isAix() && is64Bit()) {
      return String.format("%sjni-aix64", name);
    } else if (isSolaris()) {
@@ -77,7 +71,7 @@ public class Environment {
  }

  private static String appendLibOsSuffix(final String libraryFileName, final boolean shared) {
-    if (isUnix() || isAix() || isSolaris() || isFreeBSD()) {
+    if (isUnix() || isAix() || isSolaris()) {
      return libraryFileName + ".so";
    } else if (isMac()) {
      return libraryFileName + (shared ? ".dylib" : ".jnilib");
@@ -279,7 +279,7 @@ struct InlineSkipList<Comparator>::Node {
  // next_[0].  This is used for passing data from AllocateKey to Insert.
  void StashHeight(const int height) {
    assert(sizeof(int) <= sizeof(next_[0]));
-    memcpy(static_cast<void*>(&next_[0]), &height, sizeof(int));
+    memcpy(&next_[0], &height, sizeof(int));
  }

  // Retrieves the value passed to StashHeight.  Undefined after a call
@@ -299,30 +299,30 @@ struct InlineSkipList<Comparator>::Node {
    assert(n >= 0);
    // Use an 'acquire load' so that we observe a fully initialized
    // version of the returned Node.
-    return ((&next_[0] - n)->load(std::memory_order_acquire));
+    return (next_[-n].load(std::memory_order_acquire));
  }

  void SetNext(int n, Node* x) {
    assert(n >= 0);
    // Use a 'release store' so that anybody who reads through this
    // pointer observes a fully initialized version of the inserted node.
-    (&next_[0] - n)->store(x, std::memory_order_release);
+    next_[-n].store(x, std::memory_order_release);
  }

  bool CASNext(int n, Node* expected, Node* x) {
    assert(n >= 0);
-    return (&next_[0] - n)->compare_exchange_strong(expected, x);
+    return next_[-n].compare_exchange_strong(expected, x);
  }

  // No-barrier variants that can be safely used in a few locations.
  Node* NoBarrier_Next(int n) {
    assert(n >= 0);
-    return (&next_[0] - n)->load(std::memory_order_relaxed);
+    return next_[-n].load(std::memory_order_relaxed);
  }

  void NoBarrier_SetNext(int n, Node* x) {
    assert(n >= 0);
-    (&next_[0] - n)->store(x, std::memory_order_relaxed);
+    next_[-n].store(x, std::memory_order_relaxed);
  }

  // Insert node after prev on specific level.
@@ -32,7 +32,7 @@ namespace port {

 namespace {

-#if defined(OS_LINUX) || defined(OS_FREEBSD)
+#ifdef OS_LINUX
 const char* GetExecutableName() {
  static char name[1024];

@@ -96,6 +96,7 @@ LIB_SOURCES =                                                   \
  table/block_based_table_factory.cc                            \
  table/block_based_table_reader.cc                             \
  table/block_builder.cc                                        \
+  table/block_fetcher.cc                                        \
  table/block_prefix_index.cc                                   \
  table/bloom_block.cc                                          \
  table/cuckoo_table_builder.cc                                 \
@@ -30,6 +30,7 @@
 #include "table/block.h"
 #include "table/block_based_filter_block.h"
 #include "table/block_based_table_factory.h"
+#include "table/block_fetcher.h"
 #include "table/block_prefix_index.h"
 #include "table/filter_block.h"
 #include "table/format.h"
@@ -78,9 +79,10 @@ Status ReadBlockFromFile(
    const PersistentCacheOptions& cache_options, SequenceNumber global_seqno,
    size_t read_amp_bytes_per_bit) {
  BlockContents contents;
-  Status s = ReadBlockContents(file, prefetch_buffer, footer, options, handle,
-                               &contents, ioptions, do_uncompress,
-                               compression_dict, cache_options);
+  BlockFetcher block_fetcher(file, prefetch_buffer, footer, options, handle,
+                             &contents, ioptions, do_uncompress,
+                             compression_dict, cache_options);
+  Status s = block_fetcher.ReadBlockContents();
  if (s.ok()) {
    result->reset(new Block(std::move(contents), global_seqno,
                            read_amp_bytes_per_bit, ioptions.statistics));
@@ -425,20 +427,23 @@ class HashIndexReader : public IndexReader {
      return Status::OK();
    }

+    Slice dummy_comp_dict;
    // Read contents for the blocks
    BlockContents prefixes_contents;
-    s = ReadBlockContents(file, prefetch_buffer, footer, ReadOptions(),
-                          prefixes_handle, &prefixes_contents, ioptions,
-                          true /* decompress */, Slice() /*compression dict*/,
-                          cache_options);
+    BlockFetcher prefixes_block_fetcher(
+        file, prefetch_buffer, footer, ReadOptions(), prefixes_handle,
+        &prefixes_contents, ioptions, true /* decompress */,
+        dummy_comp_dict /*compression dict*/, cache_options);
+    s = prefixes_block_fetcher.ReadBlockContents();
    if (!s.ok()) {
      return s;
    }
    BlockContents prefixes_meta_contents;
-    s = ReadBlockContents(file, prefetch_buffer, footer, ReadOptions(),
-                          prefixes_meta_handle, &prefixes_meta_contents,
-                          ioptions, true /* decompress */,
-                          Slice() /*compression dict*/, cache_options);
+    BlockFetcher prefixes_meta_block_fetcher(
+        file, prefetch_buffer, footer, ReadOptions(), prefixes_meta_handle,
+        &prefixes_meta_contents, ioptions, true /* decompress */,
+        dummy_comp_dict /*compression dict*/, cache_options);
+    prefixes_meta_block_fetcher.ReadBlockContents();
    if (!s.ok()) {
      // TODO: log error
      return Status::OK();
@@ -1193,11 +1198,16 @@ FilterBlockReader* BlockBasedTable::ReadFilter(
    return nullptr;
  }
  BlockContents block;
-  if (!ReadBlockContents(rep->file.get(), prefetch_buffer, rep->footer,
-                         ReadOptions(), filter_handle, &block, rep->ioptions,
-                         false /* decompress */, Slice() /*compression dict*/,
-                         rep->persistent_cache_options)
-           .ok()) {
+
+  Slice dummy_comp_dict;
+
+  BlockFetcher block_fetcher(rep->file.get(), prefetch_buffer, rep->footer,
+                             ReadOptions(), filter_handle, &block,
+                             rep->ioptions, false /* decompress */,
+                             dummy_comp_dict, rep->persistent_cache_options);
+  Status s = block_fetcher.ReadBlockContents();
+
+  if (!s.ok()) {
    // Error reading the block
    return nullptr;
  }
@@ -1983,11 +1993,13 @@ Status BlockBasedTable::VerifyChecksumInBlocks(InternalIterator* index_iter) {
      break;
    }
    BlockContents contents;
-    s = ReadBlockContents(rep_->file.get(), nullptr /* prefetch buffer */,
-                          rep_->footer, ReadOptions(), handle, &contents,
-                          rep_->ioptions, false /* decompress */,
-                          Slice() /*compression dict*/,
-                          rep_->persistent_cache_options);
+    Slice dummy_comp_dict;
+    BlockFetcher block_fetcher(rep_->file.get(), nullptr /* prefetch buffer */,
+                               rep_->footer, ReadOptions(), handle, &contents,
+                               rep_->ioptions, false /* decompress */,
+                               dummy_comp_dict /*compression dict*/,
+                               rep_->persistent_cache_options);
+    s = block_fetcher.ReadBlockContents();
    if (!s.ok()) {
      break;
    }
@@ -2272,12 +2284,14 @@ Status BlockBasedTable::DumpTable(WritableFile* out_file) {
      BlockHandle handle;
      if (FindMetaBlock(meta_iter.get(), filter_block_key, &handle).ok()) {
        BlockContents block;
-        if (ReadBlockContents(rep_->file.get(), nullptr /* prefetch_buffer */,
-                              rep_->footer, ReadOptions(), handle, &block,
-                              rep_->ioptions, false /*decompress*/,
-                              Slice() /*compression dict*/,
-                              rep_->persistent_cache_options)
-                .ok()) {
+        Slice dummy_comp_dict;
+        BlockFetcher block_fetcher(
+            rep_->file.get(), nullptr /* prefetch_buffer */, rep_->footer,
+            ReadOptions(), handle, &block, rep_->ioptions, false /*decompress*/,
+            dummy_comp_dict /*compression dict*/,
+            rep_->persistent_cache_options);
+        s = block_fetcher.ReadBlockContents();
+        if (!s.ok()) {
          rep_->filter.reset(new BlockBasedFilterBlockReader(
              rep_->ioptions.prefix_extractor, table_options,
              table_options.whole_key_filtering, std::move(block),
@@ -0,0 +1,233 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "table/block_fetcher.h"
+
+#include <string>
+#include <inttypes.h>
+
+#include "monitoring/perf_context_imp.h"
+#include "monitoring/statistics.h"
+#include "rocksdb/env.h"
+#include "table/block.h"
+#include "table/block_based_table_reader.h"
+#include "table/persistent_cache_helper.h"
+#include "table/format.h"
+#include "util/coding.h"
+#include "util/compression.h"
+#include "util/crc32c.h"
+#include "util/file_reader_writer.h"
+#include "util/logging.h"
+#include "util/stop_watch.h"
+#include "util/string_util.h"
+#include "util/xxhash.h"
+
+namespace rocksdb {
+
+void BlockFetcher::CheckBlockChecksum() {
+  // Check the crc of the type and the block contents
+  if (read_options_.verify_checksums) {
+    const char* data = slice_.data();  // Pointer to where Read put the data
+    PERF_TIMER_GUARD(block_checksum_time);
+    uint32_t value = DecodeFixed32(data + block_size_ + 1);
+    uint32_t actual = 0;
+    switch (footer_.checksum()) {
+      case kNoChecksum:
+        break;
+      case kCRC32c:
+        value = crc32c::Unmask(value);
+        actual = crc32c::Value(data, block_size_ + 1);
+        break;
+      case kxxHash:
+        actual = XXH32(data, static_cast<int>(block_size_) + 1, 0);
+        break;
+      default:
+        status_ = Status::Corruption(
+            "unknown checksum type " + ToString(footer_.checksum()) + " in " +
+            file_->file_name() + " offset " + ToString(handle_.offset()) +
+            " size " + ToString(block_size_));
+    }
+    if (status_.ok() && actual != value) {
+      status_ = Status::Corruption(
+          "block checksum mismatch: expected " + ToString(actual) + ", got " +
+          ToString(value) + "  in " + file_->file_name() + " offset " +
+          ToString(handle_.offset()) + " size " + ToString(block_size_));
+    }
+  }
+}
+
+bool BlockFetcher::TryGetUncompressBlockFromPersistentCache() {
+  if (cache_options_.persistent_cache &&
+      !cache_options_.persistent_cache->IsCompressed()) {
+    Status status = PersistentCacheHelper::LookupUncompressedPage(
+        cache_options_, handle_, contents_);
+    if (status.ok()) {
+      // uncompressed page is found for the block handle
+      return true;
+    } else {
+      // uncompressed page is not found
+      if (ioptions_.info_log && !status.IsNotFound()) {
+        assert(!status.ok());
+        ROCKS_LOG_INFO(ioptions_.info_log,
+                       "Error reading from persistent cache. %s",
+                       status.ToString().c_str());
+      }
+    }
+  }
+  return false;
+}
+
+bool BlockFetcher::TryGetFromPrefetchBuffer() {
+  if (prefetch_buffer_ != nullptr &&
+      prefetch_buffer_->TryReadFromCache(
+          handle_.offset(),
+          static_cast<size_t>(handle_.size()) + kBlockTrailerSize, &slice_)) {
+    block_size_ = static_cast<size_t>(handle_.size());
+    CheckBlockChecksum();
+    if (!status_.ok()) {
+      return true;
+    }
+    got_from_prefetch_buffer_ = true;
+    used_buf_ = const_cast<char*>(slice_.data());
+  }
+  return got_from_prefetch_buffer_;
+}
+
+bool BlockFetcher::TryGetCompressedBlockFromPersistentCache() {
+  if (cache_options_.persistent_cache &&
+      cache_options_.persistent_cache->IsCompressed()) {
+    // lookup uncompressed cache mode p-cache
+    status_ = PersistentCacheHelper::LookupRawPage(
+        cache_options_, handle_, &heap_buf_, block_size_ + kBlockTrailerSize);
+    if (status_.ok()) {
+      used_buf_ = heap_buf_.get();
+      slice_ = Slice(heap_buf_.get(), block_size_);
+      return true;
+    } else if (!status_.IsNotFound() && ioptions_.info_log) {
+      assert(!status_.ok());
+      ROCKS_LOG_INFO(ioptions_.info_log,
+                     "Error reading from persistent cache. %s",
+                     status_.ToString().c_str());
+    }
+  }
+  return false;
+}
+
+void BlockFetcher::PrepareBufferForBlockFromFile() {
+  // cache miss read from device
+  if (do_uncompress_ &&
+      block_size_ + kBlockTrailerSize < kDefaultStackBufferSize) {
+    // If we've got a small enough hunk of data, read it in to the
+    // trivially allocated stack buffer instead of needing a full malloc()
+    used_buf_ = &stack_buf_[0];
+  } else {
+    heap_buf_ =
+        std::unique_ptr<char[]>(new char[block_size_ + kBlockTrailerSize]);
+    used_buf_ = heap_buf_.get();
+  }
+}
+
+void BlockFetcher::InsertCompressedBlockToPersistentCacheIfNeeded() {
+  if (status_.ok() && read_options_.fill_cache &&
+      cache_options_.persistent_cache &&
+      cache_options_.persistent_cache->IsCompressed()) {
+    // insert to raw cache
+    PersistentCacheHelper::InsertRawPage(cache_options_, handle_, used_buf_,
+                                         block_size_ + kBlockTrailerSize);
+  }
+}
+
+void BlockFetcher::InsertUncompressedBlockToPersistentCacheIfNeeded() {
+  if (status_.ok() && !got_from_prefetch_buffer_ && read_options_.fill_cache &&
+      cache_options_.persistent_cache &&
+      !cache_options_.persistent_cache->IsCompressed()) {
+    // insert to uncompressed cache
+    PersistentCacheHelper::InsertUncompressedPage(cache_options_, handle_,
+                                                  *contents_);
+  }
+}
+
+void BlockFetcher::GetBlockContents() {
+  if (slice_.data() != used_buf_) {
+    // the slice content is not the buffer provided
+    *contents_ = BlockContents(Slice(slice_.data(), block_size_), false,
+                               compression_type);
+  } else {
+    // page is uncompressed, the buffer either stack or heap provided
+    if (got_from_prefetch_buffer_ || used_buf_ == &stack_buf_[0]) {
+      heap_buf_ = std::unique_ptr<char[]>(new char[block_size_]);
+      memcpy(heap_buf_.get(), used_buf_, block_size_);
+    }
+    *contents_ = BlockContents(std::move(heap_buf_), block_size_, true,
+                               compression_type);
+  }
+}
+
+Status BlockFetcher::ReadBlockContents() {
+  block_size_ = static_cast<size_t>(handle_.size());
+
+  if (TryGetUncompressBlockFromPersistentCache()) {
+    return Status::OK();
+  }
+  if (TryGetFromPrefetchBuffer()) {
+    if (!status_.ok()) {
+      return status_;
+    }
+  } else if (!TryGetCompressedBlockFromPersistentCache()) {
+    PrepareBufferForBlockFromFile();
+    Status s;
+
+    {
+      PERF_TIMER_GUARD(block_read_time);
+      // Actual file read
+      status_ = file_->Read(handle_.offset(), block_size_ + kBlockTrailerSize,
+                            &slice_, used_buf_);
+    }
+    PERF_COUNTER_ADD(block_read_count, 1);
+    PERF_COUNTER_ADD(block_read_byte, block_size_ + kBlockTrailerSize);
+    if (!status_.ok()) {
+      return status_;
+    }
+
+    if (slice_.size() != block_size_ + kBlockTrailerSize) {
+      return Status::Corruption("truncated block read from " +
+                                file_->file_name() + " offset " +
+                                ToString(handle_.offset()) + ", expected " +
+                                ToString(block_size_ + kBlockTrailerSize) +
+                                " bytes, got " + ToString(slice_.size()));
+    }
+
+    CheckBlockChecksum();
+    if (status_.ok()) {
+      InsertCompressedBlockToPersistentCacheIfNeeded();
+    } else {
+      return status_;
+    }
+  }
+
+  PERF_TIMER_GUARD(block_decompress_time);
+
+  compression_type =
+      static_cast<rocksdb::CompressionType>(slice_.data()[block_size_]);
+
+  if (do_uncompress_ && compression_type != kNoCompression) {
+    // compressed page, uncompress, update cache
+    status_ = UncompressBlockContents(slice_.data(), block_size_, contents_,
+                                      footer_.version(), compression_dict_,
+                                      ioptions_);
+  } else {
+    GetBlockContents();
+  }
+
+  InsertUncompressedBlockToPersistentCacheIfNeeded();
+
+  return status_;
+}
+
+}  // namespace rocksdb
@@ -0,0 +1,75 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include "table/block.h"
+#include "table/format.h"
+
+namespace rocksdb {
+class BlockFetcher {
+ public:
+  // Read the block identified by "handle" from "file".
+  // The only relevant option is options.verify_checksums for now.
+  // On failure return non-OK.
+  // On success fill *result and return OK - caller owns *result
+  // @param compression_dict Data for presetting the compression library's
+  //    dictionary.
+  BlockFetcher(RandomAccessFileReader* file,
+               FilePrefetchBuffer* prefetch_buffer, const Footer& footer,
+               const ReadOptions& read_options, const BlockHandle& handle,
+               BlockContents* contents,
+               const ImmutableCFOptions& ioptions,
+               bool do_uncompress, const Slice& compression_dict,
+               const PersistentCacheOptions& cache_options)
+      : file_(file),
+        prefetch_buffer_(prefetch_buffer),
+        footer_(footer),
+        read_options_(read_options),
+        handle_(handle),
+        contents_(contents),
+        ioptions_(ioptions),
+        do_uncompress_(do_uncompress),
+        compression_dict_(compression_dict),
+        cache_options_(cache_options) {}
+  Status ReadBlockContents();
+
+ private:
+  static const uint32_t kDefaultStackBufferSize = 5000;
+
+  RandomAccessFileReader* file_;
+  FilePrefetchBuffer* prefetch_buffer_;
+  const Footer& footer_;
+  const ReadOptions read_options_;
+  const BlockHandle& handle_;
+  BlockContents* contents_;
+  const ImmutableCFOptions& ioptions_;
+  bool do_uncompress_;
+  const Slice& compression_dict_;
+  const PersistentCacheOptions& cache_options_;
+  Status status_;
+  Slice slice_;
+  char* used_buf_ = nullptr;
+  size_t block_size_;
+  std::unique_ptr<char[]> heap_buf_;
+  char stack_buf_[kDefaultStackBufferSize];
+  bool got_from_prefetch_buffer_ = false;
+  rocksdb::CompressionType compression_type;
+
+  // return true if found
+  bool TryGetUncompressBlockFromPersistentCache();
+  // return true if found
+  bool TryGetFromPrefetchBuffer();
+  bool TryGetCompressedBlockFromPersistentCache();
+  void PrepareBufferForBlockFromFile();
+  void GetBlockContents();
+  void InsertCompressedBlockToPersistentCacheIfNeeded();
+  void InsertUncompressedBlockToPersistentCacheIfNeeded();
+  void CheckBlockChecksum();
+};
+}  // namespace rocksdb
@@ -49,7 +49,6 @@ class CuckooBuilderTest : public testing::Test {
    uint64_t read_file_size;
    ASSERT_OK(env_->GetFileSize(fname, &read_file_size));

-   // @lint-ignore TXT2 T25377293 Grandfathered in
 	  Options options;
 	  options.allow_mmap_reads = true;
 	  ImmutableCFOptions ioptions(options);
@@ -17,6 +17,7 @@
 #include "rocksdb/env.h"
 #include "table/block.h"
 #include "table/block_based_table_reader.h"
+#include "table/block_fetcher.h"
 #include "table/persistent_cache_helper.h"
 #include "util/coding.h"
 #include "util/compression.h"
@@ -40,7 +41,6 @@ extern const uint64_t kPlainTableMagicNumber;
 const uint64_t kLegacyPlainTableMagicNumber = 0;
 const uint64_t kPlainTableMagicNumber = 0;
 #endif
-const uint32_t DefaultStackBufferSize = 5000;

 bool ShouldReportDetailedTime(Env* env, Statistics* stats) {
  return env != nullptr && stats != nullptr &&
@@ -264,205 +264,6 @@ Status ReadFooterFromFile(RandomAccessFileReader* file,
  return Status::OK();
 }

-// Without anonymous namespace here, we fail the warning -Wmissing-prototypes
-namespace {
-Status CheckBlockChecksum(const ReadOptions& options, const Footer& footer,
-                          const Slice& contents, size_t block_size,
-                          RandomAccessFileReader* file,
-                          const BlockHandle& handle) {
-  Status s;
-  // Check the crc of the type and the block contents
-  if (options.verify_checksums) {
-    const char* data = contents.data();  // Pointer to where Read put the data
-    PERF_TIMER_GUARD(block_checksum_time);
-    uint32_t value = DecodeFixed32(data + block_size + 1);
-    uint32_t actual = 0;
-    switch (footer.checksum()) {
-      case kNoChecksum:
-        break;
-      case kCRC32c:
-        value = crc32c::Unmask(value);
-        actual = crc32c::Value(data, block_size + 1);
-        break;
-      case kxxHash:
-        actual = XXH32(data, static_cast<int>(block_size) + 1, 0);
-        break;
-      default:
-        s = Status::Corruption(
-            "unknown checksum type " + ToString(footer.checksum()) + " in " +
-            file->file_name() + " offset " + ToString(handle.offset()) +
-            " size " + ToString(block_size));
-    }
-    if (s.ok() && actual != value) {
-      s = Status::Corruption(
-          "block checksum mismatch: expected " + ToString(actual) + ", got " +
-          ToString(value) + "  in " + file->file_name() + " offset " +
-          ToString(handle.offset()) + " size " + ToString(block_size));
-    }
-    if (!s.ok()) {
-      return s;
-    }
-  }
-  return s;
-}
-
-// Read a block and check its CRC
-// contents is the result of reading.
-// According to the implementation of file->Read, contents may not point to buf
-Status ReadBlock(RandomAccessFileReader* file, const Footer& footer,
-                 const ReadOptions& options, const BlockHandle& handle,
-                 Slice* contents, /* result of reading */ char* buf) {
-  size_t n = static_cast<size_t>(handle.size());
-  Status s;
-
-  {
-    PERF_TIMER_GUARD(block_read_time);
-    s = file->Read(handle.offset(), n + kBlockTrailerSize, contents, buf);
-  }
-
-  PERF_COUNTER_ADD(block_read_count, 1);
-  PERF_COUNTER_ADD(block_read_byte, n + kBlockTrailerSize);
-
-  if (!s.ok()) {
-    return s;
-  }
-  if (contents->size() != n + kBlockTrailerSize) {
-    return Status::Corruption("truncated block read from " + file->file_name() +
-                              " offset " + ToString(handle.offset()) +
-                              ", expected " + ToString(n + kBlockTrailerSize) +
-                              " bytes, got " + ToString(contents->size()));
-  }
-  return CheckBlockChecksum(options, footer, *contents, n, file, handle);
-}
-
-}  // namespace
-
-Status ReadBlockContents(RandomAccessFileReader* file,
-                         FilePrefetchBuffer* prefetch_buffer,
-                         const Footer& footer, const ReadOptions& read_options,
-                         const BlockHandle& handle, BlockContents* contents,
-                         const ImmutableCFOptions& ioptions,
-                         bool decompression_requested,
-                         const Slice& compression_dict,
-                         const PersistentCacheOptions& cache_options) {
-  Status status;
-  Slice slice;
-  size_t n = static_cast<size_t>(handle.size());
-  std::unique_ptr<char[]> heap_buf;
-  char stack_buf[DefaultStackBufferSize];
-  char* used_buf = nullptr;
-  rocksdb::CompressionType compression_type;
-
-  if (cache_options.persistent_cache &&
-      !cache_options.persistent_cache->IsCompressed()) {
-    status = PersistentCacheHelper::LookupUncompressedPage(cache_options,
-                                                           handle, contents);
-    if (status.ok()) {
-      // uncompressed page is found for the block handle
-      return status;
-    } else {
-      // uncompressed page is not found
-      if (ioptions.info_log && !status.IsNotFound()) {
-        assert(!status.ok());
-        ROCKS_LOG_INFO(ioptions.info_log,
-                       "Error reading from persistent cache. %s",
-                       status.ToString().c_str());
-      }
-    }
-  }
-
-  bool got_from_prefetch_buffer = false;
-  if (prefetch_buffer != nullptr &&
-      prefetch_buffer->TryReadFromCache(
-          handle.offset(),
-          static_cast<size_t>(handle.size()) + kBlockTrailerSize, &slice)) {
-    status =
-        CheckBlockChecksum(read_options, footer, slice,
-                           static_cast<size_t>(handle.size()), file, handle);
-    if (!status.ok()) {
-      return status;
-    }
-    got_from_prefetch_buffer = true;
-    used_buf = const_cast<char*>(slice.data());
-  } else if (cache_options.persistent_cache &&
-             cache_options.persistent_cache->IsCompressed()) {
-    // lookup uncompressed cache mode p-cache
-    status = PersistentCacheHelper::LookupRawPage(
-        cache_options, handle, &heap_buf, n + kBlockTrailerSize);
-  } else {
-    status = Status::NotFound();
-  }
-
-  if (!got_from_prefetch_buffer) {
-    if (status.ok()) {
-      // cache hit
-      used_buf = heap_buf.get();
-      slice = Slice(heap_buf.get(), n);
-    } else {
-      if (ioptions.info_log && !status.IsNotFound()) {
-        assert(!status.ok());
-        ROCKS_LOG_INFO(ioptions.info_log,
-                       "Error reading from persistent cache. %s",
-                       status.ToString().c_str());
-      }
-      // cache miss read from device
-      if (decompression_requested &&
-          n + kBlockTrailerSize < DefaultStackBufferSize) {
-        // If we've got a small enough hunk of data, read it in to the
-        // trivially allocated stack buffer instead of needing a full malloc()
-        used_buf = &stack_buf[0];
-      } else {
-        heap_buf = std::unique_ptr<char[]>(new char[n + kBlockTrailerSize]);
-        used_buf = heap_buf.get();
-      }
-
-      status = ReadBlock(file, footer, read_options, handle, &slice, used_buf);
-      if (status.ok() && read_options.fill_cache &&
-          cache_options.persistent_cache &&
-          cache_options.persistent_cache->IsCompressed()) {
-        // insert to raw cache
-        PersistentCacheHelper::InsertRawPage(cache_options, handle, used_buf,
-                                             n + kBlockTrailerSize);
-      }
-    }
-
-    if (!status.ok()) {
-      return status;
-    }
-  }
-
-  PERF_TIMER_GUARD(block_decompress_time);
-
-  compression_type = static_cast<rocksdb::CompressionType>(slice.data()[n]);
-
-  if (decompression_requested && compression_type != kNoCompression) {
-    // compressed page, uncompress, update cache
-    status = UncompressBlockContents(slice.data(), n, contents,
-                                     footer.version(), compression_dict,
-                                     ioptions);
-  } else if (slice.data() != used_buf) {
-    // the slice content is not the buffer provided
-    *contents = BlockContents(Slice(slice.data(), n), false, compression_type);
-  } else {
-    // page is uncompressed, the buffer either stack or heap provided
-    if (got_from_prefetch_buffer || used_buf == &stack_buf[0]) {
-      heap_buf = std::unique_ptr<char[]>(new char[n]);
-      memcpy(heap_buf.get(), used_buf, n);
-    }
-    *contents = BlockContents(std::move(heap_buf), n, true, compression_type);
-  }
-
-  if (status.ok() && !got_from_prefetch_buffer && read_options.fill_cache &&
-      cache_options.persistent_cache &&
-      !cache_options.persistent_cache->IsCompressed()) {
-    // insert to uncompressed cache
-    PersistentCacheHelper::InsertUncompressedPage(cache_options, handle,
-                                                  *contents);
-  }
-
-  return status;
-}
-
 Status UncompressBlockContentsForCompressionType(
    const char* data, size_t n, BlockContents* contents,
    uint32_t format_version, const Slice& compression_dict,
@@ -22,7 +22,6 @@

 namespace rocksdb {

-class Block;
 class RandomAccessFile;
 struct ReadOptions;

@@ -11,6 +11,7 @@
 #include "rocksdb/table.h"
 #include "rocksdb/table_properties.h"
 #include "table/block.h"
+#include "table/block_fetcher.h"
 #include "table/format.h"
 #include "table/internal_iterator.h"
 #include "table/persistent_cache_helper.h"
@@ -176,8 +177,13 @@ Status ReadProperties(const Slice& handle_value, RandomAccessFileReader* file,
  ReadOptions read_options;
  read_options.verify_checksums = false;
  Status s;
-  s = ReadBlockContents(file, prefetch_buffer, footer, read_options, handle,
-                        &block_contents, ioptions, false /* decompress */);
+  Slice compression_dict;
+  PersistentCacheOptions cache_options;
+
+  BlockFetcher block_fetcher(
+      file, prefetch_buffer, footer, read_options, handle, &block_contents,
+      ioptions, false /* decompress */, compression_dict, cache_options);
+  s = block_fetcher.ReadBlockContents();

  if (!s.ok()) {
    return s;
@@ -292,9 +298,14 @@ Status ReadTableProperties(RandomAccessFileReader* file, uint64_t file_size,
  BlockContents metaindex_contents;
  ReadOptions read_options;
  read_options.verify_checksums = false;
-  s = ReadBlockContents(file, nullptr /* prefetch_buffer */, footer,
-                        read_options, metaindex_handle, &metaindex_contents,
-                        ioptions, false /* decompress */);
+  Slice compression_dict;
+  PersistentCacheOptions cache_options;
+
+  BlockFetcher block_fetcher(
+      file, nullptr /* prefetch_buffer */, footer, read_options,
+      metaindex_handle, &metaindex_contents, ioptions, false /* decompress */,
+      compression_dict, cache_options);
+  s = block_fetcher.ReadBlockContents();
  if (!s.ok()) {
    return s;
  }
@@ -350,9 +361,13 @@ Status FindMetaBlock(RandomAccessFileReader* file, uint64_t file_size,
  BlockContents metaindex_contents;
  ReadOptions read_options;
  read_options.verify_checksums = false;
-  s = ReadBlockContents(file, nullptr /* prefetch_buffer */, footer,
-                        read_options, metaindex_handle, &metaindex_contents,
-                        ioptions, false /* do decompression */);
+  Slice compression_dict;
+  PersistentCacheOptions cache_options;
+  BlockFetcher block_fetcher(
+      file, nullptr /* prefetch_buffer */, footer, read_options,
+      metaindex_handle, &metaindex_contents, ioptions,
+      false /* do decompression */, compression_dict, cache_options);
+  s = block_fetcher.ReadBlockContents();
  if (!s.ok()) {
    return s;
  }
@@ -384,9 +399,14 @@ Status ReadMetaBlock(RandomAccessFileReader* file,
  BlockContents metaindex_contents;
  ReadOptions read_options;
  read_options.verify_checksums = false;
-  status = ReadBlockContents(file, prefetch_buffer, footer, read_options,
+  Slice compression_dict;
+  PersistentCacheOptions cache_options;
+
+  BlockFetcher block_fetcher(file, prefetch_buffer, footer, read_options,
                             metaindex_handle, &metaindex_contents, ioptions,
-                             false /* decompress */);
+                             false /* decompress */, compression_dict,
+                             cache_options);
+  status = block_fetcher.ReadBlockContents();
  if (!status.ok()) {
    return status;
  }
@@ -406,9 +426,10 @@ Status ReadMetaBlock(RandomAccessFileReader* file,
  }

  // Reading metablock
-  return ReadBlockContents(file, prefetch_buffer, footer, read_options,
-                           block_handle, contents, ioptions,
-                           false /* decompress */);
+  BlockFetcher block_fetcher2(
+      file, prefetch_buffer, footer, read_options, block_handle, contents,
+      ioptions, false /* decompress */, compression_dict, cache_options);
+  return block_fetcher2.ReadBlockContents();
 }

 }  // namespace rocksdb
@@ -1,4 +1,3 @@
-# shellcheck disable=SC2148
 TMP_DIR="${TMPDIR:-/tmp}/rocksdb-sanity-test"

 if [ "$#" -lt 2 ]; then
@@ -449,7 +449,6 @@ echo "===== Benchmark ====="

 # Run!!!
 IFS=',' read -a jobs <<< $1
-# shellcheck disable=SC2068
 for job in ${jobs[@]}; do

  if [ $job != debug ]; then
@@ -151,7 +151,6 @@ echo "===== Benchmark ====="

 # Run!!!
 IFS=',' read -a jobs <<< $1
-# shellcheck disable=SC2068
 for job in ${jobs[@]}; do

  if [ $job != debug ]; then
@@ -2752,10 +2752,8 @@ void VerifyDBFromDB(std::string& truth_db_name) {

  void Crc32c(ThreadState* thread) {
    // Checksum about 500MB of data total
-    const int size = FLAGS_block_size; // use --block_size option for db_bench
-    std::string labels = "(" + ToString(FLAGS_block_size) + " per op)";
-    const char* label = labels.c_str();
-
+    const int size = 4096;
+    const char* label = "(4K per op)";
    std::string data(size, 'x');
    int64_t bytes = 0;
    uint32_t crc = 0;
@@ -3661,7 +3659,9 @@ void VerifyDBFromDB(std::string& truth_db_name) {
        }
      }
      if (!use_blob_db_) {
+#ifndef ROCKSDB_LITE
        s = db_with_cfh->db->Write(write_options_, &batch);
+#endif  //  ROCKSDB_LITE
      }
      thread->stats.FinishedOps(db_with_cfh, db_with_cfh->db,
                                entries_per_batch_, kWrite);
@@ -317,7 +317,7 @@ def whitebox_crash_main(args):
        cmd = gen_cmd(dict(cmd_params.items() + additional_opts.items()
                           + {'db': dbname}.items()))

-        print "Running:" + ' '.join(cmd) + "\n"  # noqa: E999 T25377293 Grandfathered in
+        print "Running:" + ' '.join(cmd) + "\n"

        popen = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
@@ -51,6 +51,8 @@ int main() {
 #include "rocksdb/slice_transform.h"
 #include "rocksdb/statistics.h"
 #include "rocksdb/utilities/db_ttl.h"
+#include "rocksdb/utilities/transaction.h"
+#include "rocksdb/utilities/transaction_db.h"
 #include "rocksdb/write_batch.h"
 #include "util/coding.h"
 #include "util/compression.h"
@@ -350,6 +352,10 @@ DEFINE_uint64(rate_limiter_bytes_per_sec, 0, "Set options.rate_limiter value.");
 DEFINE_bool(rate_limit_bg_reads, false,
            "Use options.rate_limiter on compaction reads");

+DEFINE_bool(use_txn, false,
+            "Use TransactionDB. Currently the default write policy is "
+            "TxnDBWritePolicy::WRITE_PREPARED");
+
 // Temporarily disable this to allows it to detect new bugs
 DEFINE_int32(compact_files_one_in, 0,
             "If non-zero, then CompactFiles() will be called one for every N "
@@ -981,11 +987,22 @@ const uint32_t SharedState::SENTINEL = 0xffffffff;

 // Per-thread state for concurrent executions of the same benchmark.
 struct ThreadState {
-  uint32_t tid; // 0..n-1
-  Random rand;  // Has different seeds for different threads
+  uint32_t tid;  // 0..n-1
+  Random rand;   // Has different seeds for different threads
  SharedState* shared;
  Stats stats;
-  std::queue<std::pair<uint64_t, const Snapshot*> > snapshot_queue;
+  struct SnapshotState {
+    const Snapshot* snapshot;
+    // The cf from which we did a Get at this stapshot
+    int cf_at;
+    // The key with which we did a Get at this stapshot
+    std::string key;
+    // The status of the Get
+    Status status;
+    // The value of the Get
+    std::string value;
+  };
+  std::queue<std::pair<uint64_t, SnapshotState> > snapshot_queue;

  ThreadState(uint32_t index, SharedState* _shared)
      : tid(index), rand(1000 + index + _shared->GetSeed()), shared(_shared) {}
@@ -1109,6 +1126,9 @@ class StressTest {
                                 : NewBloomFilterPolicy(FLAGS_bloom_bits, false)
                           : nullptr),
        db_(nullptr),
+#ifndef ROCKSDB_LITE
+        txn_db_(nullptr),
+#endif
        new_column_family_name_(1),
        num_times_reopened_(0) {
    if (FLAGS_destroy_db_initially) {
@@ -1316,6 +1336,32 @@ class StressTest {
  }

 private:
+  Status AssertSame(DB* db, ColumnFamilyHandle* cf,
+                    ThreadState::SnapshotState& snap_state) {
+    ReadOptions ropt;
+    ropt.snapshot = snap_state.snapshot;
+    PinnableSlice exp_v(&snap_state.value);
+    exp_v.PinSelf();
+    Status s;
+    PinnableSlice v;
+    s = db->Get(ropt, cf, snap_state.key, &v);
+    if (!s.ok() && !s.IsNotFound()) {
+      return s;
+    }
+    if (snap_state.status != s) {
+      return Status::Corruption("The snapshot gave inconsistent results: (" +
+                                snap_state.status.ToString() + ") vs. (" +
+                                s.ToString() + ")");
+    }
+    if (s.ok()) {
+      if (exp_v != v) {
+        return Status::Corruption("The snapshot gave inconsistent values: (" +
+                                  exp_v.ToString() + ") vs. (" + v.ToString() +
+                                  ")");
+      }
+    }
+    return Status::OK();
+  }

  static void ThreadBody(void* v) {
    ThreadState* thread = reinterpret_cast<ThreadState*>(v);
@@ -1641,6 +1687,32 @@ class StressTest {
    return db_->SetOptions(cfh, opts);
  }

+#ifndef ROCKSDB_LITE
+  Status NewTxn(WriteOptions& write_opts, Transaction** txn) {
+    if (!FLAGS_use_txn) {
+      return Status::InvalidArgument("NewTxn when FLAGS_use_txn is not set");
+    }
+    static std::atomic<uint64_t> txn_id = {0};
+    TransactionOptions txn_options;
+    *txn = txn_db_->BeginTransaction(write_opts, txn_options);
+    auto istr = std::to_string(txn_id.fetch_add(1));
+    Status s = (*txn)->SetName("xid" + istr);
+    return s;
+  }
+
+  Status CommitTxn(Transaction* txn) {
+    if (!FLAGS_use_txn) {
+      return Status::InvalidArgument("CommitTxn when FLAGS_use_txn is not set");
+    }
+    Status s = txn->Prepare();
+    if (s.ok()) {
+      s = txn->Commit();
+    }
+    delete txn;
+    return s;
+  }
+#endif
+
  void OperateDb(ThreadState* thread) {
    ReadOptions read_opts(FLAGS_verify_checksum, true);
    WriteOptions write_opts;
@@ -1667,7 +1739,8 @@ class StressTest {
          thread->stats.FinishedSingleOp();
          MutexLock l(thread->shared->GetMutex());
          while (!thread->snapshot_queue.empty()) {
-            db_->ReleaseSnapshot(thread->snapshot_queue.front().second);
+            db_->ReleaseSnapshot(
+                thread->snapshot_queue.front().second.snapshot);
            thread->snapshot_queue.pop();
          }
          thread->shared->IncVotedReopen();
@@ -1781,19 +1854,6 @@ class StressTest {
        }
      }
 #endif                // !ROCKSDB_LITE
-      if (FLAGS_acquire_snapshot_one_in > 0 &&
-          thread->rand.Uniform(FLAGS_acquire_snapshot_one_in) == 0) {
-        thread->snapshot_queue.emplace(
-            std::min(FLAGS_ops_per_thread - 1, i + FLAGS_snapshot_hold_ops),
-            db_->GetSnapshot());
-      }
-      if (!thread->snapshot_queue.empty()) {
-        while (i == thread->snapshot_queue.front().first) {
-          db_->ReleaseSnapshot(thread->snapshot_queue.front().second);
-          thread->snapshot_queue.pop();
-        }
-      }
-
      const double completed_ratio =
          static_cast<double>(i) / FLAGS_ops_per_thread;
      const int64_t base_key = static_cast<int64_t>(
@@ -1807,8 +1867,38 @@ class StressTest {
        l.reset(new MutexLock(
            shared->GetMutexForKey(rand_column_family, rand_key)));
      }
+
      auto column_family = column_families_[rand_column_family];

+      if (FLAGS_acquire_snapshot_one_in > 0 &&
+          thread->rand.Uniform(FLAGS_acquire_snapshot_one_in) == 0) {
+        auto snapshot = db_->GetSnapshot();
+        ReadOptions ropt;
+        ropt.snapshot = snapshot;
+        std::string value_at;
+        // When taking a snapshot, we also read a key from that snapshot. We
+        // will later read the same key before releasing the snapshot and verify
+        // that the results are the same.
+        auto status_at = db_->Get(ropt, column_family, key, &value_at);
+        ThreadState::SnapshotState snap_state = {snapshot, rand_column_family,
+                                                 keystr, status_at, value_at};
+        thread->snapshot_queue.emplace(
+            std::min(FLAGS_ops_per_thread - 1, i + FLAGS_snapshot_hold_ops),
+            snap_state);
+      }
+      while (!thread->snapshot_queue.empty() &&
+             i == thread->snapshot_queue.front().first) {
+        auto snap_state = thread->snapshot_queue.front().second;
+        assert(snap_state.snapshot);
+        Status s =
+            AssertSame(db_, column_families_[snap_state.cf_at], snap_state);
+        if (!s.ok()) {
+          VerificationAbort(shared, "Snapshot gave inconsistent state", s);
+        }
+        db_->ReleaseSnapshot(snap_state.snapshot);
+        thread->snapshot_queue.pop();
+      }
+
      int prob_op = thread->rand.Uniform(100);
      if (prob_op >= 0 && prob_op < (int)FLAGS_readpercent) {
        // OPERATION read
@@ -1885,9 +1975,35 @@ class StressTest {
          shared->Put(rand_column_family, rand_key, value_base);
          Status s;
          if (FLAGS_use_merge) {
-            s = db_->Merge(write_opts, column_family, key, v);
+            if (!FLAGS_use_txn) {
+              s = db_->Merge(write_opts, column_family, key, v);
+            } else {
+#ifndef ROCKSDB_LITE
+              Transaction* txn;
+              s = NewTxn(write_opts, &txn);
+              if (s.ok()) {
+                s = txn->Merge(column_family, key, v);
+                if (s.ok()) {
+                  s = CommitTxn(txn);
+                }
+              }
+#endif
+            }
          } else {
-            s = db_->Put(write_opts, column_family, key, v);
+            if (!FLAGS_use_txn) {
+              s = db_->Put(write_opts, column_family, key, v);
+            } else {
+#ifndef ROCKSDB_LITE
+              Transaction* txn;
+              s = NewTxn(write_opts, &txn);
+              if (s.ok()) {
+                s = txn->Put(column_family, key, v);
+                if (s.ok()) {
+                  s = CommitTxn(txn);
+                }
+              }
+#endif
+            }
          }
          if (!s.ok()) {
            fprintf(stderr, "put or merge error: %s\n", s.ToString().c_str());
@@ -1921,7 +2037,21 @@ class StressTest {
          // otherwise.
          if (shared->AllowsOverwrite(rand_column_family, rand_key)) {
            shared->Delete(rand_column_family, rand_key);
-            Status s = db_->Delete(write_opts, column_family, key);
+            Status s;
+            if (!FLAGS_use_txn) {
+              s = db_->Delete(write_opts, column_family, key);
+            } else {
+#ifndef ROCKSDB_LITE
+              Transaction* txn;
+              s = NewTxn(write_opts, &txn);
+              if (s.ok()) {
+                s = txn->Delete(column_family, key);
+                if (s.ok()) {
+                  s = CommitTxn(txn);
+                }
+              }
+#endif
+            }
            thread->stats.AddDeletes(1);
            if (!s.ok()) {
              fprintf(stderr, "delete error: %s\n", s.ToString().c_str());
@@ -1929,7 +2059,21 @@ class StressTest {
            }
          } else {
            shared->SingleDelete(rand_column_family, rand_key);
-            Status s = db_->SingleDelete(write_opts, column_family, key);
+            Status s;
+            if (!FLAGS_use_txn) {
+              s = db_->SingleDelete(write_opts, column_family, key);
+            } else {
+#ifndef ROCKSDB_LITE
+              Transaction* txn;
+              s = NewTxn(write_opts, &txn);
+              if (s.ok()) {
+                s = txn->SingleDelete(column_family, key);
+                if (s.ok()) {
+                  s = CommitTxn(txn);
+                }
+              }
+#endif
+            }
            thread->stats.AddSingleDeletes(1);
            if (!s.ok()) {
              fprintf(stderr, "single delete error: %s\n",
@@ -2067,6 +2211,12 @@ class StressTest {
    }
  }

+  void VerificationAbort(SharedState* shared, std::string msg, Status s) const {
+    printf("Verification failed: %s. Status is %s\n", msg.c_str(),
+           s.ToString().c_str());
+    shared->SetVerificationFailure();
+  }
+
  void VerificationAbort(SharedState* shared, std::string msg, int cf,
                         int64_t key) const {
    printf("Verification failed for column family %d key %" PRIi64 ": %s\n", cf, key,
@@ -2138,6 +2288,8 @@ class StressTest {
  void PrintEnv() const {
    fprintf(stdout, "RocksDB version           : %d.%d\n", kMajorVersion,
            kMinorVersion);
+    fprintf(stdout, "TransactionDB             : %s\n",
+            FLAGS_use_txn ? "true" : "false");
    fprintf(stdout, "Column families           : %d\n", FLAGS_column_families);
    if (!FLAGS_test_batches_snapshots) {
      fprintf(stdout, "Clear CFs one in          : %d\n",
@@ -2210,6 +2362,9 @@ class StressTest {

  void Open() {
    assert(db_ == nullptr);
+#ifndef ROCKSDB_LITE
+    assert(txn_db_ == nullptr);
+#endif
    BlockBasedTableOptions block_based_options;
    block_based_options.block_cache = cache_;
    block_based_options.block_cache_compressed = compressed_cache_;
@@ -2382,8 +2537,19 @@ class StressTest {
      options_.listeners.emplace_back(
          new DbStressListener(FLAGS_db, options_.db_paths));
      options_.create_missing_column_families = true;
-      s = DB::Open(DBOptions(options_), FLAGS_db, cf_descriptors,
-                   &column_families_, &db_);
+      if (!FLAGS_use_txn) {
+        s = DB::Open(DBOptions(options_), FLAGS_db, cf_descriptors,
+                     &column_families_, &db_);
+      } else {
+#ifndef ROCKSDB_LITE
+        TransactionDBOptions txn_db_options;
+        // For the moment it is sufficient to test WRITE_PREPARED policy
+        txn_db_options.write_policy = TxnDBWritePolicy::WRITE_PREPARED;
+        s = TransactionDB::Open(options_, txn_db_options, FLAGS_db,
+                                cf_descriptors, &column_families_, &txn_db_);
+        db_ = txn_db_;
+#endif
+      }
      assert(!s.ok() || column_families_.size() ==
                            static_cast<size_t>(FLAGS_column_families));
    } else {
@@ -2409,6 +2575,9 @@ class StressTest {
    column_families_.clear();
    delete db_;
    db_ = nullptr;
+#ifndef ROCKSDB_LITE
+    txn_db_ = nullptr;
+#endif

    num_times_reopened_++;
    auto now = FLAGS_env->NowMicros();
@@ -2429,6 +2598,9 @@ class StressTest {
  std::shared_ptr<Cache> compressed_cache_;
  std::shared_ptr<const FilterPolicy> filter_policy_;
  DB* db_;
+#ifndef ROCKSDB_LITE
+  TransactionDB* txn_db_;
+#endif
  Options options_;
  std::vector<ColumnFamilyHandle*> column_families_;
  std::vector<std::string> column_family_names_;
@@ -229,7 +229,7 @@ class LDBTestCase(unittest.TestCase):
        self.assertRunFAIL("get --ttl a3")
        self.assertRunOK("checkconsistency", "OK")

-    def testInvalidCmdLines(self):  # noqa: F811 T25377293 Grandfathered in
+    def testInvalidCmdLines(self):
        print "Running testInvalidCmdLines..."
        # db not specified
        self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
@@ -516,7 +516,7 @@ class LDBTestCase(unittest.TestCase):

    def testColumnFamilies(self):
        print "Running testColumnFamilies..."
-        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)  # noqa: F841 T25377293 Grandfathered in
+        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
        self.assertRunOK("put cf1_1 1 --create_if_missing", "OK")
        self.assertRunOK("put cf1_2 2 --create_if_missing", "OK")
        self.assertRunOK("put cf1_3 3 --try_load_options", "OK")
@@ -465,5 +465,4 @@ function cleanup_test_directory {

 ############################################################################

-# shellcheck disable=SC2068
 main $@
@@ -1,4 +1,3 @@
-# shellcheck disable=SC2148
 TESTDIR=`mktemp -d ${TMPDIR:-/tmp}/rocksdb-dump-test.XXXXX`
 DUMPFILE="tools/sample-dump.dmp"

@@ -21,7 +21,7 @@ def generate_runtimes(total_runtime):

 def main(args):
    runtimes = generate_runtimes(int(args.runtime_sec))
-    print "Going to execute write stress for " + str(runtimes)  # noqa: E999 T25377293 Grandfathered in
+    print "Going to execute write stress for " + str(runtimes)
    first_time = True

    for runtime in runtimes:
@@ -9,11 +9,12 @@
 //
 // A portable implementation of crc32c, optimized to handle
 // four bytes at a time.
+
 #include "util/crc32c.h"
+
 #include <stdint.h>
 #ifdef HAVE_SSE42
 #include <nmmintrin.h>
-#include <wmmintrin.h>
 #endif
 #include "util/coding.h"

@@ -351,7 +352,6 @@ static inline void Fast_CRC32(uint64_t* l, uint8_t const **p) {

 template<void (*CRC32)(uint64_t*, uint8_t const**)>
 uint32_t ExtendImpl(uint32_t crc, const char* buf, size_t size) {
-
  const uint8_t *p = reinterpret_cast<const uint8_t *>(buf);
  const uint8_t *e = p + size;
  uint64_t l = crc ^ 0xffffffffu;
@@ -395,14 +395,13 @@ uint32_t ExtendImpl(uint32_t crc, const char* buf, size_t size) {

 // Detect if SS42 or not.
 #ifndef HAVE_POWER8
-
 static bool isSSE42() {
 #ifndef HAVE_SSE42
  return false;
 #elif defined(__GNUC__) && defined(__x86_64__) && !defined(IOS_CROSS_COMPILE)
  uint32_t c_;
  __asm__("cpuid" : "=c"(c_) : "a"(1) : "ebx", "edx");
-  return c_ & (1U << 20);  // copied from CpuId.h in Folly. Test SSE42
+  return c_ & (1U << 20);  // copied from CpuId.h in Folly.
 #elif defined(_WIN64)
  int info[4];
  __cpuidex(info, 0x00000001, 0);
@@ -411,26 +410,7 @@ static bool isSSE42() {
  return false;
 #endif
 }
-
-static bool isPCLMULQDQ() {
-#ifndef HAVE_SSE42
-// in build_detect_platform we set this macro when both SSE42 and PCLMULQDQ are
-// supported by compiler
-  return false;
-#elif defined(__GNUC__) && defined(__x86_64__) && !defined(IOS_CROSS_COMPILE)
-  uint32_t c_;
-  __asm__("cpuid" : "=c"(c_) : "a"(1) : "ebx", "edx");
-  return c_ & (1U << 1);  // PCLMULQDQ is in bit 1 (not bit 0)
-#elif defined(_WIN64)
-  int info[4];
-  __cpuidex(info, 0x00000001, 0);
-  return (info[2] & ((int)1 << 1)) != 0;
-#else
-  return false;
 #endif
-}
-
-#endif  // HAVE_POWER8

 typedef uint32_t (*Function)(uint32_t, const char*, size_t);

@@ -460,6 +440,13 @@ static bool isAltiVec() {
 }
 #endif

+static inline Function Choose_Extend() {
+#ifndef HAVE_POWER8
+  return isSSE42() ? ExtendImpl<Fast_CRC32> : ExtendImpl<Slow_CRC32>;
+#else
+  return isAltiVec() ? ExtendPPCImpl : ExtendImpl<Slow_CRC32>;
+#endif
+}

 std::string IsFastCrc32Supported() {
  bool has_fast_crc = false;
@@ -488,727 +475,11 @@ std::string IsFastCrc32Supported() {
  return fast_zero_msg;
 }

-
-/*
- * Copyright 2016 Ferry Toth, Exalon Delft BV, The Netherlands
- *  This software is provided 'as-is', without any express or implied
- * warranty.  In no event will the author be held liable for any damages
- * arising from the use of this software.
- *  Permission is granted to anyone to use this software for any purpose,
- * including commercial applications, and to alter it and redistribute it
- * freely, subject to the following restrictions:
- *  1. The origin of this software must not be misrepresented; you must not
- *   claim that you wrote the original software. If you use this software
- *   in a product, an acknowledgment in the product documentation would be
- *   appreciated but is not required.
- * 2. Altered source versions must be plainly marked as such, and must not be
- *   misrepresented as being the original software.
- * 3. This notice may not be removed or altered from any source distribution.
- *  Ferry Toth
- * ftoth@exalondelft.nl
- *
- * https://github.com/htot/crc32c
- *
- * Modified by Facebook
- *
- * Original intel whitepaper:
- * "Fast CRC Computation for iSCSI Polynomial Using CRC32 Instruction"
- * https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/crc-iscsi-polynomial-crc32-instruction-paper.pdf
- *
- * This version is from the folly library, created by Dave Watson <davejwatson@fb.com>
- *
-*/
-#if defined HAVE_SSE42 && defined HAVE_PCLMUL
-
-#define CRCtriplet(crc, buf, offset)                  \
-  crc##0 = _mm_crc32_u64(crc##0, *(buf##0 + offset)); \
-  crc##1 = _mm_crc32_u64(crc##1, *(buf##1 + offset)); \
-  crc##2 = _mm_crc32_u64(crc##2, *(buf##2 + offset));
-
-#define CRCduplet(crc, buf, offset)                   \
-  crc##0 = _mm_crc32_u64(crc##0, *(buf##0 + offset)); \
-  crc##1 = _mm_crc32_u64(crc##1, *(buf##1 + offset));
-
-#define CRCsinglet(crc, buf, offset)                    \
-  crc = _mm_crc32_u64(crc, *(uint64_t*)(buf + offset));
-
-
-// Numbers taken directly from intel whitepaper.
-// clang-format off
-const uint64_t clmul_constants[] = {
-    0x14cd00bd6, 0x105ec76f0, 0x0ba4fc28e, 0x14cd00bd6,
-    0x1d82c63da, 0x0f20c0dfe, 0x09e4addf8, 0x0ba4fc28e,
-    0x039d3b296, 0x1384aa63a, 0x102f9b8a2, 0x1d82c63da,
-    0x14237f5e6, 0x01c291d04, 0x00d3b6092, 0x09e4addf8,
-    0x0c96cfdc0, 0x0740eef02, 0x18266e456, 0x039d3b296,
-    0x0daece73e, 0x0083a6eec, 0x0ab7aff2a, 0x102f9b8a2,
-    0x1248ea574, 0x1c1733996, 0x083348832, 0x14237f5e6,
-    0x12c743124, 0x02ad91c30, 0x0b9e02b86, 0x00d3b6092,
-    0x018b33a4e, 0x06992cea2, 0x1b331e26a, 0x0c96cfdc0,
-    0x17d35ba46, 0x07e908048, 0x1bf2e8b8a, 0x18266e456,
-    0x1a3e0968a, 0x11ed1f9d8, 0x0ce7f39f4, 0x0daece73e,
-    0x061d82e56, 0x0f1d0f55e, 0x0d270f1a2, 0x0ab7aff2a,
-    0x1c3f5f66c, 0x0a87ab8a8, 0x12ed0daac, 0x1248ea574,
-    0x065863b64, 0x08462d800, 0x11eef4f8e, 0x083348832,
-    0x1ee54f54c, 0x071d111a8, 0x0b3e32c28, 0x12c743124,
-    0x0064f7f26, 0x0ffd852c6, 0x0dd7e3b0c, 0x0b9e02b86,
-    0x0f285651c, 0x0dcb17aa4, 0x010746f3c, 0x018b33a4e,
-    0x1c24afea4, 0x0f37c5aee, 0x0271d9844, 0x1b331e26a,
-    0x08e766a0c, 0x06051d5a2, 0x093a5f730, 0x17d35ba46,
-    0x06cb08e5c, 0x11d5ca20e, 0x06b749fb2, 0x1bf2e8b8a,
-    0x1167f94f2, 0x021f3d99c, 0x0cec3662e, 0x1a3e0968a,
-    0x19329634a, 0x08f158014, 0x0e6fc4e6a, 0x0ce7f39f4,
-    0x08227bb8a, 0x1a5e82106, 0x0b0cd4768, 0x061d82e56,
-    0x13c2b89c4, 0x188815ab2, 0x0d7a4825c, 0x0d270f1a2,
-    0x10f5ff2ba, 0x105405f3e, 0x00167d312, 0x1c3f5f66c,
-    0x0f6076544, 0x0e9adf796, 0x026f6a60a, 0x12ed0daac,
-    0x1a2adb74e, 0x096638b34, 0x19d34af3a, 0x065863b64,
-    0x049c3cc9c, 0x1e50585a0, 0x068bce87a, 0x11eef4f8e,
-    0x1524fa6c6, 0x19f1c69dc, 0x16cba8aca, 0x1ee54f54c,
-    0x042d98888, 0x12913343e, 0x1329d9f7e, 0x0b3e32c28,
-    0x1b1c69528, 0x088f25a3a, 0x02178513a, 0x0064f7f26,
-    0x0e0ac139e, 0x04e36f0b0, 0x0170076fa, 0x0dd7e3b0c,
-    0x141a1a2e2, 0x0bd6f81f8, 0x16ad828b4, 0x0f285651c,
-    0x041d17b64, 0x19425cbba, 0x1fae1cc66, 0x010746f3c,
-    0x1a75b4b00, 0x18db37e8a, 0x0f872e54c, 0x1c24afea4,
-    0x01e41e9fc, 0x04c144932, 0x086d8e4d2, 0x0271d9844,
-    0x160f7af7a, 0x052148f02, 0x05bb8f1bc, 0x08e766a0c,
-    0x0a90fd27a, 0x0a3c6f37a, 0x0b3af077a, 0x093a5f730,
-    0x04984d782, 0x1d22c238e, 0x0ca6ef3ac, 0x06cb08e5c,
-    0x0234e0b26, 0x063ded06a, 0x1d88abd4a, 0x06b749fb2,
-    0x04597456a, 0x04d56973c, 0x0e9e28eb4, 0x1167f94f2,
-    0x07b3ff57a, 0x19385bf2e, 0x0c9c8b782, 0x0cec3662e,
-    0x13a9cba9e, 0x0e417f38a, 0x093e106a4, 0x19329634a,
-    0x167001a9c, 0x14e727980, 0x1ddffc5d4, 0x0e6fc4e6a,
-    0x00df04680, 0x0d104b8fc, 0x02342001e, 0x08227bb8a,
-    0x00a2a8d7e, 0x05b397730, 0x168763fa6, 0x0b0cd4768,
-    0x1ed5a407a, 0x0e78eb416, 0x0d2c3ed1a, 0x13c2b89c4,
-    0x0995a5724, 0x1641378f0, 0x19b1afbc4, 0x0d7a4825c,
-    0x109ffedc0, 0x08d96551c, 0x0f2271e60, 0x10f5ff2ba,
-    0x00b0bf8ca, 0x00bf80dd2, 0x123888b7a, 0x00167d312,
-    0x1e888f7dc, 0x18dcddd1c, 0x002ee03b2, 0x0f6076544,
-    0x183e8d8fe, 0x06a45d2b2, 0x133d7a042, 0x026f6a60a,
-    0x116b0f50c, 0x1dd3e10e8, 0x05fabe670, 0x1a2adb74e,
-    0x130004488, 0x0de87806c, 0x000bcf5f6, 0x19d34af3a,
-    0x18f0c7078, 0x014338754, 0x017f27698, 0x049c3cc9c,
-    0x058ca5f00, 0x15e3e77ee, 0x1af900c24, 0x068bce87a,
-    0x0b5cfca28, 0x0dd07448e, 0x0ded288f8, 0x1524fa6c6,
-    0x059f229bc, 0x1d8048348, 0x06d390dec, 0x16cba8aca,
-    0x037170390, 0x0a3e3e02c, 0x06353c1cc, 0x042d98888,
-    0x0c4584f5c, 0x0d73c7bea, 0x1f16a3418, 0x1329d9f7e,
-    0x0531377e2, 0x185137662, 0x1d8d9ca7c, 0x1b1c69528,
-    0x0b25b29f2, 0x18a08b5bc, 0x19fb2a8b0, 0x02178513a,
-    0x1a08fe6ac, 0x1da758ae0, 0x045cddf4e, 0x0e0ac139e,
-    0x1a91647f2, 0x169cf9eb0, 0x1a0f717c4, 0x0170076fa,
-};
-
-// Compute the crc32c value for buffer smaller than 8
-inline void align_to_8(
-    size_t len,
-    uint64_t& crc0, // crc so far, updated on return
-    const unsigned char*& next) { // next data pointer, updated on return
-  uint32_t crc32bit = static_cast<uint32_t>(crc0);
-  if (len & 0x04) {
-    crc32bit = _mm_crc32_u32(crc32bit, *(uint32_t*)next);
-    next += sizeof(uint32_t);
-  }
-  if (len & 0x02) {
-    crc32bit = _mm_crc32_u16(crc32bit, *(uint16_t*)next);
-    next += sizeof(uint16_t);
-  }
-  if (len & 0x01) {
-    crc32bit = _mm_crc32_u8(crc32bit, *(next));
-    next++;
-  }
-  crc0 = crc32bit;
-}
-
-//
-// CombineCRC performs pclmulqdq multiplication of 2 partial CRC's and a well
-// chosen constant and xor's these with the remaining CRC.
-//
-inline uint64_t CombineCRC(
-    size_t block_size,
-    uint64_t crc0,
-    uint64_t crc1,
-    uint64_t crc2,
-    const uint64_t* next2) {
-  const auto multiplier =
-      *(reinterpret_cast<const __m128i*>(clmul_constants) + block_size - 1);
-  const auto crc0_xmm = _mm_set_epi64x(0, crc0);
-  const auto res0 = _mm_clmulepi64_si128(crc0_xmm, multiplier, 0x00);
-  const auto crc1_xmm = _mm_set_epi64x(0, crc1);
-  const auto res1 = _mm_clmulepi64_si128(crc1_xmm, multiplier, 0x10);
-  const auto res = _mm_xor_si128(res0, res1);
-  crc0 = _mm_cvtsi128_si64(res);
-  crc0 = crc0 ^ *((uint64_t*)next2 - 1);
-  crc2 = _mm_crc32_u64(crc2, crc0);
-  return crc2;
-}
-
-// Compute CRC-32C using the Intel hardware instruction.
-uint32_t crc32c_3way(uint32_t crc, const char* buf, size_t len) {
-  const unsigned char* next = (const unsigned char*)buf;
-  uint64_t count;
-  uint64_t crc0, crc1, crc2;
-  crc0 = crc ^ 0xffffffffu;
-
-
-  if (len >= 8) {
-    // if len > 216 then align and use triplets
-    if (len > 216) {
-      {
-        // Work on the bytes (< 8) before the first 8-byte alignment addr starts
-        uint64_t align_bytes = (8 - (uintptr_t)next) & 7;
-        len -= align_bytes;
-        align_to_8(align_bytes, crc0, next);
-      }
-
-      // Now work on the remaining blocks
-      count = len / 24; // number of triplets
-      len %= 24; // bytes remaining
-      uint64_t n = count >> 7; // #blocks = first block + full blocks
-      uint64_t block_size = count & 127;
-      if (block_size == 0) {
-        block_size = 128;
-      } else {
-        n++;
-      }
-      // points to the first byte of the next block
-      const uint64_t* next0 = (uint64_t*)next + block_size;
-      const uint64_t* next1 = next0 + block_size;
-      const uint64_t* next2 = next1 + block_size;
-
-      crc1 = crc2 = 0;
-      // Use Duff's device, a for() loop inside a switch()
-      // statement. This needs to execute at least once, round len
-      // down to nearest triplet multiple
-      switch (block_size) {
-        case 128:
-          do {
-            // jumps here for a full block of len 128
-            CRCtriplet(crc, next, -128);
-            // FALLTHRU
-            case 127:
-              // jumps here or below for the first block smaller
-              CRCtriplet(crc, next, -127);
-              // FALLTHRU
-            case 126:
-              CRCtriplet(crc, next, -126); // than 128
-              // FALLTHRU
-            case 125:
-              CRCtriplet(crc, next, -125);
-              // FALLTHRU
-            case 124:
-              CRCtriplet(crc, next, -124);
-              // FALLTHRU
-            case 123:
-              CRCtriplet(crc, next, -123);
-              // FALLTHRU
-            case 122:
-              CRCtriplet(crc, next, -122);
-              // FALLTHRU
-            case 121:
-              CRCtriplet(crc, next, -121);
-              // FALLTHRU
-            case 120:
-              CRCtriplet(crc, next, -120);
-              // FALLTHRU
-            case 119:
-              CRCtriplet(crc, next, -119);
-              // FALLTHRU
-            case 118:
-              CRCtriplet(crc, next, -118);
-              // FALLTHRU
-            case 117:
-              CRCtriplet(crc, next, -117);
-              // FALLTHRU
-            case 116:
-              CRCtriplet(crc, next, -116);
-              // FALLTHRU
-            case 115:
-              CRCtriplet(crc, next, -115);
-              // FALLTHRU
-            case 114:
-              CRCtriplet(crc, next, -114);
-              // FALLTHRU
-            case 113:
-              CRCtriplet(crc, next, -113);
-              // FALLTHRU
-            case 112:
-              CRCtriplet(crc, next, -112);
-              // FALLTHRU
-            case 111:
-              CRCtriplet(crc, next, -111);
-              // FALLTHRU
-            case 110:
-              CRCtriplet(crc, next, -110);
-              // FALLTHRU
-            case 109:
-              CRCtriplet(crc, next, -109);
-              // FALLTHRU
-            case 108:
-              CRCtriplet(crc, next, -108);
-              // FALLTHRU
-            case 107:
-              CRCtriplet(crc, next, -107);
-              // FALLTHRU
-            case 106:
-              CRCtriplet(crc, next, -106);
-              // FALLTHRU
-            case 105:
-              CRCtriplet(crc, next, -105);
-              // FALLTHRU
-            case 104:
-              CRCtriplet(crc, next, -104);
-              // FALLTHRU
-            case 103:
-              CRCtriplet(crc, next, -103);
-              // FALLTHRU
-            case 102:
-              CRCtriplet(crc, next, -102);
-              // FALLTHRU
-            case 101:
-              CRCtriplet(crc, next, -101);
-              // FALLTHRU
-            case 100:
-              CRCtriplet(crc, next, -100);
-              // FALLTHRU
-            case 99:
-              CRCtriplet(crc, next, -99);
-              // FALLTHRU
-            case 98:
-              CRCtriplet(crc, next, -98);
-              // FALLTHRU
-            case 97:
-              CRCtriplet(crc, next, -97);
-              // FALLTHRU
-            case 96:
-              CRCtriplet(crc, next, -96);
-              // FALLTHRU
-            case 95:
-              CRCtriplet(crc, next, -95);
-              // FALLTHRU
-            case 94:
-              CRCtriplet(crc, next, -94);
-              // FALLTHRU
-            case 93:
-              CRCtriplet(crc, next, -93);
-              // FALLTHRU
-            case 92:
-              CRCtriplet(crc, next, -92);
-              // FALLTHRU
-            case 91:
-              CRCtriplet(crc, next, -91);
-              // FALLTHRU
-            case 90:
-              CRCtriplet(crc, next, -90);
-              // FALLTHRU
-            case 89:
-              CRCtriplet(crc, next, -89);
-              // FALLTHRU
-            case 88:
-              CRCtriplet(crc, next, -88);
-              // FALLTHRU
-            case 87:
-              CRCtriplet(crc, next, -87);
-              // FALLTHRU
-            case 86:
-              CRCtriplet(crc, next, -86);
-              // FALLTHRU
-            case 85:
-              CRCtriplet(crc, next, -85);
-              // FALLTHRU
-            case 84:
-              CRCtriplet(crc, next, -84);
-              // FALLTHRU
-            case 83:
-              CRCtriplet(crc, next, -83);
-              // FALLTHRU
-            case 82:
-              CRCtriplet(crc, next, -82);
-              // FALLTHRU
-            case 81:
-              CRCtriplet(crc, next, -81);
-              // FALLTHRU
-            case 80:
-              CRCtriplet(crc, next, -80);
-              // FALLTHRU
-            case 79:
-              CRCtriplet(crc, next, -79);
-              // FALLTHRU
-            case 78:
-              CRCtriplet(crc, next, -78);
-              // FALLTHRU
-            case 77:
-              CRCtriplet(crc, next, -77);
-              // FALLTHRU
-            case 76:
-              CRCtriplet(crc, next, -76);
-              // FALLTHRU
-            case 75:
-              CRCtriplet(crc, next, -75);
-              // FALLTHRU
-            case 74:
-              CRCtriplet(crc, next, -74);
-              // FALLTHRU
-            case 73:
-              CRCtriplet(crc, next, -73);
-              // FALLTHRU
-            case 72:
-              CRCtriplet(crc, next, -72);
-              // FALLTHRU
-            case 71:
-              CRCtriplet(crc, next, -71);
-              // FALLTHRU
-            case 70:
-              CRCtriplet(crc, next, -70);
-              // FALLTHRU
-            case 69:
-              CRCtriplet(crc, next, -69);
-              // FALLTHRU
-            case 68:
-              CRCtriplet(crc, next, -68);
-              // FALLTHRU
-            case 67:
-              CRCtriplet(crc, next, -67);
-              // FALLTHRU
-            case 66:
-              CRCtriplet(crc, next, -66);
-              // FALLTHRU
-            case 65:
-              CRCtriplet(crc, next, -65);
-              // FALLTHRU
-            case 64:
-              CRCtriplet(crc, next, -64);
-              // FALLTHRU
-            case 63:
-              CRCtriplet(crc, next, -63);
-              // FALLTHRU
-            case 62:
-              CRCtriplet(crc, next, -62);
-              // FALLTHRU
-            case 61:
-              CRCtriplet(crc, next, -61);
-              // FALLTHRU
-            case 60:
-              CRCtriplet(crc, next, -60);
-              // FALLTHRU
-            case 59:
-              CRCtriplet(crc, next, -59);
-              // FALLTHRU
-            case 58:
-              CRCtriplet(crc, next, -58);
-              // FALLTHRU
-            case 57:
-              CRCtriplet(crc, next, -57);
-              // FALLTHRU
-            case 56:
-              CRCtriplet(crc, next, -56);
-              // FALLTHRU
-            case 55:
-              CRCtriplet(crc, next, -55);
-              // FALLTHRU
-            case 54:
-              CRCtriplet(crc, next, -54);
-              // FALLTHRU
-            case 53:
-              CRCtriplet(crc, next, -53);
-              // FALLTHRU
-            case 52:
-              CRCtriplet(crc, next, -52);
-              // FALLTHRU
-            case 51:
-              CRCtriplet(crc, next, -51);
-              // FALLTHRU
-            case 50:
-              CRCtriplet(crc, next, -50);
-              // FALLTHRU
-            case 49:
-              CRCtriplet(crc, next, -49);
-              // FALLTHRU
-            case 48:
-              CRCtriplet(crc, next, -48);
-              // FALLTHRU
-            case 47:
-              CRCtriplet(crc, next, -47);
-              // FALLTHRU
-            case 46:
-              CRCtriplet(crc, next, -46);
-              // FALLTHRU
-            case 45:
-              CRCtriplet(crc, next, -45);
-              // FALLTHRU
-            case 44:
-              CRCtriplet(crc, next, -44);
-              // FALLTHRU
-            case 43:
-              CRCtriplet(crc, next, -43);
-              // FALLTHRU
-            case 42:
-              CRCtriplet(crc, next, -42);
-              // FALLTHRU
-            case 41:
-              CRCtriplet(crc, next, -41);
-              // FALLTHRU
-            case 40:
-              CRCtriplet(crc, next, -40);
-              // FALLTHRU
-            case 39:
-              CRCtriplet(crc, next, -39);
-              // FALLTHRU
-            case 38:
-              CRCtriplet(crc, next, -38);
-              // FALLTHRU
-            case 37:
-              CRCtriplet(crc, next, -37);
-              // FALLTHRU
-            case 36:
-              CRCtriplet(crc, next, -36);
-              // FALLTHRU
-            case 35:
-              CRCtriplet(crc, next, -35);
-              // FALLTHRU
-            case 34:
-              CRCtriplet(crc, next, -34);
-              // FALLTHRU
-            case 33:
-              CRCtriplet(crc, next, -33);
-              // FALLTHRU
-            case 32:
-              CRCtriplet(crc, next, -32);
-              // FALLTHRU
-            case 31:
-              CRCtriplet(crc, next, -31);
-              // FALLTHRU
-            case 30:
-              CRCtriplet(crc, next, -30);
-              // FALLTHRU
-            case 29:
-              CRCtriplet(crc, next, -29);
-              // FALLTHRU
-            case 28:
-              CRCtriplet(crc, next, -28);
-              // FALLTHRU
-            case 27:
-              CRCtriplet(crc, next, -27);
-              // FALLTHRU
-            case 26:
-              CRCtriplet(crc, next, -26);
-              // FALLTHRU
-            case 25:
-              CRCtriplet(crc, next, -25);
-              // FALLTHRU
-            case 24:
-              CRCtriplet(crc, next, -24);
-              // FALLTHRU
-            case 23:
-              CRCtriplet(crc, next, -23);
-              // FALLTHRU
-            case 22:
-              CRCtriplet(crc, next, -22);
-              // FALLTHRU
-            case 21:
-              CRCtriplet(crc, next, -21);
-              // FALLTHRU
-            case 20:
-              CRCtriplet(crc, next, -20);
-              // FALLTHRU
-            case 19:
-              CRCtriplet(crc, next, -19);
-              // FALLTHRU
-            case 18:
-              CRCtriplet(crc, next, -18);
-              // FALLTHRU
-            case 17:
-              CRCtriplet(crc, next, -17);
-              // FALLTHRU
-            case 16:
-              CRCtriplet(crc, next, -16);
-              // FALLTHRU
-            case 15:
-              CRCtriplet(crc, next, -15);
-              // FALLTHRU
-            case 14:
-              CRCtriplet(crc, next, -14);
-              // FALLTHRU
-            case 13:
-              CRCtriplet(crc, next, -13);
-              // FALLTHRU
-            case 12:
-              CRCtriplet(crc, next, -12);
-              // FALLTHRU
-            case 11:
-              CRCtriplet(crc, next, -11);
-              // FALLTHRU
-            case 10:
-              CRCtriplet(crc, next, -10);
-              // FALLTHRU
-            case 9:
-              CRCtriplet(crc, next, -9);
-              // FALLTHRU
-            case 8:
-              CRCtriplet(crc, next, -8);
-              // FALLTHRU
-            case 7:
-              CRCtriplet(crc, next, -7);
-              // FALLTHRU
-            case 6:
-              CRCtriplet(crc, next, -6);
-              // FALLTHRU
-            case 5:
-              CRCtriplet(crc, next, -5);
-              // FALLTHRU
-            case 4:
-              CRCtriplet(crc, next, -4);
-              // FALLTHRU
-            case 3:
-              CRCtriplet(crc, next, -3);
-              // FALLTHRU
-            case 2:
-              CRCtriplet(crc, next, -2);
-              // FALLTHRU
-            case 1:
-              CRCduplet(crc, next, -1); // the final triplet is actually only 2
-              //{ CombineCRC(); }
-              crc0 = CombineCRC(block_size, crc0, crc1, crc2, next2);
-              if (--n > 0) {
-                crc1 = crc2 = 0;
-                block_size = 128;
-                // points to the first byte of the next block
-                next0 = next2 + 128;
-                next1 = next0 + 128; // from here on all blocks are 128 long
-                next2 = next1 + 128;
-              }
-              // FALLTHRU
-            case 0:;
-          } while (n > 0);
-      }
-      next = (const unsigned char*)next2;
-    }
-    uint64_t count2 = len >> 3; // 216 of less bytes is 27 or less singlets
-    len = len & 7;
-    next += (count2 * 8);
-    switch (count2) {
-      case 27:
-        CRCsinglet(crc0, next, -27 * 8);
-        // FALLTHRU
-      case 26:
-        CRCsinglet(crc0, next, -26 * 8);
-        // FALLTHRU
-      case 25:
-        CRCsinglet(crc0, next, -25 * 8);
-        // FALLTHRU
-      case 24:
-        CRCsinglet(crc0, next, -24 * 8);
-        // FALLTHRU
-      case 23:
-        CRCsinglet(crc0, next, -23 * 8);
-        // FALLTHRU
-      case 22:
-        CRCsinglet(crc0, next, -22 * 8);
-        // FALLTHRU
-      case 21:
-        CRCsinglet(crc0, next, -21 * 8);
-        // FALLTHRU
-      case 20:
-        CRCsinglet(crc0, next, -20 * 8);
-        // FALLTHRU
-      case 19:
-        CRCsinglet(crc0, next, -19 * 8);
-        // FALLTHRU
-      case 18:
-        CRCsinglet(crc0, next, -18 * 8);
-        // FALLTHRU
-      case 17:
-        CRCsinglet(crc0, next, -17 * 8);
-        // FALLTHRU
-      case 16:
-        CRCsinglet(crc0, next, -16 * 8);
-        // FALLTHRU
-      case 15:
-        CRCsinglet(crc0, next, -15 * 8);
-        // FALLTHRU
-      case 14:
-        CRCsinglet(crc0, next, -14 * 8);
-        // FALLTHRU
-      case 13:
-        CRCsinglet(crc0, next, -13 * 8);
-        // FALLTHRU
-      case 12:
-        CRCsinglet(crc0, next, -12 * 8);
-        // FALLTHRU
-      case 11:
-        CRCsinglet(crc0, next, -11 * 8);
-        // FALLTHRU
-      case 10:
-        CRCsinglet(crc0, next, -10 * 8);
-        // FALLTHRU
-      case 9:
-        CRCsinglet(crc0, next, -9 * 8);
-        // FALLTHRU
-      case 8:
-        CRCsinglet(crc0, next, -8 * 8);
-        // FALLTHRU
-      case 7:
-        CRCsinglet(crc0, next, -7 * 8);
-        // FALLTHRU
-      case 6:
-        CRCsinglet(crc0, next, -6 * 8);
-        // FALLTHRU
-      case 5:
-        CRCsinglet(crc0, next, -5 * 8);
-        // FALLTHRU
-      case 4:
-        CRCsinglet(crc0, next, -4 * 8);
-        // FALLTHRU
-      case 3:
-        CRCsinglet(crc0, next, -3 * 8);
-        // FALLTHRU
-      case 2:
-        CRCsinglet(crc0, next, -2 * 8);
-        // FALLTHRU
-      case 1:
-        CRCsinglet(crc0, next, -1 * 8);
-        // FALLTHRU
-      case 0:;
-    }
-  }
-  {
-    align_to_8(len, crc0, next);
-    return (uint32_t)crc0 ^ 0xffffffffu;
-  }
-}
-
-#endif //HAVE_SSE42 && HAVE_PCLMUL
-
-static inline Function Choose_Extend() {
-#ifndef HAVE_POWER8
-  if (isSSE42()) {
-    if (isPCLMULQDQ()) {
-#if defined HAVE_SSE42  && defined HAVE_PCLMUL && !defined NO_THREEWAY_CRC32C
-      return crc32c_3way;
-#else
-    return ExtendImpl<Fast_CRC32>; // Fast_CRC32 will check HAVE_SSE42 itself
-#endif
-    }
-    else {  // no runtime PCLMULQDQ support but has SSE42 support
-      return ExtendImpl<Fast_CRC32>;
-    }
-  } // end of isSSE42()
-  else {
-    return ExtendImpl<Slow_CRC32>;
-  }
-#else  //HAVE_POWER8
-  return isAltiVec() ? ExtendPPCImpl : ExtendImpl<Slow_CRC32>;
-#endif
-}
-
 static Function ChosenExtend = Choose_Extend();
+
 uint32_t Extend(uint32_t crc, const char* buf, size_t size) {
  return ChosenExtend(crc, buf, size);
 }

-
 }  // namespace crc32c
 }  // namespace rocksdb
@@ -6,6 +6,7 @@
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
+
 #include "util/crc32c.h"
 #include "util/testharness.h"

@@ -14,57 +15,7 @@ namespace crc32c {

 class CRC { };

-
-// Tests for 3-way crc32c algorithm. We need these tests because it uses
-// different lookup tables than the original Fast_CRC32
-const unsigned int BUFFER_SIZE = 512 * 1024 * sizeof(uint64_t);
-char buffer[BUFFER_SIZE];
-
-struct ExpectedResult {
-  size_t offset;
-  size_t length;
-  uint32_t crc32c;
-};
-
-ExpectedResult expectedResults[] = {
-    // Zero-byte input
-    { 0, 0, ~0U },
-    // Small aligned inputs to test special cases in SIMD implementations
-    { 8, 1, 1543413366 },
-    { 8, 2, 523493126 },
-    { 8, 3, 1560427360 },
-    { 8, 4, 3422504776 },
-    { 8, 5, 447841138 },
-    { 8, 6, 3910050499 },
-    { 8, 7, 3346241981 },
-    // Small unaligned inputs
-    { 9, 1, 3855826643 },
-    { 10, 2, 560880875 },
-    { 11, 3, 1479707779 },
-    { 12, 4, 2237687071 },
-    { 13, 5, 4063855784 },
-    { 14, 6, 2553454047 },
-    { 15, 7, 1349220140 },
-    // Larger inputs to test leftover chunks at the end of aligned blocks
-    { 8, 8, 627613930 },
-    { 8, 9, 2105929409 },
-    { 8, 10, 2447068514 },
-    { 8, 11, 863807079 },
-    { 8, 12, 292050879 },
-    { 8, 13, 1411837737 },
-    { 8, 14, 2614515001 },
-    { 8, 15, 3579076296 },
-    { 8, 16, 2897079161 },
-    { 8, 17, 675168386 },
-    // // Much larger inputs
-    { 0, BUFFER_SIZE, 2096790750 },
-    { 1, BUFFER_SIZE / 2, 3854797577 },
-
-};
-
 TEST(CRC, StandardResults) {
-
-  // Original Fast_CRC32 tests.
  // From rfc3720 section B.4.
  char buf[32];

@@ -99,24 +50,6 @@ TEST(CRC, StandardResults) {
    0x00, 0x00, 0x00, 0x00,
  };
  ASSERT_EQ(0xd9963a56, Value(reinterpret_cast<char*>(data), sizeof(data)));
-
-  // 3-Way Crc32c tests ported from folly.
-  // Test 1: single computation
-  for (auto expected : expectedResults) {
-    uint32_t result = Value(buffer + expected.offset, expected.length);
-    EXPECT_EQ(~expected.crc32c, result);
-  }
-
-  // Test 2: stitching two computations
-  for (auto expected : expectedResults) {
-    size_t partialLength = expected.length / 2;
-    uint32_t partialChecksum = Value(buffer + expected.offset, partialLength);
-    uint32_t result = Extend(partialChecksum,
-        buffer + expected.offset + partialLength,
-        expected.length - partialLength);
-    EXPECT_EQ(~expected.crc32c, result);
-  }
-
 }

 TEST(CRC, Values) {
@@ -139,36 +72,7 @@ TEST(CRC, Mask) {
 }  // namespace crc32c
 }  // namespace rocksdb

-// copied from folly
-const uint64_t FNV_64_HASH_START = 14695981039346656037ULL;
-inline uint64_t fnv64_buf(const void* buf,
-                          size_t n,
-                          uint64_t hash = FNV_64_HASH_START) {
-  // forcing signed char, since other platforms can use unsigned
-  const signed char* char_buf = reinterpret_cast<const signed char*>(buf);
-
-  for (size_t i = 0; i < n; ++i) {
-    hash += (hash << 1) + (hash << 4) + (hash << 5) + (hash << 7) +
-      (hash << 8) + (hash << 40);
-    hash ^= char_buf[i];
-  }
-  return hash;
-}
-
 int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
-
-  // Populate a buffer with a deterministic pattern
-  // on which to compute checksums
-
-  const uint8_t* src = (uint8_t*)rocksdb::crc32c::buffer;
-  uint64_t* dst = (uint64_t*)rocksdb::crc32c::buffer;
-  const uint64_t* end = (const uint64_t*)(rocksdb::crc32c::buffer + rocksdb::crc32c::BUFFER_SIZE);
-  *dst++ = 0;
-  while (dst < end) {
-    *dst++ = fnv64_buf((const char*)src, sizeof(uint64_t));
-    src += sizeof(uint64_t);
-  }
-
  return RUN_ALL_TESTS();
 }
@@ -151,6 +151,7 @@ Status DeleteScheduler::MarkAsTrash(const std::string& file_path,
  Status s;
  if (DeleteScheduler::IsTrashFile(file_path)) {
    // This is already a trash file
+    *trash_file = file_path;
    return s;
  }

@@ -239,9 +239,6 @@ Status WritableFileWriter::Close() {
  // we need to let the file know where data ends.
  if (use_direct_io()) {
    interim = writable_file_->Truncate(filesize_);
-    if (interim.ok()) {
-      interim = writable_file_->Fsync();
-    }
    if (!interim.ok() && s.ok()) {
      s = interim;
    }
@@ -24,13 +24,6 @@ namespace rocksdb {
 // pointer (if not NULL) when one of the following happens:
 // (1) a thread terminates
 // (2) a ThreadLocalPtr is destroyed
-//
-// Warning: this function is called while holding a global mutex. The same mutex
-// is used (at least in some cases) by most methods of ThreadLocalPtr, and it's
-// shared across all instances of ThreadLocalPtr. Thereforere extra care
-// is needed to avoid deadlocks. In particular, the handler shouldn't lock any
-// mutexes and shouldn't call any methods of any ThreadLocalPtr instances,
-// unless you know what you're doing.
 typedef void (*UnrefHandler)(void* ptr);

 // ThreadLocalPtr stores only values of pointer type.  Different from
@@ -77,10 +77,10 @@ Status RandomTransactionInserter::DBGet(
    uint64_t ikey, bool get_for_update, uint64_t* int_value,
    std::string* full_key, bool* unexpected_error) {
  Status s;
-  // Five digits (since the largest uint16_t is 65535) plus the NUL
-  // end char.
-  char prefix_buf[6];
+  // four digits and zero end char
+  char prefix_buf[5];
  // Pad prefix appropriately so we can iterate over each set
+  assert(set_i + 1 <= 9999);
  snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1);
  // key format:  [SET#][random#]
  std::string skey = ToString(ikey);
@@ -125,7 +125,7 @@ bool RandomTransactionInserter::DoInsert(DB* db, Transaction* txn,
  bool unexpected_error = false;

  std::vector<uint16_t> set_vec(num_sets_);
-  std::iota(set_vec.begin(), set_vec.end(), static_cast<uint16_t>(0));
+  std::iota(set_vec.begin(), set_vec.end(), 0);
  std::random_shuffle(set_vec.begin(), set_vec.end(),
                      [&](uint64_t r) { return rand_->Uniform(r); });
  // For each set, pick a key at random and increment it
@@ -255,16 +255,15 @@ Status RandomTransactionInserter::Verify(DB* db, uint16_t num_sets,
  }

  std::vector<uint16_t> set_vec(num_sets);
-  std::iota(set_vec.begin(), set_vec.end(), static_cast<uint16_t>(0));
+  std::iota(set_vec.begin(), set_vec.end(), 0);
  if (rand) {
    std::random_shuffle(set_vec.begin(), set_vec.end(),
                        [&](uint64_t r) { return rand->Uniform(r); });
  }
  // For each set of keys with the same prefix, sum all the values
  for (uint16_t set_i : set_vec) {
-    // Five digits (since the largest uint16_t is 65535) plus the NUL
-    // end char.
-    char prefix_buf[6];
+    // four digits and zero end char
+    char prefix_buf[5];
    assert(set_i + 1 <= 9999);
    snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1);
    uint64_t total = 0;
@@ -146,17 +146,11 @@ class BackupEngineImpl : public BackupEngine {

  class BackupMeta {
   public:
-    BackupMeta(
-        const std::string& meta_filename, const std::string& meta_tmp_filename,
+    BackupMeta(const std::string& meta_filename,
        std::unordered_map<std::string, std::shared_ptr<FileInfo>>* file_infos,
        Env* env)
-        : timestamp_(0),
-          sequence_number_(0),
-          size_(0),
-          meta_filename_(meta_filename),
-          meta_tmp_filename_(meta_tmp_filename),
-          file_infos_(file_infos),
-          env_(env) {}
+      : timestamp_(0), sequence_number_(0), size_(0),
+        meta_filename_(meta_filename), file_infos_(file_infos), env_(env) {}

    BackupMeta(const BackupMeta&) = delete;
    BackupMeta& operator=(const BackupMeta&) = delete;
@@ -234,7 +228,6 @@ class BackupEngineImpl : public BackupEngine {
    uint64_t size_;
    std::string app_metadata_;
    std::string const meta_filename_;
-    std::string const meta_tmp_filename_;
    // files with relative paths (without "/" prefix!!)
    std::vector<std::shared_ptr<FileInfo>> files_;
    std::unordered_map<std::string, std::shared_ptr<FileInfo>>* file_infos_;
@@ -264,14 +257,12 @@ class BackupEngineImpl : public BackupEngine {
  inline std::string GetSharedFileRel(const std::string& file = "",
                                      bool tmp = false) const {
    assert(file.size() == 0 || file[0] != '/');
-    return std::string("shared/") + (tmp ? "." : "") + file +
-           (tmp ? ".tmp" : "");
+    return "shared/" + file + (tmp ? ".tmp" : "");
  }
  inline std::string GetSharedFileWithChecksumRel(const std::string& file = "",
                                                  bool tmp = false) const {
    assert(file.size() == 0 || file[0] != '/');
-    return GetSharedChecksumDirRel() + "/" + (tmp ? "." : "") + file +
-           (tmp ? ".tmp" : "");
+    return GetSharedChecksumDirRel() + "/" + file + (tmp ? ".tmp" : "");
  }
  inline std::string GetSharedFileWithChecksum(const std::string& file,
                                               const uint32_t checksum_value,
@@ -292,9 +283,8 @@ class BackupEngineImpl : public BackupEngine {
  inline std::string GetBackupMetaDir() const {
    return GetAbsolutePath("meta");
  }
-  inline std::string GetBackupMetaFile(BackupID backup_id, bool tmp) const {
-    return GetBackupMetaDir() + "/" + (tmp ? "." : "") +
-           rocksdb::ToString(backup_id) + (tmp ? ".tmp" : "");
+  inline std::string GetBackupMetaFile(BackupID backup_id) const {
+    return GetBackupMetaDir() + "/" + rocksdb::ToString(backup_id);
  }

  // If size_limit == 0, there is no size limit, copy everything.
@@ -615,11 +605,10 @@ Status BackupEngineImpl::Initialize() {
      continue;
    }
    assert(backups_.find(backup_id) == backups_.end());
-    backups_.insert(std::make_pair(
-        backup_id, unique_ptr<BackupMeta>(new BackupMeta(
-                       GetBackupMetaFile(backup_id, false /* tmp */),
-                       GetBackupMetaFile(backup_id, true /* tmp */),
-                       &backuped_file_infos_, backup_env_))));
+    backups_.insert(
+        std::make_pair(backup_id, unique_ptr<BackupMeta>(new BackupMeta(
+                                      GetBackupMetaFile(backup_id),
+                                      &backuped_file_infos_, backup_env_))));
  }

  latest_backup_id_ = 0;
@@ -747,11 +736,10 @@ Status BackupEngineImpl::CreateNewBackupWithMetadata(
  BackupID new_backup_id = latest_backup_id_ + 1;

  assert(backups_.find(new_backup_id) == backups_.end());
-  auto ret = backups_.insert(std::make_pair(
-      new_backup_id, unique_ptr<BackupMeta>(new BackupMeta(
-                         GetBackupMetaFile(new_backup_id, false /* tmp */),
-                         GetBackupMetaFile(new_backup_id, true /* tmp */),
-                         &backuped_file_infos_, backup_env_))));
+  auto ret = backups_.insert(
+      std::make_pair(new_backup_id, unique_ptr<BackupMeta>(new BackupMeta(
+                                        GetBackupMetaFile(new_backup_id),
+                                        &backuped_file_infos_, backup_env_))));
  assert(ret.second == true);
  auto& new_backup = ret.first->second;
  new_backup->RecordTimestamp();
@@ -1720,7 +1708,8 @@ Status BackupEngineImpl::BackupMeta::StoreToFile(bool sync) {
  EnvOptions env_options;
  env_options.use_mmap_writes = false;
  env_options.use_direct_writes = false;
-  s = env_->NewWritableFile(meta_tmp_filename_, &backup_meta_file, env_options);
+  s = env_->NewWritableFile(meta_filename_ + ".tmp", &backup_meta_file,
+                            env_options);
  if (!s.ok()) {
    return s;
  }
@@ -1760,7 +1749,7 @@ Status BackupEngineImpl::BackupMeta::StoreToFile(bool sync) {
    s = backup_meta_file->Close();
  }
  if (s.ok()) {
-    s = env_->RenameFile(meta_tmp_filename_, meta_filename_);
+    s = env_->RenameFile(meta_filename_ + ".tmp", meta_filename_);
  }
  return s;
 }
@@ -810,9 +810,9 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
  test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
  ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
  std::vector<std::string> should_have_written = {
-      "/shared/.00010.sst.tmp",    "/shared/.00011.sst.tmp",
+      "/shared/00010.sst.tmp",    "/shared/00011.sst.tmp",
      "/private/1.tmp/CURRENT",   "/private/1.tmp/MANIFEST-01",
-      "/private/1.tmp/00011.log", "/meta/.1.tmp"};
+      "/private/1.tmp/00011.log", "/meta/1.tmp"};
  AppendPath(backupdir_, should_have_written);
  test_backup_env_->AssertWrittenFiles(should_have_written);

@@ -828,9 +828,9 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
  ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
  // should not open 00010.sst - it's already there

-  should_have_written = {"/shared/.00015.sst.tmp", "/private/2.tmp/CURRENT",
+  should_have_written = {"/shared/00015.sst.tmp", "/private/2.tmp/CURRENT",
                         "/private/2.tmp/MANIFEST-01",
-                         "/private/2.tmp/00011.log", "/meta/.2.tmp"};
+                         "/private/2.tmp/00011.log", "/meta/2.tmp"};
  AppendPath(backupdir_, should_have_written);
  test_backup_env_->AssertWrittenFiles(should_have_written);

@@ -1169,7 +1169,7 @@ TEST_F(BackupableDBTest, DeleteTmpFiles) {
    } else {
      shared_tmp += "/shared";
    }
-    shared_tmp += "/.00006.sst.tmp";
+    shared_tmp += "/00006.sst.tmp";
    std::string private_tmp_dir = backupdir_ + "/private/10.tmp";
    std::string private_tmp_file = private_tmp_dir + "/00003.sst";
    file_manager_->WriteToFile(shared_tmp, "tmp");
@@ -317,10 +317,10 @@ TEST(RowValueTest, PurgeTtlShouldRemvoeAllColumnsExpired) {
  int64_t now = time(nullptr);

  auto row_value = CreateTestRowValue({
-    CreateTestColumnSpec(kColumn, 0, ToMicroSeconds(now)),
-    CreateTestColumnSpec(kExpiringColumn, 1, ToMicroSeconds(now - kTtl - 10)), //expired
-    CreateTestColumnSpec(kExpiringColumn, 2, ToMicroSeconds(now)), // not expired
-    CreateTestColumnSpec(kTombstone, 3, ToMicroSeconds(now))
+    std::make_tuple(kColumn, 0, ToMicroSeconds(now)),
+    std::make_tuple(kExpiringColumn, 1, ToMicroSeconds(now - kTtl - 10)), //expired
+    std::make_tuple(kExpiringColumn, 2, ToMicroSeconds(now)), // not expired
+    std::make_tuple(kTombstone, 3, ToMicroSeconds(now))
  });

  bool changed = false;
@@ -339,10 +339,10 @@ TEST(RowValueTest, ExpireTtlShouldConvertExpiredColumnsToTombstones) {
  int64_t now = time(nullptr);

  auto row_value = CreateTestRowValue({
-    CreateTestColumnSpec(kColumn, 0, ToMicroSeconds(now)),
-    CreateTestColumnSpec(kExpiringColumn, 1, ToMicroSeconds(now - kTtl - 10)), //expired
-    CreateTestColumnSpec(kExpiringColumn, 2, ToMicroSeconds(now)), // not expired
-    CreateTestColumnSpec(kTombstone, 3, ToMicroSeconds(now))
+    std::make_tuple(kColumn, 0, ToMicroSeconds(now)),
+    std::make_tuple(kExpiringColumn, 1, ToMicroSeconds(now - kTtl - 10)), //expired
+    std::make_tuple(kExpiringColumn, 2, ToMicroSeconds(now)), // not expired
+    std::make_tuple(kTombstone, 3, ToMicroSeconds(now))
  });

  bool changed = false;
@@ -145,21 +145,21 @@ TEST_F(CassandraFunctionalTest, SimpleMergeTest) {
  int64_t now = time(nullptr);

  store.Append("k1", CreateTestRowValue({
-    CreateTestColumnSpec(kTombstone, 0, ToMicroSeconds(now + 5)),
-    CreateTestColumnSpec(kColumn, 1, ToMicroSeconds(now + 8)),
-    CreateTestColumnSpec(kExpiringColumn, 2, ToMicroSeconds(now + 5)),
+    std::make_tuple(kTombstone, 0, ToMicroSeconds(now + 5)),
+    std::make_tuple(kColumn, 1, ToMicroSeconds(now + 8)),
+    std::make_tuple(kExpiringColumn, 2, ToMicroSeconds(now + 5)),
  }));
  store.Append("k1",CreateTestRowValue({
-    CreateTestColumnSpec(kColumn, 0, ToMicroSeconds(now + 2)),
-    CreateTestColumnSpec(kExpiringColumn, 1, ToMicroSeconds(now + 5)),
-    CreateTestColumnSpec(kTombstone, 2, ToMicroSeconds(now + 7)),
-    CreateTestColumnSpec(kExpiringColumn, 7, ToMicroSeconds(now + 17)),
+    std::make_tuple(kColumn, 0, ToMicroSeconds(now + 2)),
+    std::make_tuple(kExpiringColumn, 1, ToMicroSeconds(now + 5)),
+    std::make_tuple(kTombstone, 2, ToMicroSeconds(now + 7)),
+    std::make_tuple(kExpiringColumn, 7, ToMicroSeconds(now + 17)),
  }));
  store.Append("k1", CreateTestRowValue({
-    CreateTestColumnSpec(kExpiringColumn, 0, ToMicroSeconds(now + 6)),
-    CreateTestColumnSpec(kTombstone, 1, ToMicroSeconds(now + 5)),
-    CreateTestColumnSpec(kColumn, 2, ToMicroSeconds(now + 4)),
-    CreateTestColumnSpec(kTombstone, 11, ToMicroSeconds(now + 11)),
+    std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now + 6)),
+    std::make_tuple(kTombstone, 1, ToMicroSeconds(now + 5)),
+    std::make_tuple(kColumn, 2, ToMicroSeconds(now + 4)),
+    std::make_tuple(kTombstone, 11, ToMicroSeconds(now + 11)),
  }));

  auto ret = store.Get("k1");
@@ -180,16 +180,16 @@ TEST_F(CassandraFunctionalTest,
  int64_t now= time(nullptr);

  store.Append("k1", CreateTestRowValue({
-    CreateTestColumnSpec(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 20)), //expired
-    CreateTestColumnSpec(kExpiringColumn, 1, ToMicroSeconds(now - kTtl + 10)), // not expired
-    CreateTestColumnSpec(kTombstone, 3, ToMicroSeconds(now))
+    std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 20)), //expired
+    std::make_tuple(kExpiringColumn, 1, ToMicroSeconds(now - kTtl + 10)), // not expired
+    std::make_tuple(kTombstone, 3, ToMicroSeconds(now))
  }));

  store.Flush();

  store.Append("k1",CreateTestRowValue({
-    CreateTestColumnSpec(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 10)), //expired
-    CreateTestColumnSpec(kColumn, 2, ToMicroSeconds(now))
+    std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 10)), //expired
+    std::make_tuple(kColumn, 2, ToMicroSeconds(now))
  }));

  store.Flush();
@@ -213,16 +213,16 @@ TEST_F(CassandraFunctionalTest,
  int64_t now = time(nullptr);

  store.Append("k1", CreateTestRowValue({
-    CreateTestColumnSpec(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 20)), //expired
-    CreateTestColumnSpec(kExpiringColumn, 1, ToMicroSeconds(now)), // not expired
-    CreateTestColumnSpec(kTombstone, 3, ToMicroSeconds(now))
+    std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 20)), //expired
+    std::make_tuple(kExpiringColumn, 1, ToMicroSeconds(now)), // not expired
+    std::make_tuple(kTombstone, 3, ToMicroSeconds(now))
  }));

  store.Flush();

  store.Append("k1",CreateTestRowValue({
-    CreateTestColumnSpec(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 10)), //expired
-    CreateTestColumnSpec(kColumn, 2, ToMicroSeconds(now))
+    std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 10)), //expired
+    std::make_tuple(kColumn, 2, ToMicroSeconds(now))
  }));

  store.Flush();
@@ -244,14 +244,14 @@ TEST_F(CassandraFunctionalTest,
  int64_t now = time(nullptr);

  store.Append("k1", CreateTestRowValue({
-    CreateTestColumnSpec(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 20)),
-    CreateTestColumnSpec(kExpiringColumn, 1, ToMicroSeconds(now - kTtl - 20)),
+    std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 20)),
+    std::make_tuple(kExpiringColumn, 1, ToMicroSeconds(now - kTtl - 20)),
  }));

  store.Flush();

  store.Append("k1",CreateTestRowValue({
-    CreateTestColumnSpec(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 10)),
+    std::make_tuple(kExpiringColumn, 0, ToMicroSeconds(now - kTtl - 10)),
  }));

  store.Flush();
@@ -266,18 +266,18 @@ TEST_F(CassandraFunctionalTest,
  int64_t now = time(nullptr);

  store.Append("k1", CreateTestRowValue({
-    CreateTestColumnSpec(kTombstone, 0, ToMicroSeconds(now - gc_grace_period_in_seconds_ - 1)),
-    CreateTestColumnSpec(kColumn, 1, ToMicroSeconds(now))
+    std::make_tuple(kTombstone, 0, ToMicroSeconds(now - gc_grace_period_in_seconds_ - 1)),
+    std::make_tuple(kColumn, 1, ToMicroSeconds(now))
  }));

  store.Append("k2", CreateTestRowValue({
-    CreateTestColumnSpec(kColumn, 0, ToMicroSeconds(now))
+    std::make_tuple(kColumn, 0, ToMicroSeconds(now))
  }));

  store.Flush();

  store.Append("k1",CreateTestRowValue({
-    CreateTestColumnSpec(kColumn, 1, ToMicroSeconds(now)),
+    std::make_tuple(kColumn, 1, ToMicroSeconds(now)),
  }));

  store.Flush();
@@ -296,7 +296,7 @@ TEST_F(CassandraFunctionalTest, CompactionShouldRemoveTombstoneFromPut) {
  int64_t now = time(nullptr);

  store.Put("k1", CreateTestRowValue({
-    CreateTestColumnSpec(kTombstone, 0, ToMicroSeconds(now - gc_grace_period_in_seconds_ - 1)),
+    std::make_tuple(kTombstone, 0, ToMicroSeconds(now - gc_grace_period_in_seconds_ - 1)),
  }));

  store.Flush();
@@ -15,27 +15,27 @@ TEST(RowValueMergeTest, Merge) {
  std::vector<RowValue> row_values;
  row_values.push_back(
    CreateTestRowValue({
-      CreateTestColumnSpec(kTombstone, 0, 5),
-      CreateTestColumnSpec(kColumn, 1, 8),
-      CreateTestColumnSpec(kExpiringColumn, 2, 5),
+      std::make_tuple(kTombstone, 0, 5),
+      std::make_tuple(kColumn, 1, 8),
+      std::make_tuple(kExpiringColumn, 2, 5),
    })
  );

  row_values.push_back(
    CreateTestRowValue({
-      CreateTestColumnSpec(kColumn, 0, 2),
-      CreateTestColumnSpec(kExpiringColumn, 1, 5),
-      CreateTestColumnSpec(kTombstone, 2, 7),
-      CreateTestColumnSpec(kExpiringColumn, 7, 17),
+      std::make_tuple(kColumn, 0, 2),
+      std::make_tuple(kExpiringColumn, 1, 5),
+      std::make_tuple(kTombstone, 2, 7),
+      std::make_tuple(kExpiringColumn, 7, 17),
    })
  );

  row_values.push_back(
    CreateTestRowValue({
-      CreateTestColumnSpec(kExpiringColumn, 0, 6),
-      CreateTestColumnSpec(kTombstone, 1, 5),
-      CreateTestColumnSpec(kColumn, 2, 4),
-      CreateTestColumnSpec(kTombstone, 11, 11),
+      std::make_tuple(kExpiringColumn, 0, 6),
+      std::make_tuple(kTombstone, 1, 5),
+      std::make_tuple(kColumn, 2, 4),
+      std::make_tuple(kTombstone, 11, 11),
    })
  );

@@ -60,24 +60,24 @@ TEST(RowValueMergeTest, MergeWithRowTombstone) {
  // This row's timestamp is smaller than tombstone.
  row_values.push_back(
    CreateTestRowValue({
-      CreateTestColumnSpec(kColumn, 0, 5),
-      CreateTestColumnSpec(kColumn, 1, 6),
+      std::make_tuple(kColumn, 0, 5),
+      std::make_tuple(kColumn, 1, 6),
    })
  );

  // Some of the column's row is smaller, some is larger.
  row_values.push_back(
    CreateTestRowValue({
-      CreateTestColumnSpec(kColumn, 2, 10),
-      CreateTestColumnSpec(kColumn, 3, 12),
+      std::make_tuple(kColumn, 2, 10),
+      std::make_tuple(kColumn, 3, 12),
    })
  );

  // All of the column's rows are larger than tombstone.
  row_values.push_back(
    CreateTestRowValue({
-      CreateTestColumnSpec(kColumn, 4, 13),
-      CreateTestColumnSpec(kColumn, 5, 14),
+      std::make_tuple(kColumn, 4, 13),
+      std::make_tuple(kColumn, 5, 14),
    })
  );

@@ -129,7 +129,7 @@ std::shared_ptr<Tombstone> ExpiringColumn::ToTombstone() const {
  int64_t marked_for_delete_at =
    std::chrono::duration_cast<std::chrono::microseconds>(expired_at).count();
  return std::make_shared<Tombstone>(
-    static_cast<int8_t>(ColumnTypeMask::DELETION_MASK),
+    ColumnTypeMask::DELETION_MASK,
    Index(),
    local_deletion_time,
    marked_for_delete_at);
@@ -29,12 +29,6 @@ std::shared_ptr<ColumnBase> CreateTestColumn(int8_t mask,
  }
 }

-std::tuple<int8_t, int8_t, int64_t> CreateTestColumnSpec(int8_t mask,
-                                                         int8_t index,
-                                                         int64_t timestamp) {
-  return std::make_tuple(mask, index, timestamp);
-}
-
 RowValue CreateTestRowValue(
    std::vector<std::tuple<int8_t, int8_t, int64_t>> column_specs) {
  std::vector<std::shared_ptr<ColumnBase>> columns;
@@ -23,10 +23,6 @@ std::shared_ptr<ColumnBase> CreateTestColumn(int8_t mask,
                                             int8_t index,
                                             int64_t timestamp);

-std::tuple<int8_t, int8_t, int64_t> CreateTestColumnSpec(int8_t mask,
-                                                         int8_t index,
-                                                         int64_t timestamp);
-
 RowValue CreateTestRowValue(
    std::vector<std::tuple<int8_t, int8_t, int64_t>> column_specs);

@@ -207,9 +207,7 @@ Status CheckpointImpl::CreateCustomCheckpoint(

    TEST_SYNC_POINT("CheckpointImpl::CreateCheckpoint:SavedLiveFiles1");
    TEST_SYNC_POINT("CheckpointImpl::CreateCheckpoint:SavedLiveFiles2");
-    if (db_options.manual_wal_flush) {
-      db_->FlushWAL(false /* sync */);
-    }
+    db_->FlushWAL(false /* sync */);
  }
  // if we have more than one column family, we need to also get WAL files
  if (s.ok()) {
@@ -1485,5 +1485,4 @@ EnvLibrados* EnvLibrados::Default() {
                                 default_pool_name);
  return &default_env;
 }
-// @lint-ignore TXT4 T25377293 Grandfathered in
 }
@@ -89,7 +89,6 @@ class CacheActivityLogger {
    log_line += key.ToString(true);
    log_line += " - ";
    AppendNumberTo(&log_line, size);
-  // @lint-ignore TXT2 T25377293 Grandfathered in
 		log_line += "\n";

    // line format: "ADD - <KEY> - <KEY-SIZE>"
@@ -193,7 +193,6 @@ TEST_F(SimCacheTest, SimCacheLogging) {
  ASSERT_EQ(add_num, num_block_entries);

  // Log things again but stop logging automatically after reaching 512 bytes
- // @lint-ignore TXT2 T25377293 Grandfathered in
 	int max_size = 512;
  ASSERT_OK(sim_cache->StartActivityLogging(log_file, env_, max_size));
  for (int it = 0; it < 10; it++) {
@@ -436,6 +436,83 @@ TEST_P(WritePreparedTransactionTest, MaybeUpdateOldCommitMap) {
  MaybeUpdateOldCommitMapTestWithNext(p, c, s, ns, false);
 }

+// Test that the entries in old_commit_map_ get garbage collected properly
+TEST_P(WritePreparedTransactionTest, OldCommitMapGC) {
+  const size_t snapshot_cache_bits = 0;
+  const size_t commit_cache_bits = 0;
+  DBImpl* mock_db = new DBImpl(options, dbname);
+  std::unique_ptr<WritePreparedTxnDBMock> wp_db(new WritePreparedTxnDBMock(
+      mock_db, txn_db_options, snapshot_cache_bits, commit_cache_bits));
+
+  SequenceNumber seq = 0;
+  // Take the first snapshot that overlaps with two txn
+  auto prep_seq = ++seq;
+  wp_db->AddPrepared(prep_seq);
+  auto prep_seq2 = ++seq;
+  wp_db->AddPrepared(prep_seq2);
+  auto snap_seq1 = seq;
+  wp_db->TakeSnapshot(snap_seq1);
+  auto commit_seq = ++seq;
+  wp_db->AddCommitted(prep_seq, commit_seq);
+  auto commit_seq2 = ++seq;
+  wp_db->AddCommitted(prep_seq2, commit_seq2);
+  // Take the 2nd and 3rd snapshot that overlap with the same txn
+  prep_seq = ++seq;
+  wp_db->AddPrepared(prep_seq);
+  auto snap_seq2 = seq;
+  wp_db->TakeSnapshot(snap_seq2);
+  seq++;
+  auto snap_seq3 = seq;
+  wp_db->TakeSnapshot(snap_seq3);
+  seq++;
+  commit_seq = ++seq;
+  wp_db->AddCommitted(prep_seq, commit_seq);
+  // Make sure max_evicted_seq_ will be larger than 2nd snapshot by evicting the
+  // only item in the commit_cache_ via another commit.
+  prep_seq = ++seq;
+  wp_db->AddPrepared(prep_seq);
+  commit_seq = ++seq;
+  wp_db->AddCommitted(prep_seq, commit_seq);
+
+  // Verify that the evicted commit entries for all snapshots are in the
+  // old_commit_map_
+  {
+    ASSERT_FALSE(wp_db->old_commit_map_empty_.load());
+    ReadLock rl(&wp_db->old_commit_map_mutex_);
+    ASSERT_EQ(3, wp_db->old_commit_map_.size());
+    ASSERT_EQ(2, wp_db->old_commit_map_[snap_seq1].size());
+    ASSERT_EQ(1, wp_db->old_commit_map_[snap_seq2].size());
+    ASSERT_EQ(1, wp_db->old_commit_map_[snap_seq3].size());
+  }
+
+  // Verify that the 2nd snapshot is cleaned up after the release
+  wp_db->ReleaseSnapshotInternal(snap_seq2);
+  {
+    ASSERT_FALSE(wp_db->old_commit_map_empty_.load());
+    ReadLock rl(&wp_db->old_commit_map_mutex_);
+    ASSERT_EQ(2, wp_db->old_commit_map_.size());
+    ASSERT_EQ(2, wp_db->old_commit_map_[snap_seq1].size());
+    ASSERT_EQ(1, wp_db->old_commit_map_[snap_seq3].size());
+  }
+
+  // Verify that the 1st snapshot is cleaned up after the release
+  wp_db->ReleaseSnapshotInternal(snap_seq1);
+  {
+    ASSERT_FALSE(wp_db->old_commit_map_empty_.load());
+    ReadLock rl(&wp_db->old_commit_map_mutex_);
+    ASSERT_EQ(1, wp_db->old_commit_map_.size());
+    ASSERT_EQ(1, wp_db->old_commit_map_[snap_seq3].size());
+  }
+
+  // Verify that the 3rd snapshot is cleaned up after the release
+  wp_db->ReleaseSnapshotInternal(snap_seq3);
+  {
+    ASSERT_TRUE(wp_db->old_commit_map_empty_.load());
+    ReadLock rl(&wp_db->old_commit_map_mutex_);
+    ASSERT_EQ(0, wp_db->old_commit_map_.size());
+  }
+}
+
 TEST_P(WritePreparedTransactionTest, CheckAgainstSnapshotsTest) {
  std::vector<SequenceNumber> snapshots = {100l, 200l, 300l, 400l, 500l,
                                           600l, 700l, 800l, 900l};
@@ -480,20 +557,24 @@ TEST_P(WritePreparedTransactionTest, SnapshotConcurrentAccessTest) {
  // We have a sync point in the method under test after checking each snapshot.
  // If you increase the max number of snapshots in this test, more sync points
  // in the methods must also be added.
-  const std::vector<SequenceNumber> snapshots = {10l, 20l, 30l, 40l, 50l,
-                                                 60l, 70l, 80l, 90l, 100l};
+  const std::vector<SequenceNumber> snapshots = {10l, 20l, 30l, 40l, 50l, 60l};
+  // TODO(myabandeh): increase the snapshots list for pre-release tests
+  // const std::vector<SequenceNumber> snapshots = {10l, 20l, 30l, 40l, 50l,
+  //                                               60l, 70l, 80l, 90l, 100l};
  const size_t snapshot_cache_bits = 2;
  // Safety check to express the intended size in the test. Can be adjusted if
  // the snapshots lists changed.
-  assert((1ul << snapshot_cache_bits) * 2 + 2 == snapshots.size());
+  assert((1ul << snapshot_cache_bits) + 2 == snapshots.size());
  SequenceNumber version = 1000l;
  // Choose the cache size so that the new snapshot list could replace all the
  // existing items in the cache and also have some overflow.
  DBImpl* mock_db = new DBImpl(options, dbname);
  std::unique_ptr<WritePreparedTxnDBMock> wp_db(
      new WritePreparedTxnDBMock(mock_db, txn_db_options, snapshot_cache_bits));
-  // Add up to 2 items that do not fit into the cache
-  for (size_t old_size = 1; old_size <= wp_db->SNAPSHOT_CACHE_SIZE + 2;
+  // TODO(myabandeh): increase this number for pre-release tests
+  const size_t extra = 1;
+  // Add up to extra items that do not fit into the cache
+  for (size_t old_size = 1; old_size <= wp_db->SNAPSHOT_CACHE_SIZE + extra;
       old_size++) {
    const std::vector<SequenceNumber> old_snapshots(
        snapshots.begin(), snapshots.begin() + old_size);
@@ -12,6 +12,7 @@
 #include "utilities/transactions/write_prepared_txn_db.h"

 #include <inttypes.h>
+#include <algorithm>
 #include <string>
 #include <unordered_set>
 #include <vector>
@@ -234,8 +235,8 @@ bool WritePreparedTxnDB::IsInSnapshot(uint64_t prep_seq,
  // delayed_prepared_. Also we move evicted entries from commit cache to
  // old_commit_map_ if it overlaps with any snapshot. Since prep_seq <=
  // max_evicted_seq_, we have three cases: i) in delayed_prepared_, ii) in
-  // old_commit_map_, iii) committed with no conflict with any snapshot (i)
-  // delayed_prepared_ is checked above
+  // old_commit_map_, iii) committed with no conflict with any snapshot. Case
+  // (i) delayed_prepared_ is checked above
  if (max_evicted_seq < snapshot_seq) {  // then (ii) cannot be the case
    // only (iii) is the case: committed
    // commit_seq <= max_evicted_seq_ < snapshot_seq => commit_seq <
@@ -255,12 +256,17 @@ bool WritePreparedTxnDB::IsInSnapshot(uint64_t prep_seq,
    return true;
  }
  {
-    // We should not normally reach here
-    // TODO(myabandeh): check only if snapshot_seq is in the list of snaphots
+    // We should not normally reach here unless sapshot_seq is old. This is a
+    // rare case and it is ok to pay the cost of mutex ReadLock for such old,
+    // reading transactions.
    ReadLock rl(&old_commit_map_mutex_);
-    auto old_commit_entry = old_commit_map_.find(prep_seq);
-    if (old_commit_entry == old_commit_map_.end() ||
-        old_commit_entry->second <= snapshot_seq) {
+    auto prep_set_entry = old_commit_map_.find(snapshot_seq);
+    bool found = prep_set_entry != old_commit_map_.end();
+    if (found) {
+      auto& vec = prep_set_entry->second;
+      found = std::binary_search(vec.begin(), vec.end(), prep_seq);
+    }
+    if (!found) {
      ROCKSDB_LOG_DETAILS(
          info_log_, "IsInSnapshot %" PRIu64 " in %" PRIu64 " returns %" PRId32,
          prep_seq, snapshot_seq, 1);
@@ -322,6 +328,9 @@ void WritePreparedTxnDB::AddCommitted(uint64_t prepare_seq,
  bool to_be_evicted = GetCommitEntry(indexed_seq, &evicted_64b, &evicted);
  if (to_be_evicted) {
    auto prev_max = max_evicted_seq_.load(std::memory_order_acquire);
+    ROCKSDB_LOG_DETAILS(info_log_,
+                        "Evicting %" PRIu64 ",%" PRIu64 " with max %" PRIu64,
+                        evicted.prep_seq, evicted.commit_seq, prev_max);
    if (prev_max < evicted.commit_seq) {
      // Inc max in larger steps to avoid frequent updates
      auto max_evicted_seq = evicted.commit_seq + INC_STEP_FOR_MAX_EVICTED;
@@ -431,6 +440,36 @@ const std::vector<SequenceNumber> WritePreparedTxnDB::GetSnapshotListFromDB(
  return db_impl_->snapshots().GetAll(nullptr, max);
 }

+void WritePreparedTxnDB::ReleaseSnapshot(const Snapshot* snapshot) {
+  auto snap_seq = snapshot->GetSequenceNumber();
+  ReleaseSnapshotInternal(snap_seq);
+  db_impl_->ReleaseSnapshot(snapshot);
+}
+
+void WritePreparedTxnDB::ReleaseSnapshotInternal(
+    const SequenceNumber snap_seq) {
+  // relax is enough since max increases monotonically, i.e., if snap_seq <
+  // old_max => snap_seq < new_max as well.
+  if (snap_seq < max_evicted_seq_.load(std::memory_order_relaxed)) {
+    // Then this is a rare case that transaction did not finish before max
+    // advances. It is expected for a few read-only backup snapshots. For such
+    // snapshots we might have kept around a couple of entries in the
+    // old_commit_map_. Check and do garbage collection if that is the case.
+    bool need_gc = false;
+    {
+      ReadLock rl(&old_commit_map_mutex_);
+      auto prep_set_entry = old_commit_map_.find(snap_seq);
+      need_gc = prep_set_entry != old_commit_map_.end();
+    }
+    if (need_gc) {
+      WriteLock wl(&old_commit_map_mutex_);
+      old_commit_map_.erase(snap_seq);
+      old_commit_map_empty_.store(old_commit_map_.empty(),
+                                  std::memory_order_release);
+    }
+  }
+}
+
 void WritePreparedTxnDB::UpdateSnapshots(
    const std::vector<SequenceNumber>& snapshots,
    const SequenceNumber& version) {
@@ -541,8 +580,8 @@ void WritePreparedTxnDB::CheckAgainstSnapshots(const CommitEntry& evicted) {
 bool WritePreparedTxnDB::MaybeUpdateOldCommitMap(
    const uint64_t& prep_seq, const uint64_t& commit_seq,
    const uint64_t& snapshot_seq, const bool next_is_larger = true) {
-  // If we do not store an entry in old_commit_map we assume it is committed in
-  // all snapshots. if commit_seq <= snapshot_seq, it is considered already in
+  // If we do not store an entry in old_commit_map_ we assume it is committed in
+  // all snapshots. If commit_seq <= snapshot_seq, it is considered already in
  // the snapshot so we need not to keep the entry around for this snapshot.
  if (commit_seq <= snapshot_seq) {
    // continue the search if the next snapshot could be smaller than commit_seq
@@ -552,9 +591,11 @@ bool WritePreparedTxnDB::MaybeUpdateOldCommitMap(
  if (prep_seq <= snapshot_seq) {  // overlapping range
    WriteLock wl(&old_commit_map_mutex_);
    old_commit_map_empty_.store(false, std::memory_order_release);
-    old_commit_map_[prep_seq] = commit_seq;
-    // Storing once is enough. No need to check it for other snapshots.
-    return false;
+    auto& vec = old_commit_map_[snapshot_seq];
+    vec.insert(std::upper_bound(vec.begin(), vec.end(), prep_seq), prep_seq);
+    // We need to store it once for each overlapping snapshot. Returning true to
+    // continue the search if there is more overlapping snapshot.
+    return true;
  }
  // continue the search if the next snapshot could be larger than prep_seq
  return next_is_larger;
@@ -90,6 +90,8 @@ class WritePreparedTxnDB : public PessimisticTransactionDB {
      const std::vector<ColumnFamilyHandle*>& column_families,
      std::vector<Iterator*>* iterators) override;

+  virtual void ReleaseSnapshot(const Snapshot* snapshot) override;
+
  // Check whether the transaction that wrote the value with seqeunce number seq
  // is visible to the snapshot with sequence number snapshot_seq
  bool IsInSnapshot(uint64_t seq, uint64_t snapshot_seq) const;
@@ -198,6 +200,7 @@ class WritePreparedTxnDB : public PessimisticTransactionDB {
  friend class WritePreparedTransactionTest_AdvanceMaxEvictedSeqBasicTest_Test;
  friend class WritePreparedTransactionTest_BasicRecoveryTest_Test;
  friend class WritePreparedTransactionTest_IsInSnapshotEmptyMapTest_Test;
+  friend class WritePreparedTransactionTest_OldCommitMapGC_Test;
  friend class WritePreparedTransactionTest_RollbackTest_Test;

  void Init(const TransactionDBOptions& /* unused */);
@@ -269,6 +272,10 @@ class WritePreparedTxnDB : public PessimisticTransactionDB {
  virtual const std::vector<SequenceNumber> GetSnapshotListFromDB(
      SequenceNumber max);

+  // Will be called by the public ReleaseSnapshot method. Does the maintenance
+  // internal to WritePreparedTxnDB
+  void ReleaseSnapshotInternal(const SequenceNumber snap_seq);
+
  // Update the list of snapshots corresponding to the soon-to-be-updated
  // max_eviceted_seq_. Thread-safety: this function can be called concurrently.
  // The concurrent invocations of this function is equivalent to a serial
@@ -287,9 +294,8 @@ class WritePreparedTxnDB : public PessimisticTransactionDB {

  // Add a new entry to old_commit_map_ if prep_seq <= snapshot_seq <
  // commit_seq. Return false if checking the next snapshot(s) is not needed.
-  // This is the case if the entry already added to old_commit_map_ or none of
-  // the next snapshots could satisfy the condition. next_is_larger: the next
-  // snapshot will be a larger value
+  // This is the case if none of the next snapshots could satisfy the condition.
+  // next_is_larger: the next snapshot will be a larger value
  bool MaybeUpdateOldCommitMap(const uint64_t& prep_seq,
                               const uint64_t& commit_seq,
                               const uint64_t& snapshot_seq,
@@ -333,10 +339,14 @@ class WritePreparedTxnDB : public PessimisticTransactionDB {
  // it to be too large either as it would cause stalls by doing too much
  // maintenance work under the lock.
  size_t INC_STEP_FOR_MAX_EVICTED = 1;
-  // A map of the evicted entries from commit_cache_ that has to be kept around
-  // to service the old snapshots. This is expected to be empty normally.
+  // A map from old snapshots (expected to be used by a few read-only txns) to
+  // prpared sequence number of the evicted entries from commit_cache_ that
+  // overlaps with such snapshot. These are the prepared sequence numbers that
+  // the snapshot, to which they are mapped, cannot assume to be committed just
+  // because it is no longer in the commit_cache_. The vector must be sorted
+  // after each update.
  // Thread-safety is provided with old_commit_map_mutex_.
-  std::map<uint64_t, uint64_t> old_commit_map_;
+  std::map<SequenceNumber, std::vector<SequenceNumber>> old_commit_map_;
  // A set of long-running prepared transactions that are not finished by the
  // time max_evicted_seq_ advances their sequence number. This is expected to
  // be empty normally. Thread-safety is provided with prepared_mutex_.
Author	SHA1	Message	Date
Siying Dong	def6a00740	Print out compression type of new SST files in logging Summary: Closes https://github.com/facebook/rocksdb/pull/3264 Differential Revision: D6552768 Pulled By: siying fbshipit-source-id: 6303110aff22f341d5cff41f8d2d4f138a53652d	2017-12-14 10:27:43 -08:00
Siying Dong	6b77c07379	NUMBER_BLOCK_COMPRESSED, etc, shouldn't be treated as timer counter Summary: NUMBER_BLOCK_DECOMPRESSED and NUMBER_BLOCK_COMPRESSED are not reported unless the stats level contain detailed timers, which is wrong. They are normal counters. Fix it. Closes https://github.com/facebook/rocksdb/pull/3263 Differential Revision: D6552519 Pulled By: siying fbshipit-source-id: 40899ccea7b2856bb39752616657c0bfd432f6f9	2017-12-14 10:27:43 -08:00
Maysam Yabandeh	cd2e5cae7f	WritePrepared Txn: make db_stress transactional Summary: Add "--use_txn" option to use transactional API in db_stress, default being WRITE_PREPARED policy, which is the main intention of modifying db_stress. It also extend the existing snapshots to verify that before releasing a snapshot a read from it returns the same value as before. Closes https://github.com/facebook/rocksdb/pull/3243 Differential Revision: D6556912 Pulled By: maysamyabandeh fbshipit-source-id: 1ae31465be362d44bd06e635e2e9e49a1da11268	2017-12-13 11:57:29 -08:00
Maysam Yabandeh	546a63272f	disableWAL with WriteImplWALOnly Summary: Currently WriteImplWALOnly simply returns when disableWAL is set. This is an incorrect behavior since it does not allocated the sequence number, which is a side-effect of writing to the WAL. This patch fixes the issue. Closes https://github.com/facebook/rocksdb/pull/3262 Differential Revision: D6550974 Pulled By: maysamyabandeh fbshipit-source-id: 745a83ae8f04e7ca6c8ffb247d6ef16c287c52e7	2017-12-13 07:57:44 -08:00
Maysam Yabandeh	35dfbd58dd	WritePrepared Txn: GC old_commit_map_ Summary: Garbage collect entries from old_commit_map_ when the corresponding snapshots are released. Closes https://github.com/facebook/rocksdb/pull/3247 Differential Revision: D6528478 Pulled By: maysamyabandeh fbshipit-source-id: 15d1566d85d4ac07036bc0dc47418f6c3228d4bf	2017-12-13 07:57:43 -08:00
Zhongyi Xie	51c2ea0feb	Reduce heavy hitter for Get operation Summary: This PR addresses the following heavy hitters in `Get` operation by moving calls to `StatisticsImpl::recordTick` from `BlockBasedTable` to `Version::Get` - rocksdb.block.cache.bytes.write - rocksdb.block.cache.add - rocksdb.block.cache.data.miss - rocksdb.block.cache.data.bytes.insert - rocksdb.block.cache.data.add - rocksdb.block.cache.hit - rocksdb.block.cache.data.hit - rocksdb.block.cache.bytes.read The db_bench statistics before and after the change are: \|1GB block read\|Children \|Self \|Command \|Shared Object \|Symbol\| \|---\|---\|---\|---\|---\|---\| \|master: \|4.22% \|1.31% \|db_bench \|db_bench \|[.] rocksdb::StatisticsImpl::recordTick\| \|updated: \|0.51% \|0.21% \|db_bench \|db_bench \|[.] rocksdb::StatisticsImpl::recordTick\| \| \|0.14% \|0.14% \|db_bench \|db_bench \|[.] rocksdb::GetContext::record_counters\| \|1MB block read\|Children \|Self \|Command \|Shared Object \|Symbol\| \|---\|---\|---\|---\|---\|---\| \|master: \|3.48% \|1.08% \|db_bench \|db_bench \|[.] rocksdb::StatisticsImpl::recordTick\| \|updated: \|0.80% \|0.31% \|db_bench \|db_bench \|[.] rocksdb::StatisticsImpl::recordTick\| \| \|0.35% \|0.35% \|db_bench \|db_bench \|[.] rocksdb::GetContext::record_counters\| Closes https://github.com/facebook/rocksdb/pull/3172 Differential Revision: D6330532 Pulled By: miasantreble fbshipit-source-id: 2b492959e00a3db29e9437ecdcc5e48ca4ec5741	2017-12-12 21:11:33 -08:00
Islam AbdelRahman	9089373a01	Fix DeleteScheduler::MarkAsTrash() handling existing trash Summary: DeleteScheduler::MarkAsTrash() don't handle existing .trash files correctly This cause rocksdb to not being able to delete existing .trash files on restart Closes https://github.com/facebook/rocksdb/pull/3261 Differential Revision: D6548003 Pulled By: IslamAbdelRahman fbshipit-source-id: c3800639412e587a690062c63076a5a08881e0e6	2017-12-12 18:17:13 -08:00
Yi Wu	7393ef779c	Fix BlockFetcher ASAN error Summary: Some call sites of BlockFetcher create temporary ReadOptions and pass to BlockFetcher. The temporary object will be gone after BlockFetcher construction but BlockFetcher keep its reference, causing stack-use-after-scope. Fixing it. Closes https://github.com/facebook/rocksdb/pull/3258 Differential Revision: D6547152 Pulled By: yiwu-arbug fbshipit-source-id: 6b49e9dd46bb72307f5d8f88ea15faacff35b9bc	2017-12-12 12:12:38 -08:00
Souvik Banerjee	4bcb7fb148	Update transaction_test_util.cc Summary: Fixes a compile error on gcc 7.2.1 (-Werror=format-truncation=). Closes https://github.com/facebook/rocksdb/pull/3248 Differential Revision: D6546515 Pulled By: yiwu-arbug fbshipit-source-id: bd78cca63f2af376faceccb1838d2d4cc9208fef	2017-12-12 12:12:38 -08:00
Yi Wu	e3a06f12d2	WritePrepared Txn: fix compaction filter snapshot checks Summary: Add snapshot_checker check whenever we need to check sequence against snapshots and decide what to do with an input key. The changes are related to one of: * compaction filter * single delete * delete at bottom level * merge Closes https://github.com/facebook/rocksdb/pull/3251 Differential Revision: D6537850 Pulled By: yiwu-arbug fbshipit-source-id: 3faba40ed5e37779f4a0cb7ae78af9546659c7f2	2017-12-12 11:12:24 -08:00
Siying Dong	a9c8d4ef15	Fix memory issue introduced by `2f1a3a4d74` Summary: Closes https://github.com/facebook/rocksdb/pull/3256 Differential Revision: D6541714 Pulled By: siying fbshipit-source-id: 40efd89b68587a9d58cfe6f4eebd771c2d9f1542	2017-12-11 18:27:28 -08:00
Zhongyi Xie	bb5ed4b1d1	exclude DynamicUniversalCompactionOptions from ROCKSDB_LITE Summary: since [SetOptions](https://github.com/facebook/rocksdb/blob/master/db/db_impl.cc#L494) is not supported in ROCKSDB_LITE Right now unit test under lite is broken Closes https://github.com/facebook/rocksdb/pull/3253 Differential Revision: D6539428 Pulled By: miasantreble fbshipit-source-id: 13172b8ecbd75682330726498ea198969bc3e637	2017-12-11 16:28:20 -08:00
Siying Dong	0d5692e02b	Switch version to 5.10 Summary: Closes https://github.com/facebook/rocksdb/pull/3252 Differential Revision: D6539373 Pulled By: siying fbshipit-source-id: ce7c3d3fe625852179055295da9cf7bc80755025	2017-12-11 15:42:01 -08:00
Siying Dong	2f1a3a4d74	Refactor ReadBlockContents() Summary: Divide ReadBlockContents() to multiple sub-functions. Maintaining the input and intermediate data in a new class BlockFetcher. I hope in general it makes the code easier to maintain. Another motivation to do it is to clearly divide the logic before file reading and after file reading. The refactor will help us evaluate how can we make I/O async in the future. Closes https://github.com/facebook/rocksdb/pull/3244 Differential Revision: D6520983 Pulled By: siying fbshipit-source-id: 338d90bc0338472d46be7a7682028dc9114b12e9	2017-12-11 15:27:32 -08:00