Fix GitHub issue #3716 : gcc-8 warnings

Summary: Fix the following gcc-8 warnings: - conflicting C language linkage declaration [-Werror] - writing to an object with no trivial copy-assignment [-Werror=class-memaccess] - array subscript -1 is below array bounds [-Werror=array-bounds] Solves https://github.com/facebook/rocksdb/issues/3716 Closes https://github.com/facebook/rocksdb/pull/3736 Differential Revision: D7684161 Pulled By: yiwu-arbug fbshipit-source-id: 47c0423d26b74add251f1d3595211eee1e41e54a
Making platform 007 (gcc 7) default in build_detect_platform.sh (#5947 )
2026-11-06 02:52:35 -05:00 · 2019-10-30 13:30:56 -07:00 · 2019-10-30 13:30:56 -07:00 · 2019-10-30 13:30:56 -07:00 · 2018-03-16 14:22:42 -07:00 · 2018-03-16 14:22:05 -07:00
77 changed files with 1099 additions and 302 deletions
@@ -1,5 +1,24 @@
 # Rocksdb Change Log
-## Unreleased
+## 5.10.5 (03/16/2018)
+### Bug fixes
+* Fix WAL corruption caused by race condition between user write thread and backup/checkpoint thread.
+
+## 5.10.4 (02/22/2018)
+### New Features
+* Follow rsync-style naming convention for BackupEngine tempfiles. This enables some optimizations when run on GlusterFS.
+* Fix regression of Java build break on Windows.
+
+## 5.10.3 (02/21/2018)
+### Bug fixes
+* Fix build break regression using gcc-7
+* Direct I/O writable file should do fsync in Close()
+
+### New Features
+* Add rocksdb.iterator.internal-key property
+
+## 5.10.1 (01/18/2018)
+### Bug Fixes
+* Fix DB::Flush() keep waiting after flush finish under certain condition.

 ## 5.10.0 (12/11/2017)
 ### Public API Change
@@ -107,6 +107,41 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
            * run `brew tap homebrew/versions; brew install gcc48 --use-llvm` to install gcc 4.8 (or higher).
    * run `brew install rocksdb`

+* **FreeBSD** (11.01):
+
+    * You can either install RocksDB from the Ports system using `cd /usr/ports/databases/rocksdb && make install`, or you can follow the details below to install dependencies and compile from source code:
+
+    * Install the dependencies for RocksDB:
+
+        export BATCH=YES
+        cd /usr/ports/devel/gmake && make install
+        cd /usr/ports/devel/gflags && make install
+
+        cd /usr/ports/archivers/snappy && make install
+        cd /usr/ports/archivers/bzip2 && make install
+        cd /usr/ports/archivers/liblz4 && make install
+        cd /usr/ports/archivesrs/zstd && make install
+
+        cd /usr/ports/devel/git && make install
+
+
+    * Install the dependencies for RocksJava (optional):
+
+        export BATCH=yes
+        cd /usr/ports/java/openjdk7 && make install
+
+    * Build RocksDB from source:
+        cd ~
+        git clone https://github.com/facebook/rocksdb.git
+        cd rocksdb
+        gmake static_lib
+
+    * Build RocksJava from source (optional):
+        cd rocksdb
+        export JAVA_HOME=/usr/local/openjdk7
+        gmake rocksdbjava
+
+
 * **iOS**:
  * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.

@@ -344,6 +344,8 @@ ifeq ($(HAVE_POWER8),1)
 LIB_CC_OBJECTS = $(LIB_SOURCES:.cc=.o)
 LIBOBJECTS += $(LIB_SOURCES_C:.c=.o)
 LIBOBJECTS += $(LIB_SOURCES_ASM:.S=.o)
+else
+LIB_CC_OBJECTS = $(LIB_SOURCES:.cc=.o)
 endif

 LIBOBJECTS += $(TOOL_LIB_SOURCES:.cc=.o)
@@ -1571,7 +1573,7 @@ else
 endif
 endif
 ifeq ($(PLATFORM), OS_FREEBSD)
-	JAVA_INCLUDE += -I$(JAVA_HOME)/include/freebsd
+	JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/freebsd
 	ROCKSDBJNILIB = librocksdbjni-freebsd$(ARCH).so
 	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-freebsd$(ARCH).jar
 endif
@@ -1597,7 +1599,7 @@ libz.a:
 		exit 1; \
 	fi
 	tar xvzf zlib-$(ZLIB_VER).tar.gz
-	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && make
+	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && $(MAKE)
 	cp zlib-$(ZLIB_VER)/libz.a .

 libbz2.a:
@@ -1609,7 +1611,7 @@ libbz2.a:
 		exit 1; \
 	fi
 	tar xvzf bzip2-$(BZIP2_VER).tar.gz
-	cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
+	cd bzip2-$(BZIP2_VER) && $(MAKE) CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
 	cp bzip2-$(BZIP2_VER)/libbz2.a .

 libsnappy.a:
@@ -1622,7 +1624,7 @@ libsnappy.a:
 	fi
 	tar xvzf snappy-$(SNAPPY_VER).tar.gz
 	cd snappy-$(SNAPPY_VER) && CFLAGS='${EXTRA_CFLAGS}' CXXFLAGS='${EXTRA_CXXFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --with-pic --enable-static --disable-shared
-	cd snappy-$(SNAPPY_VER) && make ${SNAPPY_MAKE_TARGET}
+	cd snappy-$(SNAPPY_VER) && $(MAKE) ${SNAPPY_MAKE_TARGET}
 	cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a .

 liblz4.a:
@@ -1635,7 +1637,7 @@ liblz4.a:
 		exit 1; \
 	fi
 	tar xvzf lz4-$(LZ4_VER).tar.gz
-	cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
+	cd lz4-$(LZ4_VER)/lib && $(MAKE) CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
 	cp lz4-$(LZ4_VER)/lib/liblz4.a .

 libzstd.a:
@@ -1648,29 +1650,45 @@ libzstd.a:
 		exit 1; \
 	fi
 	tar xvzf zstd-$(ZSTD_VER).tar.gz
-	cd zstd-$(ZSTD_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
+	cd zstd-$(ZSTD_VER)/lib && DESTDIR=. PREFIX= $(MAKE) CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' install
 	cp zstd-$(ZSTD_VER)/lib/libzstd.a .

 # A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries
-java_static_libobjects = $(patsubst %,jls/%,$(LIBOBJECTS))
+java_static_libobjects = $(patsubst %,jls/%,$(LIB_CC_OBJECTS))
 CLEAN_FILES += jls
+java_static_all_libobjects = $(java_static_libobjects)

 ifneq ($(ROCKSDB_JAVA_NO_COMPRESSION), 1)
 JAVA_COMPRESSIONS = libz.a libbz2.a libsnappy.a liblz4.a libzstd.a
 endif

 JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4 -DZSTD
-JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib
+JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib/include
+
+ifeq ($(HAVE_POWER8),1)
+JAVA_STATIC_C_LIBOBJECTS = $(patsubst %.c.o,jls/%.c.o,$(LIB_SOURCES_C:.c=.o))
+JAVA_STATIC_ASM_LIBOBJECTS = $(patsubst %.S.o,jls/%.S.o,$(LIB_SOURCES_ASM:.S=.o))
+
+java_static_ppc_libobjects = $(JAVA_STATIC_C_LIBOBJECTS) $(JAVA_STATIC_ASM_LIBOBJECTS)
+
+jls/util/crc32c_ppc.o: util/crc32c_ppc.c
+	$(AM_V_CC)$(CC) $(CFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -c $< -o $@
+
+jls/util/crc32c_ppc_asm.o: util/crc32c_ppc_asm.S
+	$(AM_V_CC)$(CC) $(CFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -c $< -o $@
+
+java_static_all_libobjects += $(java_static_ppc_libobjects)
+endif

 $(java_static_libobjects): jls/%.o: %.cc $(JAVA_COMPRESSIONS)
 	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -fPIC -c $< -o $@ $(COVERAGEFLAGS)

-rocksdbjavastatic: $(java_static_libobjects)
+rocksdbjavastatic: $(java_static_all_libobjects)
 	cd java;$(MAKE) javalib;
 	rm -f ./java/target/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
-	  $(java_static_libobjects) $(COVERAGEFLAGS) \
+	  $(java_static_all_libobjects) $(COVERAGEFLAGS) \
 	  $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS)
 	cd java/target;strip $(STRIPFLAGS) $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
@@ -1731,7 +1749,7 @@ JAVA_C_LIBOBJECTS = $(patsubst %.c.o,jl/%.c.o,$(JAVA_C_OBJECTS))
 JAVA_ASM_LIBOBJECTS = $(patsubst %.S.o,jl/%.S.o,$(JAVA_ASM_OBJECTS))
 endif

-java_libobjects = $(patsubst %,jl/%,$(LIBOBJECTS))
+java_libobjects = $(patsubst %,jl/%,$(LIB_CC_OBJECTS))
 CLEAN_FILES += jl
 java_all_libobjects = $(java_libobjects)

@@ -36,7 +36,7 @@ def parse_src_mk(repo_path):
 # get all .cc / .c files
 def get_cc_files(repo_path):
    cc_files = []
-    for root, dirnames, filenames in os.walk(repo_path):
+    for root, dirnames, filenames in os.walk(repo_path):  # noqa: B007 T25377293 Grandfathered in
        root = root[(len(repo_path) + 1):]
        if "java" in root:
            # Skip java
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
 # Create a tmp directory for the test to use
 TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
+# shellcheck disable=SC2068
 TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"
@@ -51,11 +51,13 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
    FBCODE_BUILD="true"
    # If we're compiling with TSAN we need pic build
    PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      source "$PWD/build_tools/fbcode_config.sh"
-    else
+    if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
      # we need this to build with MySQL. Don't use for other purposes.
      source "$PWD/build_tools/fbcode_config4.8.1.sh"
+    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
+      source "$PWD/build_tools/fbcode_config.sh"
+    else
+      source "$PWD/build_tools/fbcode_config_platform007.sh"
    fi
 fi

@@ -141,6 +143,7 @@ case "$TARGET_OS" in
        ;;
    FreeBSD)
        PLATFORM=OS_FREEBSD
+        CXX=clang++
        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
        # PORT_FILES=port/freebsd/freebsd_specific.cc
@@ -481,6 +484,9 @@ if test -z "$PORTABLE"; then
    COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
    COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
+  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^arm`"; then
+    # TODO: Handle this with approprite options.
+    COMMON_FLAGS="$COMMON_FLAGS"
  elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
    COMMON_FLAGS="$COMMON_FLAGS -march=native "
  elif test "$USE_SSE"; then
@@ -13,10 +13,12 @@ error=0

 function log {
  DATE=`date +%Y-%m-%d:%H:%M:%S`
+  # shellcheck disable=SC2068
  echo $DATE $@
 }

 function log_err {
+  # shellcheck disable=SC2145
  log "ERROR: $@ Error code: $error."
 }

@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 GCC_BASE=/mnt/gvfs/third-party2/gcc/8219ec1bcedf8ad9da05e121e193364de2cc4f61/5.x/centos6-native/c447969
 CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/64d8d58e3d84f8bde7a029763d4f5baf39d0d5b9/stable/centos6-native/6aaf4de
 LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ba9be983c81de7299b59fe71950c664a84dcb5f8/5.x/gcc-5-glibc-2.23/339d858
@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
 CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
 LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
@@ -0,0 +1,18 @@
+GCC_BASE=/mnt/gvfs/third-party2/gcc/6e8e715624fd15256a7970073387793dfcf79b46/7.x/centos7-native/b2ef2b6
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/ef37e1faa1c29782abfac1ae65a291b9b7966f6d/stable/centos7-native/c9f9104
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/c67031f0f739ac61575a061518d6ef5038f99f90/7.x/platform007/5620abc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/60d6f124a78798b73944f5ba87c2306ae3460153/2.26/platform007/f259413
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/22c2d65676fb7c23cfa797c4f6937f38b026f3cf/1.2.8/platform007/ca4da3d
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/907b498203d297947f3bb70b9466f47e100f1873/r131/platform007/ca4da3d
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/3ee276cbacfad3074e3f07bf826ac47f06970f4e/1.3.5/platform007/15a3614
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/9c910d36d6235cc40e8ff559358f1833452300ca/master/platform007/5b0f53e
+NUMA_BASE=/mnt/gvfs/third-party2/numa/9cbf2460284c669ed19c3ccb200a71f7dd7e53c7/2.0.11/platform007/ca4da3d
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/bf3d7497fe4e6d007354f0adffa16ce3003f8338/1.3/platform007/6f3e0a9
+TBB_BASE=/mnt/gvfs/third-party2/tbb/ff4e0b093534704d8abab678a4fd7f5ea7b094c7/2018_U5/platform007/ca4da3d
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/b5c4a61a5c483ba24722005ae07895971a2ac707/fb/platform007/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/92ff90349e2f43ea0a8246d8b1cf17b6869013e3/2.29.1/centos7-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/f3f697a28122e6bcd513273dd9c1ff23852fc59f/3.13.0/platform007/ca4da3d
+LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
@@ -0,0 +1,157 @@
+#!/bin/sh
+#
+# Set environment variables so that we can compile rocksdb using
+# fbcode settings.  It uses the latest g++ and clang compilers and also
+# uses jemalloc
+# Environment variables that change the behavior of this script:
+# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
+
+
+BASEDIR=`dirname $BASH_SOURCE`
+source "$BASEDIR/dependencies_platform007.sh"
+
+CFLAGS=""
+
+# libgcc
+LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
+LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
+
+# glibc
+GLIBC_INCLUDE="$GLIBC_BASE/include"
+GLIBC_LIBS=" -L $GLIBC_BASE/lib"
+
+# snappy
+SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
+if test -z $PIC_BUILD; then
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
+else
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
+fi
+CFLAGS+=" -DSNAPPY"
+
+if test -z $PIC_BUILD; then
+  # location of zlib headers and libraries
+  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
+  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
+  CFLAGS+=" -DZLIB"
+
+  # location of bzip headers and libraries
+  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
+  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
+  CFLAGS+=" -DBZIP2"
+
+  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
+  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
+  CFLAGS+=" -DLZ4"
+fi
+
+ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
+if test -z $PIC_BUILD; then
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+else
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
+fi
+CFLAGS+=" -DZSTD"
+
+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
+if test -z $PIC_BUILD; then
+  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
+else
+  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
+fi
+CFLAGS+=" -DGFLAGS=gflags"
+
+# location of jemalloc
+JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
+JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
+
+if test -z $PIC_BUILD; then
+  # location of numa
+  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
+  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
+  CFLAGS+=" -DNUMA"
+
+  # location of libunwind
+  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
+fi
+
+# location of TBB
+TBB_INCLUDE=" -isystem $TBB_BASE/include/"
+if test -z $PIC_BUILD; then
+  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
+else
+  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
+fi
+CFLAGS+=" -DTBB"
+
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
+export PORTABLE=1
+
+BINUTILS="$BINUTILS_BASE/bin"
+AR="$BINUTILS/ar"
+
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
+
+STDLIBS="-L $GCC_BASE/lib64"
+
+CLANG_BIN="$CLANG_BASE/bin"
+CLANG_LIB="$CLANG_BASE/lib"
+CLANG_SRC="$CLANG_BASE/../../src"
+
+CLANG_ANALYZER="$CLANG_BIN/clang++"
+CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
+
+if [ -z "$USE_CLANG" ]; then
+  # gcc
+  CC="$GCC_BASE/bin/gcc"
+  CXX="$GCC_BASE/bin/g++"
+
+  CFLAGS+=" -B$BINUTILS/gold"
+  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  JEMALLOC=1
+else
+  # clang
+  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
+  CC="$CLANG_BIN/clang"
+  CXX="$CLANG_BIN/clang++"
+
+  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
+
+  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $CLANG_INCLUDE"
+  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
+  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
+  CFLAGS+=" -Wno-expansion-to-defined "
+  CXXFLAGS="-nostdinc++"
+fi
+
+CFLAGS+=" $DEPS_INCLUDE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
+CXXFLAGS+=" $CFLAGS"
+
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
+EXEC_LDFLAGS+=" -B$BINUTILS/gold"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
+EXEC_LDFLAGS+=" $LIBUNWIND"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
+# required by libtbb
+EXEC_LDFLAGS+=" -ldl"
+
+PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
+
+VALGRIND_VER="$VALGRIND_BASE/bin/"
+
+# lua not supported because it's on track for deprecation, I think
+LUA_PATH=
+LUA_LIB=
+
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
@@ -1,3 +1,4 @@
+# shellcheck disable=SC1113
 #/usr/bin/env bash

 set -e
@@ -28,12 +29,14 @@ function package() {
    if dpkg --get-selections | grep --quiet $1; then
      log "$1 is already installed. skipping."
    else
+      # shellcheck disable=SC2068
      apt-get install $@ -y
    fi
  elif [[ $OS = "centos" ]]; then
    if rpm -qa | grep --quiet $1; then
      log "$1 is already installed. skipping."
    else
+      # shellcheck disable=SC2068
      yum install $@ -y
    fi
  fi
@@ -52,6 +55,7 @@ function gem_install() {
  if gem list | grep --quiet $1; then
    log "$1 is already installed. skipping."
  else
+    # shellcheck disable=SC2068
    gem install $@
  fi
 }
@@ -125,4 +129,5 @@ function main() {
    include $LIB_DIR
 }

+# shellcheck disable=SC2068
 main $@
@@ -85,8 +85,9 @@ NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
 GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
 ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
-LITE="OPT=\"-DROCKSDB_LITE -g\""
-TSAN="COMPILE_WITH_TSAN=1"
+# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090.
+# using platform007 gives us gcc-8 or higher which has that bug fixed.
+TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
@@ -53,6 +53,45 @@ function get_lib_base()
  log_variable $__res_var
 }

+###########################################################
+#                platform007 dependencies                 #
+###########################################################
+
+OUTPUT="$BASEDIR/dependencies_platform007.sh"
+
+rm -f "$OUTPUT"
+touch "$OUTPUT"
+
+echo "Writing dependencies to $OUTPUT"
+
+# Compilers locations
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/7.x/centos7-native/*/`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
+
+log_variable GCC_BASE
+log_variable CLANG_BASE
+
+# Libraries locations
+get_lib_base libgcc     7.x     platform007
+get_lib_base glibc      2.26    platform007
+get_lib_base snappy     LATEST  platform007
+get_lib_base zlib       LATEST  platform007
+get_lib_base bzip2      LATEST  platform007
+get_lib_base lz4        LATEST  platform007
+get_lib_base zstd       LATEST  platform007
+get_lib_base gflags     LATEST  platform007
+get_lib_base jemalloc   LATEST  platform007
+get_lib_base numa       LATEST  platform007
+get_lib_base libunwind  LATEST  platform007
+get_lib_base tbb        LATEST  platform007
+
+get_lib_base kernel-headers fb platform007
+get_lib_base binutils   LATEST centos7-native
+get_lib_base valgrind   LATEST platform007
+get_lib_base lua        5.3.4  platform007
+
+git diff $OUTPUT
+
 ###########################################################
 #                   5.x dependencies                      #
 ###########################################################
@@ -72,7 +72,7 @@ def display_file_coverage(per_file_coverage, total_coverage):
    header_template = \
        "%" + str(max_file_name_length) + "s\t%s\t%s"
    separator = "-" * (max_file_name_length + 10 + 20)
-    print header_template % ("Filename", "Coverage", "Lines")
+    print header_template % ("Filename", "Coverage", "Lines")  # noqa: E999 T25377293 Grandfathered in
    print separator

    # -- Print body
@@ -1388,23 +1388,24 @@ void rocksdb_writebatch_put_log_data(
  b->rep.PutLogData(Slice(blob, len));
 }

+class H : public WriteBatch::Handler {
+ public:
+  void* state_;
+  void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+  void (*deleted_)(void*, const char* k, size_t klen);
+  virtual void Put(const Slice& key, const Slice& value) override {
+    (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+  }
+  virtual void Delete(const Slice& key) override {
+    (*deleted_)(state_, key.data(), key.size());
+  }
+};
+
 void rocksdb_writebatch_iterate(
    rocksdb_writebatch_t* b,
    void* state,
    void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
    void (*deleted)(void*, const char* k, size_t klen)) {
-  class H : public WriteBatch::Handler {
-   public:
-    void* state_;
-    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-    void (*deleted_)(void*, const char* k, size_t klen);
-    virtual void Put(const Slice& key, const Slice& value) override {
-      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-    }
-    virtual void Delete(const Slice& key) override {
-      (*deleted_)(state_, key.data(), key.size());
-    }
-  };
  H handler;
  handler.state_ = state;
  handler.put_ = put;
@@ -1649,18 +1650,6 @@ void rocksdb_writebatch_wi_iterate(
    void* state,
    void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
    void (*deleted)(void*, const char* k, size_t klen)) {
-  class H : public WriteBatch::Handler {
-   public:
-    void* state_;
-    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-    void (*deleted_)(void*, const char* k, size_t klen);
-    virtual void Put(const Slice& key, const Slice& value) override {
-      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-    }
-    virtual void Delete(const Slice& key) override {
-      (*deleted_)(state_, key.data(), key.size());
-    }
-  };
  H handler;
  handler.state_ = state;
  handler.put_ = put;
@@ -3104,20 +3093,21 @@ void rocksdb_slicetransform_destroy(rocksdb_slicetransform_t* st) {
  delete st;
 }

+struct Wrapper : public rocksdb_slicetransform_t {
+  const SliceTransform* rep_;
+  ~Wrapper() { delete rep_; }
+  const char* Name() const override { return rep_->Name(); }
+  Slice Transform(const Slice& src) const override {
+    return rep_->Transform(src);
+  }
+  bool InDomain(const Slice& src) const override {
+    return rep_->InDomain(src);
+  }
+  bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+  static void DoNothing(void*) { }
+};
+
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t prefixLen) {
-  struct Wrapper : public rocksdb_slicetransform_t {
-    const SliceTransform* rep_;
-    ~Wrapper() { delete rep_; }
-    const char* Name() const override { return rep_->Name(); }
-    Slice Transform(const Slice& src) const override {
-      return rep_->Transform(src);
-    }
-    bool InDomain(const Slice& src) const override {
-      return rep_->InDomain(src);
-    }
-    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
-    static void DoNothing(void*) { }
-  };
  Wrapper* wrapper = new Wrapper;
  wrapper->rep_ = rocksdb::NewFixedPrefixTransform(prefixLen);
  wrapper->state_ = nullptr;
@@ -3126,19 +3116,6 @@ rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t pref
 }

 rocksdb_slicetransform_t* rocksdb_slicetransform_create_noop() {
-  struct Wrapper : public rocksdb_slicetransform_t {
-    const SliceTransform* rep_;
-    ~Wrapper() { delete rep_; }
-    const char* Name() const override { return rep_->Name(); }
-    Slice Transform(const Slice& src) const override {
-      return rep_->Transform(src);
-    }
-    bool InDomain(const Slice& src) const override {
-      return rep_->InDomain(src);
-    }
-    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
-    static void DoNothing(void*) { }
-  };
  Wrapper* wrapper = new Wrapper;
  wrapper->rep_ = rocksdb::NewNoopTransform();
  wrapper->state_ = nullptr;
@@ -344,12 +344,13 @@ void SuperVersionUnrefHandle(void* ptr) {
  // When latter happens, we are in ~ColumnFamilyData(), no get should happen as
  // well.
  SuperVersion* sv = static_cast<SuperVersion*>(ptr);
-  if (sv->Unref()) {
-    sv->db_mutex->Lock();
-    sv->Cleanup();
-    sv->db_mutex->Unlock();
-    delete sv;
-  }
+  bool was_last_ref __attribute__((__unused__));
+  was_last_ref = sv->Unref();
+  // Thread-local SuperVersions can't outlive ColumnFamilyData::super_version_.
+  // This is important because we can't do SuperVersion cleanup here.
+  // That would require locking DB mutex, which would deadlock because
+  // SuperVersionUnrefHandle is called with locked ThreadLocalPtr mutex.
+  assert(!was_last_ref);
 }
 }  // anonymous namespace

@@ -385,7 +386,8 @@ ColumnFamilyData::ColumnFamilyData(
      pending_flush_(false),
      pending_compaction_(false),
      prev_compaction_needed_bytes_(0),
-      allow_2pc_(db_options.allow_2pc) {
+      allow_2pc_(db_options.allow_2pc),
+      last_memtable_id_(0) {
  Ref();

  // Convert user defined table properties collector factories to internal ones.
@@ -965,6 +967,12 @@ void ColumnFamilyData::InstallSuperVersion(
      RecalculateWriteStallConditions(mutable_cf_options);

  if (old_superversion != nullptr) {
+    // Reset SuperVersions cached in thread local storage.
+    // This should be done before old_superversion->Unref(). That's to ensure
+    // that local_sv_ never holds the last reference to SuperVersion, since
+    // it has no means to safely do SuperVersion cleanup.
+    ResetThreadLocalSuperVersions();
+
    if (old_superversion->mutable_cf_options.write_buffer_size !=
        mutable_cf_options.write_buffer_size) {
      mem_->UpdateWriteBufferSize(mutable_cf_options.write_buffer_size);
@@ -980,9 +988,6 @@ void ColumnFamilyData::InstallSuperVersion(
      sv_context->superversions_to_free.push_back(old_superversion);
    }
  }
-
-  // Reset SuperVersions cached in thread local storage
-  ResetThreadLocalSuperVersions();
 }

 void ColumnFamilyData::ResetThreadLocalSuperVersions() {
@@ -994,10 +999,12 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
      continue;
    }
    auto sv = static_cast<SuperVersion*>(ptr);
-    if (sv->Unref()) {
-      sv->Cleanup();
-      delete sv;
-    }
+    bool was_last_ref __attribute__((__unused__));
+    was_last_ref = sv->Unref();
+    // sv couldn't have been the last reference because
+    // ResetThreadLocalSuperVersions() is called before
+    // unref'ing super_version_.
+    assert(!was_last_ref);
  }
 }

@@ -239,7 +239,11 @@ class ColumnFamilyData {
  void SetCurrent(Version* _current);
  uint64_t GetNumLiveVersions() const;  // REQUIRE: DB mutex held
  uint64_t GetTotalSstFilesSize() const;  // REQUIRE: DB mutex held
-  void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }
+  void SetMemtable(MemTable* new_mem) {
+    uint64_t memtable_id = last_memtable_id_.fetch_add(1) + 1;
+    new_mem->SetID(memtable_id);
+    mem_ = new_mem;
+  }

  // calculate the oldest log needed for the durability of this column family
  uint64_t OldestLogToKeep();
@@ -419,6 +423,9 @@ class ColumnFamilyData {

  // if the database was opened with 2pc enabled
  bool allow_2pc_;
+
+  // Memtable id to track flush.
+  std::atomic<uint64_t> last_memtable_id_;
 };

 // ColumnFamilySet has interesting thread-safety requirements
@@ -625,8 +625,8 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
      "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
      "files in(%d, %d) out(%d) "
      "MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
-      "write-amplify(%.1f) %s, records in: %d, records dropped: %d "
-      "output_compression: %s\n",
+      "write-amplify(%.1f) %s, records in: %" PRIu64
+      ", records dropped: %" PRIu64 " output_compression: %s\n",
      cfd->GetName().c_str(), vstorage->LevelSummary(&tmp), bytes_read_per_sec,
      bytes_written_per_sec, compact_->compaction->output_level(),
      stats.num_input_files_in_non_output_levels,
@@ -126,6 +126,41 @@ TEST_F(DBFlushTest, FlushInLowPriThreadPool) {
  ASSERT_EQ(1, num_compactions);
 }

+TEST_F(DBFlushTest, ManualFlushWithMinWriteBufferNumberToMerge) {
+  Options options = CurrentOptions();
+  options.write_buffer_size = 100;
+  options.max_write_buffer_number = 4;
+  options.min_write_buffer_number_to_merge = 3;
+  Reopen(options);
+
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::BGWorkFlush",
+        "DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:1"},
+       {"DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:2",
+        "DBImpl::FlushMemTableToOutputFile:BeforeInstallSV"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(Put("key1", "value1"));
+
+  port::Thread t([&]() {
+    // The call wait for flush to finish, i.e. with flush_options.wait = true.
+    ASSERT_OK(Flush());
+  });
+
+  // Wait for flush start.
+  TEST_SYNC_POINT("DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:1");
+  // Insert a second memtable before the manual flush finish.
+  // At the end of the manual flush job, it will check if further flush
+  // is needed, but it will not trigger flush of the second memtable because
+  // min_write_buffer_number_to_merge is not reached.
+  ASSERT_OK(Put("key2", "value2"));
+  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
+  TEST_SYNC_POINT("DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:2");
+
+  // Manual flush should return, without waiting for flush indefinitely.
+  t.join();
+}
+
 TEST_P(DBFlushDirectIOTest, DirectIO) {
  Options options;
  options.create_if_missing = true;
@@ -1682,6 +1682,12 @@ void DBImpl::ReleaseSnapshot(const Snapshot* s) {
  delete casted_s;
 }

+bool DBImpl::HasActiveSnapshotInRange(SequenceNumber lower_bound,
+                                      SequenceNumber upper_bound) {
+  InstrumentedMutexLock l(&mutex_);
+  return snapshots_.HasSnapshotInRange(lower_bound, upper_bound);
+}
+
 #ifndef ROCKSDB_LITE
 Status DBImpl::GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
                                        TablePropertiesCollection* props) {
@@ -229,6 +229,10 @@ class DBImpl : public DB {

  virtual bool SetPreserveDeletesSequenceNumber(SequenceNumber seqnum) override;

+  // Whether there is an active snapshot in range [lower_bound, upper_bound).
+  bool HasActiveSnapshotInRange(SequenceNumber lower_bound,
+                                SequenceNumber upper_bound);
+
 #ifndef ROCKSDB_LITE
  using DB::ResetStats;
  virtual Status ResetStats() override;
@@ -806,8 +810,12 @@ class DBImpl : public DB {
  Status FlushMemTable(ColumnFamilyData* cfd, const FlushOptions& options,
                       bool writes_stopped = false);

-  // Wait for memtable flushed
-  Status WaitForFlushMemTable(ColumnFamilyData* cfd);
+  // Wait for memtable flushed.
+  // If flush_memtable_id is non-null, wait until the memtable with the ID
+  // gets flush. Otherwise, wait until the column family don't have any
+  // memtable pending flush.
+  Status WaitForFlushMemTable(ColumnFamilyData* cfd,
+                              const uint64_t* flush_memtable_id = nullptr);

  // REQUIRES: mutex locked
  Status SwitchWAL(WriteContext* write_context);
@@ -134,6 +134,7 @@ Status DBImpl::FlushMemTableToOutputFile(
  }

  if (s.ok()) {
+    TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:BeforeInstallSV");
    InstallSuperVersionAndScheduleWork(cfd, &job_context->superversion_context,
                                       mutable_cf_options);
    if (made_progress) {
@@ -809,7 +810,13 @@ int DBImpl::Level0StopWriteTrigger(ColumnFamilyHandle* column_family) {
 Status DBImpl::Flush(const FlushOptions& flush_options,
                     ColumnFamilyHandle* column_family) {
  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
-  return FlushMemTable(cfh->cfd(), flush_options);
+  ROCKS_LOG_INFO(immutable_db_options_.info_log, "[%s] Manual flush start.",
+                 cfh->GetName().c_str());
+  Status s = FlushMemTable(cfh->cfd(), flush_options);
+  ROCKS_LOG_INFO(immutable_db_options_.info_log,
+                 "[%s] Manual flush finished, status: %s\n",
+                 cfh->GetName().c_str(), s.ToString().c_str());
+  return s;
 }

 Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
@@ -944,6 +951,7 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
                             const FlushOptions& flush_options,
                             bool writes_stopped) {
  Status s;
+  uint64_t flush_memtable_id = 0;
  {
    WriteContext context;
    InstrumentedMutexLock guard_lock(&mutex_);
@@ -961,6 +969,7 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,

    // SwitchMemtable() will release and reacquire mutex during execution
    s = SwitchMemtable(cfd, &context);
+    flush_memtable_id = cfd->imm()->GetLatestMemTableID();

    if (!writes_stopped) {
      write_thread_.ExitUnbatched(&w);
@@ -975,16 +984,19 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,

  if (s.ok() && flush_options.wait) {
    // Wait until the compaction completes
-    s = WaitForFlushMemTable(cfd);
+    s = WaitForFlushMemTable(cfd, &flush_memtable_id);
  }
  return s;
 }

-Status DBImpl::WaitForFlushMemTable(ColumnFamilyData* cfd) {
+Status DBImpl::WaitForFlushMemTable(ColumnFamilyData* cfd,
+                                    const uint64_t* flush_memtable_id) {
  Status s;
  // Wait until the compaction completes
  InstrumentedMutexLock l(&mutex_);
-  while (cfd->imm()->NumNotFlushed() > 0 && bg_error_.ok()) {
+  while (cfd->imm()->NumNotFlushed() > 0 && bg_error_.ok() &&
+         (flush_memtable_id == nullptr ||
+          cfd->imm()->GetEarliestMemTableID() <= *flush_memtable_id)) {
    if (shutting_down_.load(std::memory_order_acquire)) {
      return Status::ShutdownInProgress();
    }
@@ -217,6 +217,9 @@ class DBIter final: public Iterator {
        *prop = "Iterator is not valid.";
      }
      return Status::OK();
+    } else if (prop_name == "rocksdb.iterator.internal-key") {
+      *prop = saved_key_.GetUserKey().ToString();
+      return Status::OK();
    }
    return Status::InvalidArgument("Undentified property.");
  }
@@ -55,6 +55,7 @@ TEST_F(DBIteratorTest, IteratorProperty) {
  Options options = CurrentOptions();
  CreateAndReopenWithCF({"pikachu"}, options);
  Put(1, "1", "2");
+  Delete(1, "2");
  ReadOptions ropt;
  ropt.pin_data = false;
  {
@@ -64,9 +65,15 @@ TEST_F(DBIteratorTest, IteratorProperty) {
    ASSERT_NOK(iter->GetProperty("non_existing.value", &prop_value));
    ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
    ASSERT_EQ("0", prop_value);
+    ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value));
+    ASSERT_EQ("1", prop_value);
    iter->Next();
    ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
    ASSERT_EQ("Iterator is not valid.", prop_value);
+
+    // Get internal key at which the iteration stopped (tombstone in this case).
+    ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value));
+    ASSERT_EQ("2", prop_value);
  }
  Close();
 }
@@ -2157,6 +2164,48 @@ TEST_F(DBIteratorTest, SkipStatistics) {
  ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
 }

+TEST_F(DBIteratorTest, SeekAfterHittingManyInternalKeys) {
+  Options options = CurrentOptions();
+  DestroyAndReopen(options);
+  ReadOptions ropts;
+  ropts.max_skippable_internal_keys = 2;
+
+  Put("1", "val_1");
+  // Add more tombstones than max_skippable_internal_keys so that Next() fails.
+  Delete("2");
+  Delete("3");
+  Delete("4");
+  Delete("5");
+  Put("6", "val_6");
+
+  unique_ptr<Iterator> iter(db_->NewIterator(ropts));
+  iter->SeekToFirst();
+
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key().ToString(), "1");
+  ASSERT_EQ(iter->value().ToString(), "val_1");
+
+  // This should fail as incomplete due to too many non-visible internal keys on
+  // the way to the next valid user key.
+  iter->Next();
+  ASSERT_TRUE(!iter->Valid());
+  ASSERT_TRUE(iter->status().IsIncomplete());
+
+  // Get the internal key at which Next() failed.
+  std::string prop_value;
+  ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value));
+  ASSERT_EQ("4", prop_value);
+
+  // Create a new iterator to seek to the internal key.
+  unique_ptr<Iterator> iter2(db_->NewIterator(ropts));
+  iter2->Seek(prop_value);
+  ASSERT_TRUE(iter2->Valid());
+  ASSERT_OK(iter2->status());
+
+  ASSERT_EQ(iter2->key().ToString(), "6");
+  ASSERT_EQ(iter2->value().ToString(), "val_6");
+}
+
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -5651,6 +5651,50 @@ TEST_F(DBTest, PauseBackgroundWorkTest) {
  // now it's done
  ASSERT_TRUE(done.load());
 }
+
+// Keep spawning short-living threads that create an iterator and quit.
+// Meanwhile in another thread keep flushing memtables.
+// This used to cause a deadlock.
+TEST_F(DBTest, ThreadLocalPtrDeadlock) {
+  std::atomic<int> flushes_done{0};
+  std::atomic<int> threads_destroyed{0};
+  auto done = [&] {
+    return flushes_done.load() > 10;
+  };
+
+  std::thread flushing_thread([&] {
+    for (int i = 0; !done(); ++i) {
+      ASSERT_OK(db_->Put(WriteOptions(), Slice("hi"),
+                         Slice(std::to_string(i).c_str())));
+      ASSERT_OK(db_->Flush(FlushOptions()));
+      int cnt = ++flushes_done;
+      fprintf(stderr, "Flushed %d times\n", cnt);
+    }
+  });
+
+  std::vector<std::thread> thread_spawning_threads(10);
+  for (auto& t: thread_spawning_threads) {
+    t = std::thread([&] {
+      while (!done()) {
+        {
+          std::thread tmp_thread([&] {
+            auto it = db_->NewIterator(ReadOptions());
+            delete it;
+          });
+          tmp_thread.join();
+        }
+        ++threads_destroyed;
+      }
+    });
+  }
+
+  for (auto& t: thread_spawning_threads) {
+    t.join();
+  }
+  flushing_thread.join();
+  fprintf(stderr, "Done. Flushed %d times, destroyed %d threads\n",
+          flushes_done.load(), threads_destroyed.load());
+}
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -39,6 +39,52 @@ TEST_P(DBWriteTest, SyncAndDisableWAL) {
  ASSERT_TRUE(dbfull()->Write(write_options, &batch).IsInvalidArgument());
 }

+// Sequence number should be return through input write batch.
+TEST_P(DBWriteTest, ReturnSeuqneceNumber) {
+  Random rnd(4422);
+  Open();
+  for (int i = 0; i < 100; i++) {
+    WriteBatch batch;
+    batch.Put("key" + ToString(i), test::RandomHumanReadableString(&rnd, 10));
+    ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
+    ASSERT_EQ(dbfull()->GetLatestSequenceNumber(),
+              WriteBatchInternal::Sequence(&batch));
+  }
+}
+
+TEST_P(DBWriteTest, ReturnSeuqneceNumberMultiThreaded) {
+  constexpr size_t kThreads = 16;
+  constexpr size_t kNumKeys = 1000;
+  Open();
+  ASSERT_EQ(0, dbfull()->GetLatestSequenceNumber());
+  // Check each sequence is used once and only once.
+  std::vector<std::atomic_flag> flags(kNumKeys * kThreads + 1);
+  for (size_t i = 0; i < flags.size(); i++) {
+    flags[i].clear();
+  }
+  auto writer = [&](size_t id) {
+    Random rnd(4422 + static_cast<uint32_t>(id));
+    for (size_t k = 0; k < kNumKeys; k++) {
+      WriteBatch batch;
+      batch.Put("key" + ToString(id) + "-" + ToString(k),
+                test::RandomHumanReadableString(&rnd, 10));
+      ASSERT_OK(dbfull()->Write(WriteOptions(), &batch));
+      SequenceNumber sequence = WriteBatchInternal::Sequence(&batch);
+      ASSERT_GT(sequence, 0);
+      ASSERT_LE(sequence, kNumKeys * kThreads);
+      // The sequence isn't consumed by someone else.
+      ASSERT_FALSE(flags[sequence].test_and_set());
+    }
+  };
+  std::vector<port::Thread> threads;
+  for (size_t i = 0; i < kThreads; i++) {
+    threads.emplace_back(writer, i);
+  }
+  for (size_t i = 0; i < kThreads; i++) {
+    threads[i].join();
+  }
+}
+
 TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) {
  constexpr int kNumThreads = 5;
  std::unique_ptr<FaultInjectionTestEnv> mock_env(
@@ -54,6 +54,7 @@ class Reader {
  // The Reader will start reading at the first record located at physical
  // position >= initial_offset within the file.
  Reader(std::shared_ptr<Logger> info_log,
+  // @lint-ignore TXT2 T25377293 Grandfathered in
 	 unique_ptr<SequentialFileReader>&& file,
         Reporter* reporter, bool checksum, uint64_t initial_offset,
         uint64_t log_num);
@@ -368,6 +368,11 @@ class MemTable {
    return oldest_key_time_.load(std::memory_order_relaxed);
  }

+  // REQUIRES: db_mutex held.
+  void SetID(uint64_t id) { id_ = id; }
+
+  uint64_t GetID() const { return id_; }
+
 private:
  enum FlushStateEnum { FLUSH_NOT_REQUESTED, FLUSH_REQUESTED, FLUSH_SCHEDULED };

@@ -437,6 +442,9 @@ class MemTable {
  // Timestamp of oldest key
  std::atomic<uint64_t> oldest_key_time_;

+  // Memtable id to track flush.
+  uint64_t id_ = 0;
+
  // Returns a heuristic flush decision
  bool ShouldFlushNow() const;

@@ -5,11 +5,12 @@
 //
 #pragma once

-#include <string>
-#include <list>
-#include <vector>
-#include <set>
 #include <deque>
+#include <limits>
+#include <list>
+#include <set>
+#include <string>
+#include <vector>

 #include "db/dbformat.h"
 #include "db/memtable.h"
@@ -244,6 +245,22 @@ class MemTableList {

  uint64_t GetMinLogContainingPrepSection();

+  uint64_t GetEarliestMemTableID() const {
+    auto& memlist = current_->memlist_;
+    if (memlist.empty()) {
+      return std::numeric_limits<uint64_t>::max();
+    }
+    return memlist.back()->GetID();
+  }
+
+  uint64_t GetLatestMemTableID() const {
+    auto& memlist = current_->memlist_;
+    if (memlist.empty()) {
+      return 0;
+    }
+    return memlist.front()->GetID();
+  }
+
 private:
  // DB mutex held
  void InstallNewVersion();
@@ -108,6 +108,22 @@ class SnapshotList {
    return ret;
  }

+  // Whether there is an active snapshot in range [lower_bound, upper_bound).
+  bool HasSnapshotInRange(SequenceNumber lower_bound,
+                          SequenceNumber upper_bound) {
+    if (empty()) {
+      return false;
+    }
+    const SnapshotImpl* s = &list_;
+    while (s->next_ != &list_) {
+      if (s->next_->number_ >= lower_bound) {
+        return s->next_->number_ < upper_bound;
+      }
+      s = s->next_;
+    }
+    return false;
+  }
+
  // get the sequence number of the most recent snapshot
  SequenceNumber GetNewest() {
    if (empty()) {
@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 export USE_HDFS=1
 export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64:/usr/lib/hadoop/lib/native

@@ -97,6 +97,9 @@ class Iterator : public Cleanable {
  // Property "rocksdb.iterator.super-version-number":
  //   LSM version used by the iterator. The same format as DB Property
  //   kCurrentSuperVersionNumber. See its comment for more information.
+  // Property "rocksdb.iterator.internal-key":
+  //   Get the user-key portion of the internal key at which the iteration
+  //   stopped.
  virtual Status GetProperty(std::string prop_name, std::string* prop);

 private:
@@ -415,7 +415,7 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
    {BLOB_DB_WRITE_BLOB, "rocksdb.blobdb.write.blob"},
    {BLOB_DB_WRITE_BLOB_TTL, "rocksdb.blobdb.write.blob.ttl"},
    {BLOB_DB_BLOB_FILE_BYTES_WRITTEN, "rocksdb.blobdb.blob.file.bytes.written"},
-    {BLOB_DB_BLOB_FILE_BYTES_READ, "rocksdb.blobdb.blob.file.bytes.read"},
+    {BLOB_DB_BLOB_FILE_BYTES_READ, "rocksdb.blobdb.blob.file,bytes.read"},
    {BLOB_DB_BLOB_FILE_SYNCED, "rocksdb.blobdb.blob.file.synced"},
    {BLOB_DB_BLOB_INDEX_EXPIRED, "rocksdb.blobdb.blob.index.expired"},
    {BLOB_DB_GC_NUM_FILES, "rocksdb.blobdb.gc.num.files"},
@@ -6,7 +6,7 @@

 #define ROCKSDB_MAJOR 5
 #define ROCKSDB_MINOR 10
-#define ROCKSDB_PATCH 0
+#define ROCKSDB_PATCH 5

 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these
@@ -7,6 +7,8 @@ set(JNI_NATIVE_SOURCES
        rocksjni/clock_cache.cc
        rocksjni/columnfamilyhandle.cc
        rocksjni/compaction_filter.cc
+        rocksjni/compaction_filter_factory.cc
+        rocksjni/compaction_filter_factory_jnicallback.cc
        rocksjni/compaction_options_fifo.cc
        rocksjni/compaction_options_universal.cc
        rocksjni/comparator.cc
@@ -17,6 +19,7 @@ set(JNI_NATIVE_SOURCES
        rocksjni/filter.cc
        rocksjni/ingest_external_file_options.cc
        rocksjni/iterator.cc
+        rocksjni/jnicallback.cc
        rocksjni/loggerjnicallback.cc
        rocksjni/lru_cache.cc
        rocksjni/memtablejni.cc
@@ -25,6 +28,7 @@ set(JNI_NATIVE_SOURCES
        rocksjni/options_util.cc
        rocksjni/ratelimiterjni.cc
        rocksjni/remove_emptyvalue_compactionfilterjni.cc
+        rocksjni/rocks_callback_object.cc
        rocksjni/cassandra_compactionfilterjni.cc
        rocksjni/cassandra_value_operator.cc
        rocksjni/restorejni.cc
@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 PLATFORM=64
 if [ `getconf LONG_BIT` != "64" ]
 then
@@ -7,4 +8,5 @@ fi
 ROCKS_JAR=`find target -name rocksdbjni*.jar`

 echo "Running benchmark in $PLATFORM-Bit mode."
+# shellcheck disable=SC2068
 java -server -d$PLATFORM -XX:NewSize=4m -XX:+AggressiveOpts -Djava.library.path=target -cp "${ROCKS_JAR}:benchmark/target/classes" org.rocksdb.benchmark.DbBenchmark $@
@@ -34,4 +34,5 @@ void Java_org_rocksdb_AbstractCompactionFilterFactory_disposeInternal(
  auto* ptr_sptr_cff =
      reinterpret_cast<std::shared_ptr<rocksdb::CompactionFilterFactoryJniCallback> *>(jhandle);
  delete ptr_sptr_cff;
+// @lint-ignore TXT4 T25377293 Grandfathered in
 }
@@ -146,4 +146,5 @@ void Java_org_rocksdb_IngestExternalFileOptions_disposeInternal(
  auto* options =
      reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
  delete options;
+// @lint-ignore TXT4 T25377293 Grandfathered in
 }
@@ -49,4 +49,5 @@ JniCallback::~JniCallback() {

  releaseJniEnv(attached_thread);
 }
+// @lint-ignore TXT4 T25377293 Grandfathered in
 }  // namespace rocksdb
@@ -25,4 +25,5 @@ namespace rocksdb {
  };
 }

+// @lint-ignore TXT4 T25377293 Grandfathered in
 #endif  // JAVA_ROCKSJNI_JNICALLBACK_H_
@@ -24,4 +24,5 @@ void Java_org_rocksdb_RocksCallbackObject_disposeInternal(
      //   2) Comparator -> BaseComparatorJniCallback + JniCallback -> ComparatorJniCallback
      // I think this is okay, as Comparator and JniCallback both have virtual destructors...
      delete reinterpret_cast<rocksdb::JniCallback*>(handle);
+// @lint-ignore TXT4 T25377293 Grandfathered in
 }
@@ -30,4 +30,5 @@ namespace rocksdb {

    return true;
  }
+// @lint-ignore TXT4 T25377293 Grandfathered in
 };
@@ -30,4 +30,5 @@ namespace rocksdb {

 }  // namespace rocksdb

+// @lint-ignore TXT4 T25377293 Grandfathered in
 #endif  // JAVA_ROCKSJNI_STATISTICSJNI_H_
@@ -12,6 +12,10 @@ public class Environment {
    return (OS.contains("win"));
  }

+  public static boolean isFreeBSD() {
+    return (OS.contains("freebsd"));
+  }
+
  public static boolean isMac() {
    return (OS.contains("mac"));
  }
@@ -54,6 +58,8 @@ public class Environment {
      }
    } else if (isMac()) {
      return String.format("%sjni-osx", name);
+    } else if (isFreeBSD()) {
+      return String.format("%sjni-freebsd%s", name, is64Bit() ? "64" : "32");
    } else if (isAix() && is64Bit()) {
      return String.format("%sjni-aix64", name);
    } else if (isSolaris()) {
@@ -71,7 +77,7 @@ public class Environment {
  }

  private static String appendLibOsSuffix(final String libraryFileName, final boolean shared) {
-    if (isUnix() || isAix() || isSolaris()) {
+    if (isUnix() || isAix() || isSolaris() || isFreeBSD()) {
      return libraryFileName + ".so";
    } else if (isMac()) {
      return libraryFileName + (shared ? ".dylib" : ".jnilib");
@@ -279,7 +279,7 @@ struct InlineSkipList<Comparator>::Node {
  // next_[0].  This is used for passing data from AllocateKey to Insert.
  void StashHeight(const int height) {
    assert(sizeof(int) <= sizeof(next_[0]));
-    memcpy(&next_[0], &height, sizeof(int));
+    memcpy(static_cast<void*>(&next_[0]), &height, sizeof(int));
  }

  // Retrieves the value passed to StashHeight.  Undefined after a call
@@ -299,30 +299,30 @@ struct InlineSkipList<Comparator>::Node {
    assert(n >= 0);
    // Use an 'acquire load' so that we observe a fully initialized
    // version of the returned Node.
-    return (next_[-n].load(std::memory_order_acquire));
+    return ((&next_[0] - n)->load(std::memory_order_acquire));
  }

  void SetNext(int n, Node* x) {
    assert(n >= 0);
    // Use a 'release store' so that anybody who reads through this
    // pointer observes a fully initialized version of the inserted node.
-    next_[-n].store(x, std::memory_order_release);
+    (&next_[0] - n)->store(x, std::memory_order_release);
  }

  bool CASNext(int n, Node* expected, Node* x) {
    assert(n >= 0);
-    return next_[-n].compare_exchange_strong(expected, x);
+    return (&next_[0] - n)->compare_exchange_strong(expected, x);
  }

  // No-barrier variants that can be safely used in a few locations.
  Node* NoBarrier_Next(int n) {
    assert(n >= 0);
-    return next_[-n].load(std::memory_order_relaxed);
+    return (&next_[0] - n)->load(std::memory_order_relaxed);
  }

  void NoBarrier_SetNext(int n, Node* x) {
    assert(n >= 0);
-    next_[-n].store(x, std::memory_order_relaxed);
+    (&next_[0] - n)->store(x, std::memory_order_relaxed);
  }

  // Insert node after prev on specific level.
@@ -32,7 +32,7 @@ namespace port {

 namespace {

-#ifdef OS_LINUX
+#if defined(OS_LINUX) || defined(OS_FREEBSD)
 const char* GetExecutableName() {
  static char name[1024];

@@ -49,6 +49,7 @@ class CuckooBuilderTest : public testing::Test {
    uint64_t read_file_size;
    ASSERT_OK(env_->GetFileSize(fname, &read_file_size));

+   // @lint-ignore TXT2 T25377293 Grandfathered in
 	  Options options;
 	  options.allow_mmap_reads = true;
 	  ImmutableCFOptions ioptions(options);
@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 TMP_DIR="${TMPDIR:-/tmp}/rocksdb-sanity-test"

 if [ "$#" -lt 2 ]; then
@@ -449,6 +449,7 @@ echo "===== Benchmark ====="

 # Run!!!
 IFS=',' read -a jobs <<< $1
+# shellcheck disable=SC2068
 for job in ${jobs[@]}; do

  if [ $job != debug ]; then
@@ -151,6 +151,7 @@ echo "===== Benchmark ====="

 # Run!!!
 IFS=',' read -a jobs <<< $1
+# shellcheck disable=SC2068
 for job in ${jobs[@]}; do

  if [ $job != debug ]; then
@@ -3661,9 +3661,7 @@ void VerifyDBFromDB(std::string& truth_db_name) {
        }
      }
      if (!use_blob_db_) {
-#ifndef ROCKSDB_LITE
        s = db_with_cfh->db->Write(write_options_, &batch);
-#endif  //  ROCKSDB_LITE
      }
      thread->stats.FinishedOps(db_with_cfh, db_with_cfh->db,
                                entries_per_batch_, kWrite);
@@ -317,7 +317,7 @@ def whitebox_crash_main(args):
        cmd = gen_cmd(dict(cmd_params.items() + additional_opts.items()
                           + {'db': dbname}.items()))

-        print "Running:" + ' '.join(cmd) + "\n"
+        print "Running:" + ' '.join(cmd) + "\n"  # noqa: E999 T25377293 Grandfathered in

        popen = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
@@ -229,7 +229,7 @@ class LDBTestCase(unittest.TestCase):
        self.assertRunFAIL("get --ttl a3")
        self.assertRunOK("checkconsistency", "OK")

-    def testInvalidCmdLines(self):
+    def testInvalidCmdLines(self):  # noqa: F811 T25377293 Grandfathered in
        print "Running testInvalidCmdLines..."
        # db not specified
        self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
@@ -516,7 +516,7 @@ class LDBTestCase(unittest.TestCase):

    def testColumnFamilies(self):
        print "Running testColumnFamilies..."
-        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)  # noqa: F841 T25377293 Grandfathered in
        self.assertRunOK("put cf1_1 1 --create_if_missing", "OK")
        self.assertRunOK("put cf1_2 2 --create_if_missing", "OK")
        self.assertRunOK("put cf1_3 3 --try_load_options", "OK")
@@ -465,4 +465,5 @@ function cleanup_test_directory {

 ############################################################################

+# shellcheck disable=SC2068
 main $@
@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 TESTDIR=`mktemp -d ${TMPDIR:-/tmp}/rocksdb-dump-test.XXXXX`
 DUMPFILE="tools/sample-dump.dmp"

@@ -21,7 +21,7 @@ def generate_runtimes(total_runtime):

 def main(args):
    runtimes = generate_runtimes(int(args.runtime_sec))
-    print "Going to execute write stress for " + str(runtimes)
+    print "Going to execute write stress for " + str(runtimes)  # noqa: E999 T25377293 Grandfathered in
    first_time = True

    for runtime in runtimes:
@@ -688,259 +688,386 @@ uint32_t crc32c_3way(uint32_t crc, const char* buf, size_t len) {
          do {
            // jumps here for a full block of len 128
            CRCtriplet(crc, next, -128);
+            // FALLTHRU
            case 127:
              // jumps here or below for the first block smaller
              CRCtriplet(crc, next, -127);
+              // FALLTHRU
            case 126:
              CRCtriplet(crc, next, -126); // than 128
+              // FALLTHRU
            case 125:
              CRCtriplet(crc, next, -125);
+              // FALLTHRU
            case 124:
              CRCtriplet(crc, next, -124);
+              // FALLTHRU
            case 123:
              CRCtriplet(crc, next, -123);
+              // FALLTHRU
            case 122:
              CRCtriplet(crc, next, -122);
+              // FALLTHRU
            case 121:
              CRCtriplet(crc, next, -121);
+              // FALLTHRU
            case 120:
              CRCtriplet(crc, next, -120);
+              // FALLTHRU
            case 119:
              CRCtriplet(crc, next, -119);
+              // FALLTHRU
            case 118:
              CRCtriplet(crc, next, -118);
+              // FALLTHRU
            case 117:
              CRCtriplet(crc, next, -117);
+              // FALLTHRU
            case 116:
              CRCtriplet(crc, next, -116);
+              // FALLTHRU
            case 115:
              CRCtriplet(crc, next, -115);
+              // FALLTHRU
            case 114:
              CRCtriplet(crc, next, -114);
+              // FALLTHRU
            case 113:
              CRCtriplet(crc, next, -113);
+              // FALLTHRU
            case 112:
              CRCtriplet(crc, next, -112);
+              // FALLTHRU
            case 111:
              CRCtriplet(crc, next, -111);
+              // FALLTHRU
            case 110:
              CRCtriplet(crc, next, -110);
+              // FALLTHRU
            case 109:
              CRCtriplet(crc, next, -109);
+              // FALLTHRU
            case 108:
              CRCtriplet(crc, next, -108);
+              // FALLTHRU
            case 107:
              CRCtriplet(crc, next, -107);
+              // FALLTHRU
            case 106:
              CRCtriplet(crc, next, -106);
+              // FALLTHRU
            case 105:
              CRCtriplet(crc, next, -105);
+              // FALLTHRU
            case 104:
              CRCtriplet(crc, next, -104);
+              // FALLTHRU
            case 103:
              CRCtriplet(crc, next, -103);
+              // FALLTHRU
            case 102:
              CRCtriplet(crc, next, -102);
+              // FALLTHRU
            case 101:
              CRCtriplet(crc, next, -101);
+              // FALLTHRU
            case 100:
              CRCtriplet(crc, next, -100);
+              // FALLTHRU
            case 99:
              CRCtriplet(crc, next, -99);
+              // FALLTHRU
            case 98:
              CRCtriplet(crc, next, -98);
+              // FALLTHRU
            case 97:
              CRCtriplet(crc, next, -97);
+              // FALLTHRU
            case 96:
              CRCtriplet(crc, next, -96);
+              // FALLTHRU
            case 95:
              CRCtriplet(crc, next, -95);
+              // FALLTHRU
            case 94:
              CRCtriplet(crc, next, -94);
+              // FALLTHRU
            case 93:
              CRCtriplet(crc, next, -93);
+              // FALLTHRU
            case 92:
              CRCtriplet(crc, next, -92);
+              // FALLTHRU
            case 91:
              CRCtriplet(crc, next, -91);
+              // FALLTHRU
            case 90:
              CRCtriplet(crc, next, -90);
+              // FALLTHRU
            case 89:
              CRCtriplet(crc, next, -89);
+              // FALLTHRU
            case 88:
              CRCtriplet(crc, next, -88);
+              // FALLTHRU
            case 87:
              CRCtriplet(crc, next, -87);
+              // FALLTHRU
            case 86:
              CRCtriplet(crc, next, -86);
+              // FALLTHRU
            case 85:
              CRCtriplet(crc, next, -85);
+              // FALLTHRU
            case 84:
              CRCtriplet(crc, next, -84);
+              // FALLTHRU
            case 83:
              CRCtriplet(crc, next, -83);
+              // FALLTHRU
            case 82:
              CRCtriplet(crc, next, -82);
+              // FALLTHRU
            case 81:
              CRCtriplet(crc, next, -81);
+              // FALLTHRU
            case 80:
              CRCtriplet(crc, next, -80);
+              // FALLTHRU
            case 79:
              CRCtriplet(crc, next, -79);
+              // FALLTHRU
            case 78:
              CRCtriplet(crc, next, -78);
+              // FALLTHRU
            case 77:
              CRCtriplet(crc, next, -77);
+              // FALLTHRU
            case 76:
              CRCtriplet(crc, next, -76);
+              // FALLTHRU
            case 75:
              CRCtriplet(crc, next, -75);
+              // FALLTHRU
            case 74:
              CRCtriplet(crc, next, -74);
+              // FALLTHRU
            case 73:
              CRCtriplet(crc, next, -73);
+              // FALLTHRU
            case 72:
              CRCtriplet(crc, next, -72);
+              // FALLTHRU
            case 71:
              CRCtriplet(crc, next, -71);
+              // FALLTHRU
            case 70:
              CRCtriplet(crc, next, -70);
+              // FALLTHRU
            case 69:
              CRCtriplet(crc, next, -69);
+              // FALLTHRU
            case 68:
              CRCtriplet(crc, next, -68);
+              // FALLTHRU
            case 67:
              CRCtriplet(crc, next, -67);
+              // FALLTHRU
            case 66:
              CRCtriplet(crc, next, -66);
+              // FALLTHRU
            case 65:
              CRCtriplet(crc, next, -65);
+              // FALLTHRU
            case 64:
              CRCtriplet(crc, next, -64);
+              // FALLTHRU
            case 63:
              CRCtriplet(crc, next, -63);
+              // FALLTHRU
            case 62:
              CRCtriplet(crc, next, -62);
+              // FALLTHRU
            case 61:
              CRCtriplet(crc, next, -61);
+              // FALLTHRU
            case 60:
              CRCtriplet(crc, next, -60);
+              // FALLTHRU
            case 59:
              CRCtriplet(crc, next, -59);
+              // FALLTHRU
            case 58:
              CRCtriplet(crc, next, -58);
+              // FALLTHRU
            case 57:
              CRCtriplet(crc, next, -57);
+              // FALLTHRU
            case 56:
              CRCtriplet(crc, next, -56);
+              // FALLTHRU
            case 55:
              CRCtriplet(crc, next, -55);
+              // FALLTHRU
            case 54:
              CRCtriplet(crc, next, -54);
+              // FALLTHRU
            case 53:
              CRCtriplet(crc, next, -53);
+              // FALLTHRU
            case 52:
              CRCtriplet(crc, next, -52);
+              // FALLTHRU
            case 51:
              CRCtriplet(crc, next, -51);
+              // FALLTHRU
            case 50:
              CRCtriplet(crc, next, -50);
+              // FALLTHRU
            case 49:
              CRCtriplet(crc, next, -49);
+              // FALLTHRU
            case 48:
              CRCtriplet(crc, next, -48);
+              // FALLTHRU
            case 47:
              CRCtriplet(crc, next, -47);
+              // FALLTHRU
            case 46:
              CRCtriplet(crc, next, -46);
+              // FALLTHRU
            case 45:
              CRCtriplet(crc, next, -45);
+              // FALLTHRU
            case 44:
              CRCtriplet(crc, next, -44);
+              // FALLTHRU
            case 43:
              CRCtriplet(crc, next, -43);
+              // FALLTHRU
            case 42:
              CRCtriplet(crc, next, -42);
+              // FALLTHRU
            case 41:
              CRCtriplet(crc, next, -41);
+              // FALLTHRU
            case 40:
              CRCtriplet(crc, next, -40);
+              // FALLTHRU
            case 39:
              CRCtriplet(crc, next, -39);
+              // FALLTHRU
            case 38:
              CRCtriplet(crc, next, -38);
+              // FALLTHRU
            case 37:
              CRCtriplet(crc, next, -37);
+              // FALLTHRU
            case 36:
              CRCtriplet(crc, next, -36);
+              // FALLTHRU
            case 35:
              CRCtriplet(crc, next, -35);
+              // FALLTHRU
            case 34:
              CRCtriplet(crc, next, -34);
+              // FALLTHRU
            case 33:
              CRCtriplet(crc, next, -33);
+              // FALLTHRU
            case 32:
              CRCtriplet(crc, next, -32);
+              // FALLTHRU
            case 31:
              CRCtriplet(crc, next, -31);
+              // FALLTHRU
            case 30:
              CRCtriplet(crc, next, -30);
+              // FALLTHRU
            case 29:
              CRCtriplet(crc, next, -29);
+              // FALLTHRU
            case 28:
              CRCtriplet(crc, next, -28);
+              // FALLTHRU
            case 27:
              CRCtriplet(crc, next, -27);
+              // FALLTHRU
            case 26:
              CRCtriplet(crc, next, -26);
+              // FALLTHRU
            case 25:
              CRCtriplet(crc, next, -25);
+              // FALLTHRU
            case 24:
              CRCtriplet(crc, next, -24);
+              // FALLTHRU
            case 23:
              CRCtriplet(crc, next, -23);
+              // FALLTHRU
            case 22:
              CRCtriplet(crc, next, -22);
+              // FALLTHRU
            case 21:
              CRCtriplet(crc, next, -21);
+              // FALLTHRU
            case 20:
              CRCtriplet(crc, next, -20);
+              // FALLTHRU
            case 19:
              CRCtriplet(crc, next, -19);
+              // FALLTHRU
            case 18:
              CRCtriplet(crc, next, -18);
+              // FALLTHRU
            case 17:
              CRCtriplet(crc, next, -17);
+              // FALLTHRU
            case 16:
              CRCtriplet(crc, next, -16);
+              // FALLTHRU
            case 15:
              CRCtriplet(crc, next, -15);
+              // FALLTHRU
            case 14:
              CRCtriplet(crc, next, -14);
+              // FALLTHRU
            case 13:
              CRCtriplet(crc, next, -13);
+              // FALLTHRU
            case 12:
              CRCtriplet(crc, next, -12);
+              // FALLTHRU
            case 11:
              CRCtriplet(crc, next, -11);
+              // FALLTHRU
            case 10:
              CRCtriplet(crc, next, -10);
+              // FALLTHRU
            case 9:
              CRCtriplet(crc, next, -9);
+              // FALLTHRU
            case 8:
              CRCtriplet(crc, next, -8);
+              // FALLTHRU
            case 7:
              CRCtriplet(crc, next, -7);
+              // FALLTHRU
            case 6:
              CRCtriplet(crc, next, -6);
+              // FALLTHRU
            case 5:
              CRCtriplet(crc, next, -5);
+              // FALLTHRU
            case 4:
              CRCtriplet(crc, next, -4);
+              // FALLTHRU
            case 3:
              CRCtriplet(crc, next, -3);
+              // FALLTHRU
            case 2:
              CRCtriplet(crc, next, -2);
+              // FALLTHRU
            case 1:
              CRCduplet(crc, next, -1); // the final triplet is actually only 2
              //{ CombineCRC(); }
@@ -953,6 +1080,7 @@ uint32_t crc32c_3way(uint32_t crc, const char* buf, size_t len) {
                next1 = next0 + 128; // from here on all blocks are 128 long
                next2 = next1 + 128;
              }
+              // FALLTHRU
            case 0:;
          } while (n > 0);
      }
@@ -964,58 +1092,85 @@ uint32_t crc32c_3way(uint32_t crc, const char* buf, size_t len) {
    switch (count2) {
      case 27:
        CRCsinglet(crc0, next, -27 * 8);
+        // FALLTHRU
      case 26:
        CRCsinglet(crc0, next, -26 * 8);
+        // FALLTHRU
      case 25:
        CRCsinglet(crc0, next, -25 * 8);
+        // FALLTHRU
      case 24:
        CRCsinglet(crc0, next, -24 * 8);
+        // FALLTHRU
      case 23:
        CRCsinglet(crc0, next, -23 * 8);
+        // FALLTHRU
      case 22:
        CRCsinglet(crc0, next, -22 * 8);
+        // FALLTHRU
      case 21:
        CRCsinglet(crc0, next, -21 * 8);
+        // FALLTHRU
      case 20:
        CRCsinglet(crc0, next, -20 * 8);
+        // FALLTHRU
      case 19:
        CRCsinglet(crc0, next, -19 * 8);
+        // FALLTHRU
      case 18:
        CRCsinglet(crc0, next, -18 * 8);
+        // FALLTHRU
      case 17:
        CRCsinglet(crc0, next, -17 * 8);
+        // FALLTHRU
      case 16:
        CRCsinglet(crc0, next, -16 * 8);
+        // FALLTHRU
      case 15:
        CRCsinglet(crc0, next, -15 * 8);
+        // FALLTHRU
      case 14:
        CRCsinglet(crc0, next, -14 * 8);
+        // FALLTHRU
      case 13:
        CRCsinglet(crc0, next, -13 * 8);
+        // FALLTHRU
      case 12:
        CRCsinglet(crc0, next, -12 * 8);
+        // FALLTHRU
      case 11:
        CRCsinglet(crc0, next, -11 * 8);
+        // FALLTHRU
      case 10:
        CRCsinglet(crc0, next, -10 * 8);
+        // FALLTHRU
      case 9:
        CRCsinglet(crc0, next, -9 * 8);
+        // FALLTHRU
      case 8:
        CRCsinglet(crc0, next, -8 * 8);
+        // FALLTHRU
      case 7:
        CRCsinglet(crc0, next, -7 * 8);
+        // FALLTHRU
      case 6:
        CRCsinglet(crc0, next, -6 * 8);
+        // FALLTHRU
      case 5:
        CRCsinglet(crc0, next, -5 * 8);
+        // FALLTHRU
      case 4:
        CRCsinglet(crc0, next, -4 * 8);
+        // FALLTHRU
      case 3:
        CRCsinglet(crc0, next, -3 * 8);
+        // FALLTHRU
      case 2:
        CRCsinglet(crc0, next, -2 * 8);
+        // FALLTHRU
      case 1:
        CRCsinglet(crc0, next, -1 * 8);
+        // FALLTHRU
      case 0:;
    }
  }
@@ -239,6 +239,9 @@ Status WritableFileWriter::Close() {
  // we need to let the file know where data ends.
  if (use_direct_io()) {
    interim = writable_file_->Truncate(filesize_);
+    if (interim.ok()) {
+      interim = writable_file_->Fsync();
+    }
    if (!interim.ok() && s.ok()) {
      s = interim;
    }
@@ -24,6 +24,13 @@ namespace rocksdb {
 // pointer (if not NULL) when one of the following happens:
 // (1) a thread terminates
 // (2) a ThreadLocalPtr is destroyed
+//
+// Warning: this function is called while holding a global mutex. The same mutex
+// is used (at least in some cases) by most methods of ThreadLocalPtr, and it's
+// shared across all instances of ThreadLocalPtr. Thereforere extra care
+// is needed to avoid deadlocks. In particular, the handler shouldn't lock any
+// mutexes and shouldn't call any methods of any ThreadLocalPtr instances,
+// unless you know what you're doing.
 typedef void (*UnrefHandler)(void* ptr);

 // ThreadLocalPtr stores only values of pointer type.  Different from
@@ -77,8 +77,9 @@ Status RandomTransactionInserter::DBGet(
    uint64_t ikey, bool get_for_update, uint64_t* int_value,
    std::string* full_key, bool* unexpected_error) {
  Status s;
-  // four digits and zero end char
-  char prefix_buf[5];
+  // Five digits (since the largest uint16_t is 65535) plus the NUL
+  // end char.
+  char prefix_buf[6];
  // Pad prefix appropriately so we can iterate over each set
  snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1);
  // key format:  [SET#][random#]
@@ -261,8 +262,10 @@ Status RandomTransactionInserter::Verify(DB* db, uint16_t num_sets,
  }
  // For each set of keys with the same prefix, sum all the values
  for (uint16_t set_i : set_vec) {
-    // four digits and zero end char
-    char prefix_buf[5];
+    // Five digits (since the largest uint16_t is 65535) plus the NUL
+    // end char.
+    char prefix_buf[6];
+    assert(set_i + 1 <= 9999);
    snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1);
    uint64_t total = 0;

@@ -146,11 +146,17 @@ class BackupEngineImpl : public BackupEngine {

  class BackupMeta {
   public:
-    BackupMeta(const std::string& meta_filename,
+    BackupMeta(
+        const std::string& meta_filename, const std::string& meta_tmp_filename,
        std::unordered_map<std::string, std::shared_ptr<FileInfo>>* file_infos,
        Env* env)
-      : timestamp_(0), sequence_number_(0), size_(0),
-        meta_filename_(meta_filename), file_infos_(file_infos), env_(env) {}
+        : timestamp_(0),
+          sequence_number_(0),
+          size_(0),
+          meta_filename_(meta_filename),
+          meta_tmp_filename_(meta_tmp_filename),
+          file_infos_(file_infos),
+          env_(env) {}

    BackupMeta(const BackupMeta&) = delete;
    BackupMeta& operator=(const BackupMeta&) = delete;
@@ -228,6 +234,7 @@ class BackupEngineImpl : public BackupEngine {
    uint64_t size_;
    std::string app_metadata_;
    std::string const meta_filename_;
+    std::string const meta_tmp_filename_;
    // files with relative paths (without "/" prefix!!)
    std::vector<std::shared_ptr<FileInfo>> files_;
    std::unordered_map<std::string, std::shared_ptr<FileInfo>>* file_infos_;
@@ -257,12 +264,14 @@ class BackupEngineImpl : public BackupEngine {
  inline std::string GetSharedFileRel(const std::string& file = "",
                                      bool tmp = false) const {
    assert(file.size() == 0 || file[0] != '/');
-    return "shared/" + file + (tmp ? ".tmp" : "");
+    return std::string("shared/") + (tmp ? "." : "") + file +
+           (tmp ? ".tmp" : "");
  }
  inline std::string GetSharedFileWithChecksumRel(const std::string& file = "",
                                                  bool tmp = false) const {
    assert(file.size() == 0 || file[0] != '/');
-    return GetSharedChecksumDirRel() + "/" + file + (tmp ? ".tmp" : "");
+    return GetSharedChecksumDirRel() + "/" + (tmp ? "." : "") + file +
+           (tmp ? ".tmp" : "");
  }
  inline std::string GetSharedFileWithChecksum(const std::string& file,
                                               const uint32_t checksum_value,
@@ -283,8 +292,9 @@ class BackupEngineImpl : public BackupEngine {
  inline std::string GetBackupMetaDir() const {
    return GetAbsolutePath("meta");
  }
-  inline std::string GetBackupMetaFile(BackupID backup_id) const {
-    return GetBackupMetaDir() + "/" + rocksdb::ToString(backup_id);
+  inline std::string GetBackupMetaFile(BackupID backup_id, bool tmp) const {
+    return GetBackupMetaDir() + "/" + (tmp ? "." : "") +
+           rocksdb::ToString(backup_id) + (tmp ? ".tmp" : "");
  }

  // If size_limit == 0, there is no size limit, copy everything.
@@ -605,10 +615,11 @@ Status BackupEngineImpl::Initialize() {
      continue;
    }
    assert(backups_.find(backup_id) == backups_.end());
-    backups_.insert(
-        std::make_pair(backup_id, unique_ptr<BackupMeta>(new BackupMeta(
-                                      GetBackupMetaFile(backup_id),
-                                      &backuped_file_infos_, backup_env_))));
+    backups_.insert(std::make_pair(
+        backup_id, unique_ptr<BackupMeta>(new BackupMeta(
+                       GetBackupMetaFile(backup_id, false /* tmp */),
+                       GetBackupMetaFile(backup_id, true /* tmp */),
+                       &backuped_file_infos_, backup_env_))));
  }

  latest_backup_id_ = 0;
@@ -736,10 +747,11 @@ Status BackupEngineImpl::CreateNewBackupWithMetadata(
  BackupID new_backup_id = latest_backup_id_ + 1;

  assert(backups_.find(new_backup_id) == backups_.end());
-  auto ret = backups_.insert(
-      std::make_pair(new_backup_id, unique_ptr<BackupMeta>(new BackupMeta(
-                                        GetBackupMetaFile(new_backup_id),
-                                        &backuped_file_infos_, backup_env_))));
+  auto ret = backups_.insert(std::make_pair(
+      new_backup_id, unique_ptr<BackupMeta>(new BackupMeta(
+                         GetBackupMetaFile(new_backup_id, false /* tmp */),
+                         GetBackupMetaFile(new_backup_id, true /* tmp */),
+                         &backuped_file_infos_, backup_env_))));
  assert(ret.second == true);
  auto& new_backup = ret.first->second;
  new_backup->RecordTimestamp();
@@ -1708,8 +1720,7 @@ Status BackupEngineImpl::BackupMeta::StoreToFile(bool sync) {
  EnvOptions env_options;
  env_options.use_mmap_writes = false;
  env_options.use_direct_writes = false;
-  s = env_->NewWritableFile(meta_filename_ + ".tmp", &backup_meta_file,
-                            env_options);
+  s = env_->NewWritableFile(meta_tmp_filename_, &backup_meta_file, env_options);
  if (!s.ok()) {
    return s;
  }
@@ -1749,7 +1760,7 @@ Status BackupEngineImpl::BackupMeta::StoreToFile(bool sync) {
    s = backup_meta_file->Close();
  }
  if (s.ok()) {
-    s = env_->RenameFile(meta_filename_ + ".tmp", meta_filename_);
+    s = env_->RenameFile(meta_tmp_filename_, meta_filename_);
  }
  return s;
 }
@@ -810,9 +810,9 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
  test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
  ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
  std::vector<std::string> should_have_written = {
-      "/shared/00010.sst.tmp",    "/shared/00011.sst.tmp",
+      "/shared/.00010.sst.tmp",    "/shared/.00011.sst.tmp",
      "/private/1.tmp/CURRENT",   "/private/1.tmp/MANIFEST-01",
-      "/private/1.tmp/00011.log", "/meta/1.tmp"};
+      "/private/1.tmp/00011.log", "/meta/.1.tmp"};
  AppendPath(backupdir_, should_have_written);
  test_backup_env_->AssertWrittenFiles(should_have_written);

@@ -828,9 +828,9 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
  ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
  // should not open 00010.sst - it's already there

-  should_have_written = {"/shared/00015.sst.tmp", "/private/2.tmp/CURRENT",
+  should_have_written = {"/shared/.00015.sst.tmp", "/private/2.tmp/CURRENT",
                         "/private/2.tmp/MANIFEST-01",
-                         "/private/2.tmp/00011.log", "/meta/2.tmp"};
+                         "/private/2.tmp/00011.log", "/meta/.2.tmp"};
  AppendPath(backupdir_, should_have_written);
  test_backup_env_->AssertWrittenFiles(should_have_written);

@@ -1169,7 +1169,7 @@ TEST_F(BackupableDBTest, DeleteTmpFiles) {
    } else {
      shared_tmp += "/shared";
    }
-    shared_tmp += "/00006.sst.tmp";
+    shared_tmp += "/.00006.sst.tmp";
    std::string private_tmp_dir = backupdir_ + "/private/10.tmp";
    std::string private_tmp_file = private_tmp_dir + "/00003.sst";
    file_manager_->WriteToFile(shared_tmp, "tmp");
@@ -73,8 +73,6 @@ void BlobDBOptions::Dump(Logger* log) const {
                   blob_dir_size);
  ROCKS_LOG_HEADER(log, "           blob_db_options.ttl_range_secs: %" PRIu32,
                   ttl_range_secs);
-  ROCKS_LOG_HEADER(log, "            blob_db_options.min_blob_size: %" PRIu64,
-                   min_blob_size);
  ROCKS_LOG_HEADER(log, "           blob_db_options.bytes_per_sync: %" PRIu64,
                   bytes_per_sync);
  ROCKS_LOG_HEADER(log, "           blob_db_options.blob_file_size: %" PRIu64,
@@ -56,10 +56,8 @@ struct BlobDBOptions {
  // will be inlined in base DB together with the key.
  uint64_t min_blob_size = 0;

-  // Allows OS to incrementally sync blob files to disk for every
-  // bytes_per_sync bytes written. Users shouldn't rely on it for
-  // persistency guarantee.
-  uint64_t bytes_per_sync = 512 * 1024;
+  // at what bytes will the blob files be synced to blob log.
+  uint64_t bytes_per_sync = 0;

  // the target size of each blob file. File will become immutable
  // after it exceeds that size
@@ -78,13 +76,6 @@ struct BlobDBOptions {
  // blob files will be cleanup based on TTL.
  bool enable_garbage_collection = false;

-  // Time interval to trigger garbage collection, in seconds.
-  uint64_t garbage_collection_interval_secs = 60;
-
-  // If garbage collection is enabled, blob files with deleted size no less
-  // than this ratio will become candidates to be cleanup.
-  double garbage_collection_deletion_size_threshold = 0.75;
-
  // Disable all background job. Used for test only.
  bool disable_background_tasks = false;

@@ -212,8 +203,6 @@ class BlobDB : public StackableDB {

  virtual BlobDBOptions GetBlobDBOptions() const = 0;

-  virtual Status SyncBlobFiles() = 0;
-
  virtual ~BlobDB() {}

 protected:
@@ -111,6 +111,7 @@ BlobDBImpl::BlobDBImpl(const std::string& dbname,
      cf_options_(cf_options),
      env_options_(db_options),
      statistics_(db_options_.statistics.get()),
+      dir_change_(false),
      next_file_number_(1),
      epoch_of_(0),
      shutdown_(false),
@@ -123,7 +124,6 @@ BlobDBImpl::BlobDBImpl(const std::string& dbname,
  blob_dir_ = (bdb_options_.path_relative)
                  ? dbname + "/" + bdb_options_.blob_dir
                  : bdb_options_.blob_dir;
-  env_options_.bytes_per_sync = blob_db_options.bytes_per_sync;
 }

 BlobDBImpl::~BlobDBImpl() {
@@ -202,7 +202,6 @@ Status BlobDBImpl::Open(std::vector<ColumnFamilyHandle*>* handles) {
  }

  ROCKS_LOG_INFO(db_options_.info_log, "BlobDB pointer %p", this);
-  bdb_options_.Dump(db_options_.info_log.get());
  return s;
 }

@@ -211,8 +210,7 @@ void BlobDBImpl::StartBackgroundTasks() {
  tqueue_.add(
      kReclaimOpenFilesPeriodMillisecs,
      std::bind(&BlobDBImpl::ReclaimOpenFiles, this, std::placeholders::_1));
-  tqueue_.add(static_cast<int64_t>(
-                  bdb_options_.garbage_collection_interval_secs * 1000),
+  tqueue_.add(kGCCheckPeriodMillisecs,
              std::bind(&BlobDBImpl::RunGC, this, std::placeholders::_1));
  if (bdb_options_.enable_garbage_collection) {
    tqueue_.add(
@@ -227,6 +225,8 @@ void BlobDBImpl::StartBackgroundTasks() {
      std::bind(&BlobDBImpl::DeleteObsoleteFiles, this, std::placeholders::_1));
  tqueue_.add(kSanityCheckPeriodMillisecs,
              std::bind(&BlobDBImpl::SanityCheck, this, std::placeholders::_1));
+  tqueue_.add(kFSyncFilesPeriodMillisecs,
+              std::bind(&BlobDBImpl::FsyncFiles, this, std::placeholders::_1));
  tqueue_.add(
      kCheckSeqFilesPeriodMillisecs,
      std::bind(&BlobDBImpl::CheckSeqFiles, this, std::placeholders::_1));
@@ -472,6 +472,7 @@ std::shared_ptr<BlobFile> BlobDBImpl::SelectBlobFile() {
    return nullptr;
  }

+  dir_change_.store(true);
  blob_files_.insert(std::make_pair(bfile->BlobFileNumber(), bfile));
  open_non_ttl_file_ = bfile;
  return bfile;
@@ -546,6 +547,7 @@ std::shared_ptr<BlobFile> BlobDBImpl::SelectBlobFileTTL(uint64_t expiration) {
    return nullptr;
  }

+  dir_change_.store(true);
  blob_files_.insert(std::make_pair(bfile->BlobFileNumber(), bfile));
  open_ttl_files_.insert(bfile);
  epoch_of_++;
@@ -569,14 +571,18 @@ class BlobDBImpl::BlobInserter : public WriteBatch::Handler {
  const WriteOptions& options_;
  BlobDBImpl* blob_db_impl_;
  uint32_t default_cf_id_;
+  SequenceNumber sequence_;
  WriteBatch batch_;

 public:
  BlobInserter(const WriteOptions& options, BlobDBImpl* blob_db_impl,
-               uint32_t default_cf_id)
+               uint32_t default_cf_id, SequenceNumber seq)
      : options_(options),
        blob_db_impl_(blob_db_impl),
-        default_cf_id_(default_cf_id) {}
+        default_cf_id_(default_cf_id),
+        sequence_(seq) {}
+
+  SequenceNumber sequence() { return sequence_; }

  WriteBatch* batch() { return &batch_; }

@@ -591,7 +597,8 @@ class BlobDBImpl::BlobInserter : public WriteBatch::Handler {
    uint64_t expiration =
        blob_db_impl_->ExtractExpiration(key, value, &value_slice, &new_value);
    Status s = blob_db_impl_->PutBlobValue(options_, key, value_slice,
-                                           expiration, &batch_);
+                                           expiration, sequence_, &batch_);
+    sequence_++;
    return s;
  }

@@ -602,6 +609,7 @@ class BlobDBImpl::BlobInserter : public WriteBatch::Handler {
          "Blob DB doesn't support non-default column family.");
    }
    Status s = WriteBatchInternal::Delete(&batch_, column_family_id, key);
+    sequence_++;
    return s;
  }

@@ -613,6 +621,7 @@ class BlobDBImpl::BlobInserter : public WriteBatch::Handler {
    }
    Status s = WriteBatchInternal::DeleteRange(&batch_, column_family_id,
                                               begin_key, end_key);
+    sequence_++;
    return s;
  }

@@ -634,8 +643,12 @@ Status BlobDBImpl::Write(const WriteOptions& options, WriteBatch* updates) {
  RecordTick(statistics_, BLOB_DB_NUM_WRITE);
  uint32_t default_cf_id =
      reinterpret_cast<ColumnFamilyHandleImpl*>(DefaultColumnFamily())->GetID();
+  // TODO(yiwu): In case there are multiple writers the latest sequence would
+  // not be the actually sequence we are writting. Need to get the sequence
+  // from write batch after DB write instead.
+  SequenceNumber current_seq = GetLatestSequenceNumber() + 1;
  Status s;
-  BlobInserter blob_inserter(options, this, default_cf_id);
+  BlobInserter blob_inserter(options, this, default_cf_id, current_seq);
  {
    // Release write_mutex_ before DB write to avoid race condition with
    // flush begin listener, which also require write_mutex_ to sync
@@ -680,8 +693,6 @@ Status BlobDBImpl::Write(const WriteOptions& options, WriteBatch* updates) {

  if (bdb_options_.enable_garbage_collection) {
    // add deleted key to list of keys that have been deleted for book-keeping
-    SequenceNumber current_seq =
-        WriteBatchInternal::Sequence(blob_inserter.batch());
    DeleteBookkeeper delete_bookkeeper(this, current_seq);
    s = updates->Iterate(&delete_bookkeeper);
  }
@@ -750,7 +761,11 @@ Status BlobDBImpl::PutUntil(const WriteOptions& options, const Slice& key,
    // flush begin listener, which also require write_mutex_ to sync
    // blob files.
    MutexLock l(&write_mutex_);
-    s = PutBlobValue(options, key, value, expiration, &batch);
+    // TODO(yiwu): In case there are multiple writers the latest sequence would
+    // not be the actually sequence we are writting. Need to get the sequence
+    // from write batch after DB write instead.
+    SequenceNumber sequence = GetLatestSequenceNumber() + 1;
+    s = PutBlobValue(options, key, value, expiration, sequence, &batch);
  }
  if (s.ok()) {
    s = db_->Write(options, &batch);
@@ -761,7 +776,7 @@ Status BlobDBImpl::PutUntil(const WriteOptions& options, const Slice& key,

 Status BlobDBImpl::PutBlobValue(const WriteOptions& options, const Slice& key,
                                const Slice& value, uint64_t expiration,
-                                WriteBatch* batch) {
+                                SequenceNumber sequence, WriteBatch* batch) {
  Status s;
  std::string index_entry;
  uint32_t column_family_id =
@@ -802,6 +817,7 @@ Status BlobDBImpl::PutBlobValue(const WriteOptions& options, const Slice& key,
    }

    if (s.ok()) {
+      bfile->ExtendSequenceRange(sequence);
      if (expiration != kNoExpiration) {
        bfile->ExtendExpirationRange(expiration);
      }
@@ -860,9 +876,6 @@ std::shared_ptr<BlobFile> BlobDBImpl::GetOldestBlobFile() {
  CopyBlobFiles(&blob_files, [](const std::shared_ptr<BlobFile>& f) {
    return !f->Obsolete() && f->Immutable();
  });
-  if (blob_files.empty()) {
-    return nullptr;
-  }
  blobf_compare_ttl compare;
  return *std::min_element(blob_files.begin(), blob_files.end(), compare);
 }
@@ -885,7 +898,7 @@ bool BlobDBImpl::EvictOldestBlobFile() {
                   total_blob_space_.load(), bdb_options_.blob_dir_size,
                   oldest_file->BlobFileNumber(), expiration_range.first,
                   expiration_range.second);
-    oldest_file->MarkObsolete(GetLatestSequenceNumber());
+    oldest_file->MarkObsolete(oldest_file->GetSequenceRange().second);
    obsolete_files_.push_back(oldest_file);
    oldest_file_evicted_.store(true);
    RecordTick(statistics_, BLOB_DB_FIFO_NUM_FILES_EVICTED);
@@ -893,7 +906,6 @@ bool BlobDBImpl::EvictOldestBlobFile() {
               oldest_file->BlobCount());
    RecordTick(statistics_, BLOB_DB_FIFO_BYTES_EVICTED,
               oldest_file->GetFileSize());
-    TEST_SYNC_POINT("BlobDBImpl::EvictOldestBlobFile:Evicted");
    return true;
  }

@@ -1060,58 +1072,58 @@ Status BlobDBImpl::GetBlobValue(const Slice& key, const Slice& index_entry,
  std::shared_ptr<RandomAccessFileReader> reader =
      GetOrOpenRandomAccessReader(bfile, env_, env_options_);

-  assert(blob_index.offset() > key.size() + sizeof(uint32_t));
-  uint64_t record_offset = blob_index.offset() - key.size() - sizeof(uint32_t);
-  uint64_t record_size = sizeof(uint32_t) + key.size() + blob_index.size();
+  std::string* valueptr = value->GetSelf();
+  std::string value_c;
+  if (bdb_options_.compression != kNoCompression) {
+    valueptr = &value_c;
+  }

  // Allocate the buffer. This is safe in C++11
-  std::string buffer_str(record_size, static_cast<char>(0));
-  char* buffer = &buffer_str[0];
+  // Note that std::string::reserved() does not work, since previous value
+  // of the buffer can be larger than blob_index.size().
+  valueptr->resize(blob_index.size());
+  char* buffer = &(*valueptr)[0];

-  // A partial blob record contain checksum, key and value.
-  Slice blob_record;
+  Slice blob_value;
  {
    StopWatch read_sw(env_, statistics_, BLOB_DB_BLOB_FILE_READ_MICROS);
-    s = reader->Read(record_offset, record_size, &blob_record, buffer);
-    RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, blob_record.size());
+    s = reader->Read(blob_index.offset(), blob_index.size(), &blob_value,
+                     buffer);
+    RecordTick(statistics_, BLOB_DB_BLOB_FILE_BYTES_READ, blob_value.size());
  }
-  if (!s.ok()) {
-    ROCKS_LOG_DEBUG(db_options_.info_log,
-                    "Failed to read blob from blob file %" PRIu64
-                    ", blob_offset: %" PRIu64 ", blob_size: %" PRIu64
-                    ", key_size: " PRIu64 ", read " PRIu64
-                    "bytes, status: '%s'",
-                    bfile->BlobFileNumber(), blob_index.offset(),
-                    blob_index.size(), key.size(), s.ToString().c_str());
-    return s;
+  if (!s.ok() || blob_value.size() != blob_index.size()) {
+    if (debug_level_ >= 2) {
+      ROCKS_LOG_ERROR(db_options_.info_log,
+                      "Failed to read blob from file: %s blob_offset: %" PRIu64
+                      " blob_size: %" PRIu64 " read: %d key: %s status: '%s'",
+                      bfile->PathName().c_str(), blob_index.offset(),
+                      blob_index.size(), static_cast<int>(blob_value.size()),
+                      key.data(), s.ToString().c_str());
+    }
+    return Status::NotFound("Blob Not Found as couldnt retrieve Blob");
  }
-  if (blob_record.size() != record_size) {
-    ROCKS_LOG_DEBUG(db_options_.info_log,
-                    "Failed to read blob from blob file %" PRIu64
-                    ", blob_offset: %" PRIu64 ", blob_size: %" PRIu64
-                    ", key_size: " PRIu64 ", read " PRIu64
-                    "bytes, status: '%s'",
-                    bfile->BlobFileNumber(), blob_index.offset(),
-                    blob_index.size(), key.size(), s.ToString().c_str());

-    return Status::Corruption("Failed to retrieve blob from blob index.");
-  }
-  Slice crc_slice(blob_record.data(), sizeof(uint32_t));
-  Slice blob_value(blob_record.data() + sizeof(uint32_t) + key.size(),
-                   blob_index.size());
+  // TODO(yiwu): Add an option to skip crc checking.
+  Slice crc_slice;
  uint32_t crc_exp;
-  if (!GetFixed32(&crc_slice, &crc_exp)) {
-    ROCKS_LOG_DEBUG(db_options_.info_log,
-                    "Unable to decode CRC from blob file %" PRIu64
-                    ", blob_offset: %" PRIu64 ", blob_size: %" PRIu64
-                    ", key size: %" PRIu64 ", status: '%s'",
-                    bfile->BlobFileNumber(), blob_index.offset(),
-                    blob_index.size(), key.size(), s.ToString().c_str());
-    return Status::Corruption("Unable to decode checksum.");
+  std::string crc_str;
+  crc_str.resize(sizeof(uint32_t));
+  char* crc_buffer = &(crc_str[0]);
+  s = reader->Read(blob_index.offset() - (key.size() + sizeof(uint32_t)),
+                   sizeof(uint32_t), &crc_slice, crc_buffer);
+  if (!s.ok() || !GetFixed32(&crc_slice, &crc_exp)) {
+    if (debug_level_ >= 2) {
+      ROCKS_LOG_ERROR(db_options_.info_log,
+                      "Failed to fetch blob crc file: %s blob_offset: %" PRIu64
+                      " blob_size: %" PRIu64 " key: %s status: '%s'",
+                      bfile->PathName().c_str(), blob_index.offset(),
+                      blob_index.size(), key.data(), s.ToString().c_str());
+    }
+    return Status::NotFound("Blob Not Found as couldnt retrieve CRC");
  }

-  uint32_t crc = crc32c::Value(blob_record.data() + sizeof(uint32_t),
-                               blob_record.size() - sizeof(uint32_t));
+  uint32_t crc = crc32c::Value(key.data(), key.size());
+  crc = crc32c::Extend(crc, blob_value.data(), blob_value.size());
  crc = crc32c::Mask(crc);  // Adjust for storage
  if (crc != crc_exp) {
    if (debug_level_ >= 2) {
@@ -1124,9 +1136,7 @@ Status BlobDBImpl::GetBlobValue(const Slice& key, const Slice& index_entry,
    return Status::Corruption("Corruption. Blob CRC mismatch");
  }

-  if (bfile->compression() == kNoCompression) {
-    value->PinSelf(blob_value);
-  } else {
+  if (bfile->compression() != kNoCompression) {
    BlockContents contents;
    auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(DefaultColumnFamily());
    {
@@ -1137,9 +1147,11 @@ Status BlobDBImpl::GetBlobValue(const Slice& key, const Slice& index_entry,
          kBlockBasedTableVersionFormat, Slice(), bfile->compression(),
          *(cfh->cfd()->ioptions()));
    }
-    value->PinSelf(contents.data);
+    *(value->GetSelf()) = contents.data.ToString();
  }

+  value->PinSelf();
+
  return s;
 }

@@ -1259,26 +1271,9 @@ Status BlobDBImpl::CloseBlobFileIfNeeded(std::shared_ptr<BlobFile>& bfile) {
 bool BlobDBImpl::VisibleToActiveSnapshot(
    const std::shared_ptr<BlobFile>& bfile) {
  assert(bfile->Obsolete());
-
-  // We check whether the oldest snapshot is no less than the last sequence
-  // by the time the blob file become obsolete. If so, the blob file is not
-  // visible to all existing snapshots.
-  //
-  // If we keep track of the earliest sequence of the keys in the blob file,
-  // we could instead check if there's a snapshot falls in range
-  // [earliest_sequence, obsolete_sequence). But doing so will make the
-  // implementation more complicated.
+  SequenceNumber first_sequence = bfile->GetSequenceRange().first;
  SequenceNumber obsolete_sequence = bfile->GetObsoleteSequence();
-  SequenceNumber oldest_snapshot = 0;
-  {
-    // Need to lock DBImpl mutex before access snapshot list.
-    InstrumentedMutexLock l(db_impl_->mutex());
-    auto snapshots = db_impl_->snapshots();
-    if (!snapshots.empty()) {
-      oldest_snapshot = snapshots.oldest()->GetSequenceNumber();
-    }
-  }
-  return oldest_snapshot < obsolete_sequence;
+  return db_impl_->HasActiveSnapshotInRange(first_sequence, obsolete_sequence);
 }

 bool BlobDBImpl::FindFileAndEvictABlob(uint64_t file_number, uint64_t key_size,
@@ -1436,6 +1431,14 @@ std::pair<bool, int64_t> BlobDBImpl::CheckSeqFiles(bool aborted) {
  return std::make_pair(true, -1);
 }

+std::pair<bool, int64_t> BlobDBImpl::FsyncFiles(bool aborted) {
+  if (aborted || shutdown_) {
+    return std::make_pair(false, -1);
+  }
+  SyncBlobFiles();
+  return std::make_pair(true, -1);
+}
+
 Status BlobDBImpl::SyncBlobFiles() {
  MutexLock l(&write_mutex_);

@@ -1443,30 +1446,35 @@ Status BlobDBImpl::SyncBlobFiles() {
  {
    ReadLock rl(&mutex_);
    for (auto fitr : open_ttl_files_) {
-      process_files.push_back(fitr);
+      if (fitr->NeedsFsync(true, bdb_options_.bytes_per_sync))
+        process_files.push_back(fitr);
    }
-    if (open_non_ttl_file_ != nullptr) {
+
+    if (open_non_ttl_file_ != nullptr &&
+        open_non_ttl_file_->NeedsFsync(true, bdb_options_.bytes_per_sync)) {
      process_files.push_back(open_non_ttl_file_);
    }
  }

  Status s;
+
  for (auto& blob_file : process_files) {
-    s = blob_file->Fsync();
-    if (!s.ok()) {
-      ROCKS_LOG_ERROR(db_options_.info_log,
-                      "Failed to sync blob file %" PRIu64 ", status: %s",
-                      blob_file->BlobFileNumber(), s.ToString().c_str());
-      return s;
+    if (blob_file->NeedsFsync(true, bdb_options_.bytes_per_sync)) {
+      s = blob_file->Fsync();
+      if (!s.ok()) {
+        ROCKS_LOG_ERROR(db_options_.info_log,
+                        "Failed to sync blob file %" PRIu64 ", status: %s",
+                        blob_file->BlobFileNumber(), s.ToString().c_str());
+        return s;
+      }
    }
  }

-  s = dir_ent_->Fsync();
-  if (!s.ok()) {
-    ROCKS_LOG_ERROR(db_options_.info_log,
-                    "Failed to sync blob directory, status: %s",
-                    s.ToString().c_str());
+  bool expected = true;
+  if (dir_change_.compare_exchange_weak(expected, false)) {
+    s = dir_ent_->Fsync();
  }
+
  return s;
 }

@@ -1721,6 +1729,7 @@ Status BlobDBImpl::GCFileAndUpdateLSM(const std::shared_ptr<BlobFile>& bfptr,

      WriteLock wl(&mutex_);

+      dir_change_.store(true);
      blob_files_.insert(std::make_pair(newfile->BlobFileNumber(), newfile));
    }

@@ -1748,6 +1757,8 @@ Status BlobDBImpl::GCFileAndUpdateLSM(const std::shared_ptr<BlobFile>& bfptr,
                                                   &rewrite_batch, &callback);
    }
    if (rewrite_status.ok()) {
+      newfile->ExtendSequenceRange(
+          WriteBatchInternal::Sequence(&rewrite_batch));
      gc_stats->num_keys_relocated++;
      gc_stats->bytes_relocated += record.record_size();
    } else if (rewrite_status.IsBusy()) {
@@ -1764,7 +1775,10 @@ Status BlobDBImpl::GCFileAndUpdateLSM(const std::shared_ptr<BlobFile>& bfptr,
  }  // end of ReadRecord loop

  if (s.ok()) {
-    bfptr->MarkObsolete(GetLatestSequenceNumber());
+    SequenceNumber obsolete_sequence =
+        newfile == nullptr ? bfptr->GetSequenceRange().second + 1
+                           : newfile->GetSequenceRange().second;
+    bfptr->MarkObsolete(obsolete_sequence);
    if (!first_gc) {
      WriteLock wl(&mutex_);
      obsolete_files_.push_back(bfptr);
@@ -1774,9 +1788,8 @@ Status BlobDBImpl::GCFileAndUpdateLSM(const std::shared_ptr<BlobFile>& bfptr,
  ROCKS_LOG_INFO(
      db_options_.info_log,
      "%s blob file %" PRIu64 ". Total blob records: %" PRIu64
-      ", expired: %" PRIu64 " keys/%" PRIu64
-      " bytes, updated or deleted by user: %" PRIu64 " keys/%" PRIu64
-      " bytes, rewrite to new file: %" PRIu64 " keys/%" PRIu64 " bytes.",
+      ", Expired: %" PRIu64 " keys/%" PRIu64 " bytes, Overwritten: %" PRIu64
+      " keys/%" PRIu64 " bytes.",
      s.ok() ? "Successfully garbage collected" : "Failed to garbage collect",
      bfptr->BlobFileNumber(), gc_stats->blob_count, gc_stats->num_keys_expired,
      gc_stats->bytes_expired, gc_stats->num_keys_overwritten,
@@ -1830,9 +1843,14 @@ bool BlobDBImpl::ShouldGCFile(std::shared_ptr<BlobFile> bfile, uint64_t now,
      return true;
    }

+    if (bdb_options_.ttl_range_secs < kPartialExpirationGCRangeSecs) {
+      *reason = "has ttl but partial expiration not turned on";
+      return false;
+    }
+
    ReadLock lockbfile_r(&bfile->mutex_);
-    bool ret = ((bfile->deleted_size_ / bfile->file_size_.load()) >
-                bdb_options_.garbage_collection_deletion_size_threshold);
+    bool ret = ((bfile->deleted_size_ * 100.0 / bfile->file_size_.load()) >
+                kPartialExpirationPercentage);
    if (ret) {
      *reason = "deleted blobs beyond threshold";
    } else {
@@ -1851,8 +1869,8 @@ bool BlobDBImpl::ShouldGCFile(std::shared_ptr<BlobFile> bfile, uint64_t now,
  ReadLock lockbfile_r(&bfile->mutex_);

  if (bdb_options_.enable_garbage_collection) {
-    if ((bfile->deleted_size_ / bfile->file_size_.load()) >
-        bdb_options_.garbage_collection_deletion_size_threshold) {
+    if ((bfile->deleted_size_ * 100.0 / bfile->file_size_.load()) >
+        kPartialExpirationPercentage) {
      *reason = "deleted simple blobs beyond threshold";
      return true;
    }
@@ -144,6 +144,23 @@ class BlobDBImpl : public BlobDB {
  // how many random access open files can we tolerate
  static constexpr uint32_t kOpenFilesTrigger = 100;

+  // how many periods of stats do we keep.
+  static constexpr uint32_t kWriteAmplificationStatsPeriods = 24;
+
+  // we will garbage collect blob files in
+  // which entire files have expired. However if the
+  // ttl_range of files is very large say a day, we
+  // would have to wait for the entire day, before we
+  // recover most of the space.
+  static constexpr uint32_t kPartialExpirationGCRangeSecs = 4 * 3600;
+
+  // this should be based on allowed Write Amplification
+  // if 50% of the space of a blob file has been deleted/expired,
+  static constexpr uint32_t kPartialExpirationPercentage = 75;
+
+  // how often should we schedule a job to fsync open files
+  static constexpr uint32_t kFSyncFilesPeriodMillisecs = 10 * 1000;
+
  // how often to schedule reclaim open files.
  static constexpr uint32_t kReclaimOpenFilesPeriodMillisecs = 1 * 1000;

@@ -211,7 +228,7 @@ class BlobDBImpl : public BlobDB {

  Status Open(std::vector<ColumnFamilyHandle*>* handles);

-  Status SyncBlobFiles() override;
+  Status SyncBlobFiles();

 #ifndef NDEBUG
  Status TEST_GetBlobValue(const Slice& key, const Slice& index_entry,
@@ -267,7 +284,7 @@ class BlobDBImpl : public BlobDB {

  Status PutBlobValue(const WriteOptions& options, const Slice& key,
                      const Slice& value, uint64_t expiration,
-                      WriteBatch* batch);
+                      SequenceNumber sequence, WriteBatch* batch);

  Status AppendBlob(const std::shared_ptr<BlobFile>& bfile,
                    const std::string& headerbuf, const Slice& key,
@@ -296,6 +313,9 @@ class BlobDBImpl : public BlobDB {
  // Major task to garbage collect expired and deleted blobs
  std::pair<bool, int64_t> RunGC(bool aborted);

+  // asynchronous task to fsync/fdatasync the open blob files
+  std::pair<bool, int64_t> FsyncFiles(bool aborted);
+
  // periodically check if open blob files and their TTL's has expired
  // if expired, close the sequential writer and make the file immutable
  std::pair<bool, int64_t> CheckSeqFiles(bool aborted);
@@ -400,6 +420,8 @@ class BlobDBImpl : public BlobDB {
  // pointer to directory
  std::unique_ptr<Directory> dir_ent_;

+  std::atomic<bool> dir_change_;
+
  // Read Write Mutex, which protects all the data structures
  // HEAVILY TRAFFICKED
  mutable port::RWMutex mutex_;
@@ -901,18 +901,20 @@ TEST_F(BlobDBTest, SnapshotAndGarbageCollection) {
      ASSERT_EQ(1, gc_stats.blob_count);
      if (delete_key) {
        ASSERT_EQ(0, gc_stats.num_keys_relocated);
+        ASSERT_EQ(bfile->GetSequenceRange().second + 1,
+                  bfile->GetObsoleteSequence());
      } else {
        ASSERT_EQ(1, gc_stats.num_keys_relocated);
+        ASSERT_EQ(blob_db_->GetLatestSequenceNumber(),
+                  bfile->GetObsoleteSequence());
      }
-      ASSERT_EQ(blob_db_->GetLatestSequenceNumber(),
-                bfile->GetObsoleteSequence());
      if (i == 3) {
        snapshot = blob_db_->GetSnapshot();
      }
      size_t num_files = delete_key ? 3 : 4;
      ASSERT_EQ(num_files, blob_db_impl()->TEST_GetBlobFiles().size());
      blob_db_impl()->TEST_DeleteObsoleteFiles();
-      if (i == 3) {
+      if (i == 0 || i == 3 || (i == 2 && delete_key)) {
        // The snapshot shouldn't see data in bfile
        ASSERT_EQ(num_files - 1, blob_db_impl()->TEST_GetBlobFiles().size());
        blob_db_->ReleaseSnapshot(snapshot);
@@ -1061,19 +1063,15 @@ TEST_F(BlobDBTest, OutOfSpace) {
 }

 TEST_F(BlobDBTest, EvictOldestFileWhenCloseToSpaceLimit) {
+  // Use mock env to stop wall clock.
+  Options options;
  BlobDBOptions bdb_options;
  bdb_options.blob_dir_size = 270;
  bdb_options.blob_file_size = 100;
-  bdb_options.is_fifo = true;
  bdb_options.disable_background_tasks = true;
+  bdb_options.is_fifo = true;
  Open(bdb_options);

-  std::atomic<int> evict_count{0};
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobDBImpl::EvictOldestBlobFile:Evicted",
-      [&](void *) { evict_count++; });
-  SyncPoint::GetInstance()->EnableProcessing();
-
  // Each stored blob has an overhead of 32 bytes currently.
  // So a 100 byte blob should take up 132 bytes.
  std::string value(100, 'v');
@@ -1097,28 +1095,6 @@ TEST_F(BlobDBTest, EvictOldestFileWhenCloseToSpaceLimit) {
  bdb_impl->TEST_DeleteObsoleteFiles();
  obsolete_files = bdb_impl->TEST_GetObsoleteFiles();
  ASSERT_TRUE(obsolete_files.empty());
-  ASSERT_EQ(1, evict_count);
-}
-
-TEST_F(BlobDBTest, NoOldestFileToEvict) {
-  Options options;
-  BlobDBOptions bdb_options;
-  bdb_options.blob_dir_size = 1000;
-  bdb_options.blob_file_size = 5000;
-  bdb_options.is_fifo = true;
-  bdb_options.disable_background_tasks = true;
-  Open(bdb_options);
-
-  std::atomic<int> evict_count{0};
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobDBImpl::EvictOldestBlobFile:Evicted",
-      [&](void *) { evict_count++; });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  std::string value(2000, 'v');
-  ASSERT_OK(Put("foo", std::string(2000, 'v')));
-  ASSERT_OK(Put("bar", std::string(2000, 'v')));
-  ASSERT_EQ(0, evict_count);
 }

 TEST_F(BlobDBTest, InlineSmallValues) {
@@ -1135,6 +1111,10 @@ TEST_F(BlobDBTest, InlineSmallValues) {
  Open(bdb_options, options);
  std::map<std::string, std::string> data;
  std::map<std::string, KeyVersion> versions;
+  SequenceNumber first_non_ttl_seq = kMaxSequenceNumber;
+  SequenceNumber first_ttl_seq = kMaxSequenceNumber;
+  SequenceNumber last_non_ttl_seq = 0;
+  SequenceNumber last_ttl_seq = 0;
  for (size_t i = 0; i < 1000; i++) {
    bool is_small_value = rnd.Next() % 2;
    bool has_ttl = rnd.Next() % 2;
@@ -1154,6 +1134,15 @@ TEST_F(BlobDBTest, InlineSmallValues) {
    versions[key] =
        KeyVersion(key, value, sequence,
                   (is_small_value && !has_ttl) ? kTypeValue : kTypeBlobIndex);
+    if (!is_small_value) {
+      if (!has_ttl) {
+        first_non_ttl_seq = std::min(first_non_ttl_seq, sequence);
+        last_non_ttl_seq = std::max(last_non_ttl_seq, sequence);
+      } else {
+        first_ttl_seq = std::min(first_ttl_seq, sequence);
+        last_ttl_seq = std::max(last_ttl_seq, sequence);
+      }
+    }
  }
  VerifyDB(data);
  VerifyBaseDB(versions);
@@ -1170,7 +1159,11 @@ TEST_F(BlobDBTest, InlineSmallValues) {
    ttl_file = blob_files[1];
  }
  ASSERT_FALSE(non_ttl_file->HasTTL());
+  ASSERT_EQ(first_non_ttl_seq, non_ttl_file->GetSequenceRange().first);
+  ASSERT_EQ(last_non_ttl_seq, non_ttl_file->GetSequenceRange().second);
  ASSERT_TRUE(ttl_file->HasTTL());
+  ASSERT_EQ(first_ttl_seq, ttl_file->GetSequenceRange().first);
+  ASSERT_EQ(last_ttl_seq, ttl_file->GetSequenceRange().second);
 }

 TEST_F(BlobDBTest, CompactionFilterNotSupported) {
@@ -142,6 +142,8 @@ Status BlobDumpTool::DumpBlobLogFooter(uint64_t file_size,
  fprintf(stdout, "  Blob count       : %" PRIu64 "\n", footer.blob_count);
  fprintf(stdout, "  Expiration Range : %s\n",
          GetString(footer.expiration_range).c_str());
+  fprintf(stdout, "  Sequence Range   : %s\n",
+          GetString(footer.sequence_range).c_str());
  return s;
 }

@@ -43,6 +43,7 @@ BlobFile::BlobFile()
      obsolete_(false),
      gc_once_after_open_(false),
      expiration_range_({0, 0}),
+      sequence_range_({kMaxSequenceNumber, 0}),
      last_access_(-1),
      last_fsync_(0),
      header_valid_(false),
@@ -66,6 +67,7 @@ BlobFile::BlobFile(const BlobDBImpl* p, const std::string& bdir, uint64_t fn,
      obsolete_(false),
      gc_once_after_open_(false),
      expiration_range_({0, 0}),
+      sequence_range_({kMaxSequenceNumber, 0}),
      last_access_(-1),
      last_fsync_(0),
      header_valid_(false),
@@ -114,11 +116,12 @@ std::string BlobFile::DumpState() const {
           " file_size: %" PRIu64 " deleted_count: %" PRIu64
           " deleted_size: %" PRIu64
           " closed: %d obsolete: %d expiration_range: (%" PRIu64 ", %" PRIu64
-           "), writer: %d reader: %d",
+           ") sequence_range: (%" PRIu64 " %" PRIu64 "), writer: %d reader: %d",
           path_to_dir_.c_str(), file_number_, blob_count_.load(),
           gc_epoch_.load(), file_size_.load(), deleted_count_, deleted_size_,
           closed_.load(), obsolete_.load(), expiration_range_.first,
-           expiration_range_.second, (!!log_writer_), (!!ra_file_reader_));
+           expiration_range_.second, sequence_range_.first,
+           sequence_range_.second, (!!log_writer_), (!!ra_file_reader_));
  return str;
 }

@@ -141,6 +144,8 @@ Status BlobFile::WriteFooterAndCloseLocked() {
    footer.expiration_range = expiration_range_;
  }

+  footer.sequence_range = sequence_range_;
+
  // this will close the file and reset the Writable File Pointer.
  Status s = log_writer_->AppendFooter(footer);
  if (s.ok()) {
@@ -180,6 +185,7 @@ Status BlobFile::SetFromFooterLocked(const BlobLogFooter& footer) {
  last_fsync_.store(file_size_);
  blob_count_ = footer.blob_count;
  expiration_range_ = footer.expiration_range;
+  sequence_range_ = footer.sequence_range;
  closed_ = true;
  return Status::OK();
 }
@@ -84,6 +84,8 @@ class BlobFile {

  ExpirationRange expiration_range_;

+  SequenceRange sequence_range_;
+
  // Sequential/Append writer for blobs
  std::shared_ptr<Writer> log_writer_;

@@ -175,6 +177,17 @@ class BlobFile {
    expiration_range_.second = std::max(expiration_range_.second, expiration);
  }

+  SequenceRange GetSequenceRange() const { return sequence_range_; }
+
+  void SetSequenceRange(SequenceRange sequence_range) {
+    sequence_range_ = sequence_range;
+  }
+
+  void ExtendSequenceRange(SequenceNumber sequence) {
+    sequence_range_.first = std::min(sequence_range_.first, sequence);
+    sequence_range_.second = std::max(sequence_range_.second, sequence);
+  }
+
  bool HasTTL() const { return has_ttl_; }

  void SetHasTTL(bool has_ttl) { has_ttl_ = has_ttl; }
@@ -67,6 +67,8 @@ void BlobLogFooter::EncodeTo(std::string* dst) {
  PutFixed64(dst, blob_count);
  PutFixed64(dst, expiration_range.first);
  PutFixed64(dst, expiration_range.second);
+  PutFixed64(dst, sequence_range.first);
+  PutFixed64(dst, sequence_range.second);
  crc = crc32c::Value(dst->c_str(), dst->size());
  crc = crc32c::Mask(crc);
  PutFixed32(dst, crc);
@@ -80,12 +82,14 @@ Status BlobLogFooter::DecodeFrom(Slice src) {
                              "Unexpected blob file footer size");
  }
  uint32_t src_crc = 0;
-  src_crc = crc32c::Value(src.data(), BlobLogFooter::kSize - sizeof(uint32_t));
+  src_crc = crc32c::Value(src.data(), BlobLogFooter::kSize - 4);
  src_crc = crc32c::Mask(src_crc);
  uint32_t magic_number;
  if (!GetFixed32(&src, &magic_number) || !GetFixed64(&src, &blob_count) ||
      !GetFixed64(&src, &expiration_range.first) ||
-      !GetFixed64(&src, &expiration_range.second) || !GetFixed32(&src, &crc)) {
+      !GetFixed64(&src, &expiration_range.second) ||
+      !GetFixed64(&src, &sequence_range.first) ||
+      !GetFixed64(&src, &sequence_range.second) || !GetFixed32(&src, &crc)) {
    return Status::Corruption(kErrorMessage, "Error decoding content");
  }
  if (magic_number != kMagicNumber) {
@@ -24,6 +24,7 @@ constexpr uint32_t kVersion1 = 1;
 constexpr uint64_t kNoExpiration = std::numeric_limits<uint64_t>::max();

 using ExpirationRange = std::pair<uint64_t, uint64_t>;
+using SequenceRange = std::pair<uint64_t, uint64_t>;

 // Format of blob log file header (30 bytes):
 //
@@ -52,23 +53,24 @@ struct BlobLogHeader {
  Status DecodeFrom(Slice slice);
 };

-// Format of blob log file footer (32 bytes):
+// Format of blob log file footer (48 bytes):
 //
-//    +--------------+------------+-------------------+------------+
-//    | magic number | blob count | expiration range  | footer CRC |
-//    +--------------+------------+-------------------+------------+
-//    |   Fixed32    |  Fixed64   | Fixed64 + Fixed64 |   Fixed32  |
-//    +--------------+------------+-------------------+------------+
+//    +--------------+------------+-------------------+-------------------+------------+
+//    | magic number | blob count | expiration range  |  sequence range   | footer CRC |
+//    +--------------+------------+-------------------+-------------------+------------+
+//    |   Fixed32    |  Fixed64   | Fixed64 + Fixed64 | Fixed64 + Fixed64 |   Fixed32  |
+//    +--------------+------------+-------------------+-------------------+------------+
 //
 // The footer will be presented only when the blob file is properly closed.
 //
 // Unlike the same field in file header, expiration range in the footer is the
 // range of smallest and largest expiration of the data in this file.
 struct BlobLogFooter {
-  static constexpr size_t kSize = 32;
+  static constexpr size_t kSize = 48;

  uint64_t blob_count = 0;
  ExpirationRange expiration_range = std::make_pair(0, 0);
+  SequenceRange sequence_range = std::make_pair(0, 0);
  uint32_t crc = 0;

  void EncodeTo(std::string* dst);
@@ -207,7 +207,9 @@ Status CheckpointImpl::CreateCustomCheckpoint(

    TEST_SYNC_POINT("CheckpointImpl::CreateCheckpoint:SavedLiveFiles1");
    TEST_SYNC_POINT("CheckpointImpl::CreateCheckpoint:SavedLiveFiles2");
-    db_->FlushWAL(false /* sync */);
+    if (db_options.manual_wal_flush) {
+      db_->FlushWAL(false /* sync */);
+    }
  }
  // if we have more than one column family, we need to also get WAL files
  if (s.ok()) {
@@ -1485,4 +1485,5 @@ EnvLibrados* EnvLibrados::Default() {
                                 default_pool_name);
  return &default_env;
 }
+// @lint-ignore TXT4 T25377293 Grandfathered in
 }
@@ -89,6 +89,7 @@ class CacheActivityLogger {
    log_line += key.ToString(true);
    log_line += " - ";
    AppendNumberTo(&log_line, size);
+  // @lint-ignore TXT2 T25377293 Grandfathered in
 		log_line += "\n";

    // line format: "ADD - <KEY> - <KEY-SIZE>"
@@ -193,6 +193,7 @@ TEST_F(SimCacheTest, SimCacheLogging) {
  ASSERT_EQ(add_num, num_block_entries);

  // Log things again but stop logging automatically after reaching 512 bytes
+ // @lint-ignore TXT2 T25377293 Grandfathered in
 	int max_size = 512;
  ASSERT_OK(sim_cache->StartActivityLogging(log_file, env_, max_size));
  for (int it = 0; it < 10; it++) {
Author	SHA1	Message	Date
przemyslaw.skibinski@percona.com	5d5e8110ff	Fix GitHub issue #3716 : gcc-8 warnings Summary: Fix the following gcc-8 warnings: - conflicting C language linkage declaration [-Werror] - writing to an object with no trivial copy-assignment [-Werror=class-memaccess] - array subscript -1 is below array bounds [-Werror=array-bounds] Solves https://github.com/facebook/rocksdb/issues/3716 Closes https://github.com/facebook/rocksdb/pull/3736 Differential Revision: D7684161 Pulled By: yiwu-arbug fbshipit-source-id: 47c0423d26b74add251f1d3595211eee1e41e54a	2019-10-30 13:30:56 -07:00
Vijay Nadimpalli	448d0096fe	Making platform 007 (gcc 7) default in build_detect_platform.sh (#5947 ) Summary: Making platform 007 (gcc 7) default in build_detect_platform.sh. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5947 Differential Revision: D18038837 Pulled By: vjnadimpalli fbshipit-source-id: 9ac2ddaa93bf328a416faec028970e039886378e	2019-10-30 13:30:56 -07:00
Andrew Kryczka	6da4b446be	Add latest toolchain (gcc-8, etc.) build support for fbcode users (#4923 ) Summary: - When building with internal dependencies, specify this toolchain by setting `ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1` - It is not enabled by default. However, it is enabled for TSAN builds in CI since there is a known problem with TSAN in gcc-5: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090 - I did not add support for Lua since (1) we agreed to deprecate it, and (2) we only have an internal build for v5.3 with this toolchain while that has breaking changes compared to our current version (v5.2). Pull Request resolved: https://github.com/facebook/rocksdb/pull/4923 Differential Revision: D13827226 Pulled By: ajkr fbshipit-source-id: 9aa3388ed3679777cfb15ef8cbcb83c07f62f947	2019-10-30 13:30:56 -07:00
Andrew Kryczka	808e429bd6	update history and bump version to 5.10.5	2018-03-16 14:22:42 -07:00
Andrew Kryczka	8b99935452	Fix WAL corruption from checkpoint/backup race condition Summary: `Writer::WriteBuffer` was always called at the beginning of checkpoint/backup. But that log writer has no internal synchronization, which meant the same buffer could be flushed twice in a race condition case, causing a WAL entry to be duplicated. Then subsequent WAL entries would be at unexpected offsets, causing the 32KB block boundaries to be overlapped and manifesting as a corruption. This PR fixes the behavior to only use `WriteBuffer` (via `FlushWAL`) in checkpoint/backup when manual WAL flush is enabled. In that case, users are responsible for providing synchronization between WAL flushes. We can also consider removing the call entirely. Closes https://github.com/facebook/rocksdb/pull/3603 Differential Revision: D7277447 Pulled By: ajkr fbshipit-source-id: 1b15bd7fd930511222b075418c10de0aaa70a35a	2018-03-16 14:22:05 -07:00
sdong	492ab7c7d9	Update HISTORY.md	2018-03-01 11:16:05 -08:00
Adam Retter	0d4fb0ef1e	Fixes the build on Windows Summary: As discovered during v5.9.2 release, and forward-ported. Closes https://github.com/facebook/rocksdb/pull/3323 Differential Revision: D6657209 Pulled By: siying fbshipit-source-id: b560d9f8ddb89e0ffaff7c895ec80f68ddf7dab4	2018-02-26 16:10:58 -08:00
Andrew Kryczka	5b0481ef49	update history and bump version	2018-02-22 13:58:31 -08:00
Andrew Kryczka	2f078d57e7	BackupEngine gluster-friendly file naming convention Summary: Use the rsync tempfile naming convention in our `BackupEngine`. The temp file follows the format, `.<filename>.<suffix>`, which is later renamed to `<filename>`. We fix `tmp` as the `<suffix>` as we don't need to use random bytes for now. The benefit is gluster treats this tempfile naming convention specially and applies hashing only to `<filename>`, so the file won't need to be linked or moved when it's renamed. Our gluster team suggested this will make things operationally easier. Closes https://github.com/facebook/rocksdb/pull/3463 Differential Revision: D6893333 Pulled By: ajkr fbshipit-source-id: fd7622978f4b2487fce33cde40dd3124f16bcaa8	2018-02-22 13:53:42 -08:00
sdong	ac329cce6e	Update HISTORY.md	2018-02-21 15:12:10 -08:00
Sagar Vemuri	3300ca0f39	Add rocksdb.iterator.internal-key property Summary: Added a new iterator property: `rocksdb.iterator.internal-key` to get the internal-key (converted to user key) at which the iterator stopped. Closes https://github.com/facebook/rocksdb/pull/3525 Differential Revision: D7033694 Pulled By: sagar0 fbshipit-source-id: d51e6c00f5e9d766c6276ef79774b81c6c5216f8	2018-02-21 10:20:59 -08:00
Sagar Vemuri	aa3b8bb460	Bump version to 5.10.3	2018-02-16 14:18:14 -08:00
Mike Kolupaev	0493719649	Fix deadlock in ColumnFamilyData::InstallSuperVersion() Summary: Deadlock: a memtable flush holds DB::mutex_ and calls ThreadLocalPtr::Scrape(), which locks ThreadLocalPtr mutex; meanwhile, a thread exit handler locks ThreadLocalPtr mutex and calls SuperVersionUnrefHandle, which tries to lock DB::mutex_. This deadlock is hit all the time on our workload. It blocks our release. In general, the problem is that ThreadLocalPtr takes an arbitrary callback and calls it while holding a lock on a global mutex. The same global mutex is (at least in some cases) locked by almost all ThreadLocalPtr methods, on any instance of ThreadLocalPtr. So, there'll be a deadlock if the callback tries to do anything to any instance of ThreadLocalPtr, or waits for another thread to do so. So, probably the only safe way to use ThreadLocalPtr callbacks is to do only do simple and lock-free things in them. This PR fixes the deadlock by making sure that local_sv_ never holds the last reference to a SuperVersion, and therefore SuperVersionUnrefHandle never has to do any nontrivial cleanup. I also searched for other uses of ThreadLocalPtr to see if they may have similar bugs. There's only one other use, in transaction_lock_mgr.cc, and it looks fine. Closes https://github.com/facebook/rocksdb/pull/3510 Reviewed By: sagar0 Differential Revision: D7005346 Pulled By: al13n321 fbshipit-source-id: 37575591b84f07a891d6659e87e784660fde815f	2018-02-16 11:34:46 -08:00
Siying Dong	7ad79400af	Direct I/O writable file should do fsync in Close() Summary: We don't do fsync() after truncate in direct I/O writeable file (in fact we don't do any fsync ever). This can cause metadata not persistent to disk after the file is generated. We call it instead. Closes https://github.com/facebook/rocksdb/pull/3500 Differential Revision: D6981482 Pulled By: siying fbshipit-source-id: 7e2b591b7e5dd1b96fc0775515b8b9e6092980ef	2018-02-13 16:52:07 -08:00
Wouter Beek	257eb0a41c	FIXED: string buffers potentially too small to fit formatted write Summary: This fixes the following warnings when compiled with GCC7: util/transaction_test_util.cc: In static member function ‘static rocksdb::Status rocksdb::RandomTransactionInserter::DBGet(rocksdb::DB, rocksdb::Transaction, rocksdb::ReadOptions&, uint16_t, uint64_t, bool, uint64_t, std::__cxx11::string, bool)’: util/transaction_test_util.cc:75:8: error: ‘snprintf’ output may be truncated before the last format character [-Werror=format-truncation=] Status RandomTransactionInserter::DBGet( ^~~~~~~~~~~~~~~~~~~~~~~~~ util/transaction_test_util.cc:84:11: note: ‘snprintf’ output between 5 and 6 bytes into a destination of size 5 snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1); ~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ util/transaction_test_util.cc: In static member function ‘static rocksdb::Status rocksdb::RandomTransactionInserter::Verify(rocksdb::DB, uint16_t, uint64_t, bool, rocksdb::Random64)’: util/transaction_test_util.cc:245:8: error: ‘snprintf’ output may be truncated before the last format character [-Werror=format-truncation=] Status RandomTransactionInserter::Verify(DB db, uint16_t num_sets, ^~~~~~~~~~~~~~~~~~~~~~~~~ util/transaction_test_util.cc:268:13: note: ‘snprintf’ output between 5 and 6 bytes into a destination of size 5 snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1); ~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Closes https://github.com/facebook/rocksdb/pull/3295 Differential Revision: D6609411 Pulled By: maysamyabandeh fbshipit-source-id: 33f0add471056eb59db2f8bd4366e6dfbb1a187d	2018-02-13 13:37:32 -08:00
Jun Wu	54c1984579	crc32: suppress -Wimplicit-fallthrough warnings Summary: Workaround a bunch of "implicit-fallthrough" compiler errors, like: ``` util/crc32c.cc:533:7: error: this statement may fall through [-Werror=implicit-fallthrough=] crc = _mm_crc32_u64(crc, (uint64_t)(buf + offset)); ^ util/crc32c.cc:1016:9: note: in expansion of macro ‘CRCsinglet’ CRCsinglet(crc0, next, -2 * 8); ^~~~~~~~~~ util/crc32c.cc:1017:7: note: here case 1: ``` Closes https://github.com/facebook/rocksdb/pull/3339 Reviewed By: sagar0 Differential Revision: D6874736 Pulled By: quark-zju fbshipit-source-id: eec9f3bc135e12fca336928d01711006d5c3cb16	2018-02-12 12:58:31 -08:00
Mark Isaacson	7ac544656d	Suppress lint in old files Summary: Grandfather in super old lint issues to make a clean slate for moving forward that allows us to have stronger enforcement on new issues. Reviewed By: yiwu-arbug Differential Revision: D6821806 fbshipit-source-id: 22797d31ec58e9eb0255d3b66fedfcfcb0dc127c	2018-01-30 17:15:39 -08:00
Yi Wu	c1e70e73ea	Bump version to 5.10.2	2018-01-30 11:13:10 -08:00
Yi Wu	d91f420dd9	StackableDB optionally take shared ownership of the underlying DB Summary: Allow StackableDB optionally takes a shared_ptr on construction and thus hold shared ownership of the underlying DB. Closes https://github.com/facebook/rocksdb/pull/3423 Differential Revision: D6824163 Pulled By: yiwu-arbug fbshipit-source-id: dbdc30c42e007533a987ef413785e192340f03eb	2018-01-30 11:12:37 -08:00
Sagar Vemuri	1454a5ccfe	Fix PowerPC dynamic java build Summary: Java build on PPC64le has been broken since a few months, due to #2716. Fixing it with the least amount of changes. (We should cleanup a little around this code when time permits). This should fix the build failures seen in http://140.211.168.68:8080/job/Rocksdb/ . Closes https://github.com/facebook/rocksdb/pull/3359 Differential Revision: D6712938 Pulled By: sagar0 fbshipit-source-id: 3046e8f072180693de2af4762934ec1ace309ca4	2018-01-18 18:02:26 -08:00
Yi Wu	d7defe4fa3	Bump version to 5.10.1	2018-01-18 18:02:03 -08:00
Yi Wu	e7e0aa7a2a	Fix Flush() keep waiting after flush finish Summary: Flush() call could be waiting indefinitely if min_write_buffer_number_to_merge is used. Consider the sequence: 1. User call Flush() with flush_options.wait = true 2. The manual flush started in the background 3. New memtable become immutable because of writes. The new memtable will not trigger flush if min_write_buffer_number_to_merge is not reached. 4. The manual flush finish. Because of the new memtable created at step 3 not being flush, previous logic of WaitForFlushMemTable() keep waiting, despite the memtables it intent to flush has been flushed. Here instead of checking if there are any more memtables to flush, WaitForFlushMemTable() also check the id of the earliest memtable. If the id is larger than that of latest memtable at the time flush was initiated, it means all the memtable at the time of flush start has all been flush. Closes https://github.com/facebook/rocksdb/pull/3378 Differential Revision: D6746789 Pulled By: yiwu-arbug fbshipit-source-id: 35e698f71c7f90b06337a93e6825f4ea3b619bfa	2018-01-18 17:52:02 -08:00
Adam Retter	6c1a7243d8	FreeBSD build support for RocksDB and RocksJava Summary: Tested on a clean FreeBSD 11.01 x64. Closes https://github.com/facebook/rocksdb/pull/1423 Closes https://github.com/facebook/rocksdb/pull/3357 Differential Revision: D6705868 Pulled By: sagar0 fbshipit-source-id: cbccbbdafd4f42922512ca03619a5d5583a425fd	2018-01-11 13:42:22 -08:00
Andrew Kryczka	c43d91eef7	fix powerpc java static build Summary: added support for C and asm files as required for `e612e31740`. Closes https://github.com/facebook/rocksdb/pull/3299 Differential Revision: D6612479 Pulled By: ajkr fbshipit-source-id: 6263ed7c1602f249460421825c76b5721f396163	2018-01-11 12:41:44 -08:00
Sagar Vemuri	cf2c88cb4d	Fix zstd/zdict include path for java static build Summary: With the ZSTD dictionary generator support added in #3057 `PORTABLE=1 ROCKSDB_NO_FBCODE=1 make rocksdbjavastatic` fails as it can't find zdict.h. Specifically due to: https://github.com/facebook/rocksdb/blob/e3a06f12d27fd50af7b6c5941973f529601f9a3e/util/compression.h#L39 In java static builds zstd code gets directly downloaded from https://github.com/facebook/zstd , and in there zdict.h is under dictBuilder directory. So, I modified libzstd.a target to use `make install` to collect all the header files into a single location and used that as the zstd's include path. Closes https://github.com/facebook/rocksdb/pull/3260 Differential Revision: D6669850 Pulled By: sagar0 fbshipit-source-id: f8a7562a670e5aed4c4fb6034a921697590d7285	2018-01-10 17:50:53 -08:00
Yi Wu	f897be87ae	Fix db_bench write being disabled in lite build Summary: The macro was added by mistake in #2372 Closes https://github.com/facebook/rocksdb/pull/3343 Differential Revision: D6681356 Pulled By: yiwu-arbug fbshipit-source-id: 4180172fb0eaef4189c07f219241e0c261c03461	2018-01-10 17:50:03 -08:00
Siying Dong	8e42c3f8b8	Remove GCC parameter "-march=native" for ARM Summary: Most popular versions of GCC can't identify platform on ARM if "-march=native" is specified. Remove it to unblock most people. Closes https://github.com/facebook/rocksdb/pull/3346 Differential Revision: D6690544 Pulled By: siying fbshipit-source-id: bbaba9fe2645b6b37144b36ea75beeff88992b49	2018-01-10 17:49:56 -08:00
Siying Dong	0d2818d8d7	Fix a wrong log formatting Summary: I experienced weird segfault because of this mismatch of type in log formatting. Fix it. Closes https://github.com/facebook/rocksdb/pull/3345 Differential Revision: D6687224 Pulled By: siying fbshipit-source-id: c51fb1c008b7ebc3efdc353a4adad3e8f5b3e9de	2018-01-10 17:49:43 -08:00