One more include<functional>

Relax the block count check on deallocation in env_test

.travis.yml

@@ -14,6 +14,6 @@ before_install:
  - sudo dpkg -i libgflags-dev_2.0-1_amd64.deb
 # Lousy hack to disable use and testing of fallocate, which doesn't behave quite
 # as EnvPosixTest::AllocateTest expects within the Travis OpenVZ environment.
-script: OPT=-DTRAVIS make unity && make clean && OPT="-DTRAVIS -Wshadow" make check -j8
+script: OPT=-DTRAVIS make unity && make clean && OPT=-DTRAVIS make check -j8
 notifications:
     email: false

HISTORY.md

@@ -1,7 +1,18 @@
 # Rocksdb Change Log
 
-## Unreleased
+## 3.8.0 (11/14/2014)
 
+### Public API changes
+* BackupEngine::NewBackupEngine() was deprecated; please use BackupEngine::Open() from now on.
+* BackupableDB/RestoreBackupableDB have new GarbageCollect() methods, which will clean up files from corrupt and obsolete backups.
+* BackupableDB/RestoreBackupableDB have new GetCorruptedBackups() methods which list corrupt backups.
+
+### Cleanup
+* Bunch of code cleanup, some extra warnings turned on (-Wshadow, -Wshorten-64-to-32, -Wnon-virtual-dtor)
+
+### New features
+* CompactFiles and EventListener, although they are still in experimental state
+* Full ColumnFamily support in RocksJava.
 
 ## 3.7.0 (11/6/2014)
 ### Public API changes

Makefile

@@ -69,9 +69,8 @@ install:
 	@[ ! -e $(SHARED) ] || install -C -m 644 $(SHARED) $(INSTALL_PATH)/lib
 #-------------------------------------------------
 
-WARNING_FLAGS = -Wall -Werror -Wsign-compare
 CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
-CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual
+CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor
 
 LDFLAGS += $(PLATFORM_LDFLAGS)
 
@@ -147,9 +146,10 @@ TESTS = \
 	cuckoo_table_builder_test \
 	cuckoo_table_reader_test \
 	cuckoo_table_db_test \
-	write_batch_with_index_test \
 	flush_job_test \
-	wal_manager_test
+	wal_manager_test \
+	listener_test \
+	write_batch_with_index_test
 
 TOOLS = \
         sst_dump \
@@ -502,6 +502,12 @@ cuckoo_table_reader_test: table/cuckoo_table_reader_test.o $(LIBOBJECTS) $(TESTH
 cuckoo_table_db_test: db/cuckoo_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/cuckoo_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
+listener_test: db/listener_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) db/listener_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
+compactor_test: utilities/compaction/compactor_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) utilities/compaction/compactor_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
 options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) util/options_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 

build_tools/build_detect_platform

@@ -284,6 +284,14 @@ EOF
     fi
 fi
 
+# Test whether -Wshorten-64-to-32 is available
+$CXX $CFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null  <<EOF
+  int main() {}
+EOF
+if [ "$?" = 0 ]; then
+    COMMON_FLAGS="$COMMON_FLAGS -Wshorten-64-to-32"
+fi
+
 # shall we use HDFS?
 
 if test "$USE_HDFS"; then

build_tools/dependencies.sh

@@ -0,0 +1,19 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
+NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
+TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
+LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832

build_tools/fbcode_config.sh

@@ -1,106 +1,148 @@
 #!/bin/sh
 #
 # Set environment variables so that we can compile rocksdb using
-# fbcode settings.  It uses the latest g++ compiler and also
+# fbcode settings.  It uses the latest g++ and clang compilers and also
 # uses jemalloc
+# Environment variables that change the behavior of this script:
+# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
 
-# location of libgcc
-LIBGCC_BASE="/mnt/gvfs/third-party2/libgcc/7712e757d7355cb51292454ee0b7b46a467fdfed/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc"
-LIBGCC_INCLUDE="$LIBGCC_BASE/include"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/libs"
 
-# location of glibc
-GLIBC_REV=6e40560b4e0b6d690fd1cf8c7a43ad7452b04cfa
-GLIBC_INCLUDE="/mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.17/gcc-4.8.1-glibc-2.17/99df8fc/include"
-GLIBC_LIBS=" -L /mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.17/gcc-4.8.1-glibc-2.17/99df8fc/lib"
+BASEDIR=`dirname $BASH_SOURCE`
+source "$BASEDIR/dependencies.sh"
 
-# location of snappy headers and libraries
-SNAPPY_INCLUDE=" -I /mnt/gvfs/third-party2/snappy/aef17f6c0b44b4fe408bd06f67c93701ab0a6ceb/1.0.3/gcc-4.8.1-glibc-2.17/43d84e2/include"
-SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/aef17f6c0b44b4fe408bd06f67c93701ab0a6ceb/1.0.3/gcc-4.8.1-glibc-2.17/43d84e2/lib/libsnappy.a"
+CFLAGS=""
 
-# location of zlib headers and libraries
-ZLIB_INCLUDE=" -I /mnt/gvfs/third-party2/zlib/25c6216928b4d77b59ddeca0990ff6fe9ac16b81/1.2.5/gcc-4.8.1-glibc-2.17/c3f970a/include"
-ZLIB_LIBS=" /mnt/gvfs/third-party2/zlib/25c6216928b4d77b59ddeca0990ff6fe9ac16b81/1.2.5/gcc-4.8.1-glibc-2.17/c3f970a/lib/libz.a"
+# libgcc
+LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
+LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
 
-# location of bzip headers and libraries
-BZIP_INCLUDE=" -I /mnt/gvfs/third-party2/bzip2/c9ef7629c2aa0024f7a416e87602f06eb88f5eac/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a/include/"
-BZIP_LIBS=" /mnt/gvfs/third-party2/bzip2/c9ef7629c2aa0024f7a416e87602f06eb88f5eac/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a/lib/libbz2.a"
+# glibc
+GLIBC_INCLUDE="$GLIBC_BASE/include"
+GLIBC_LIBS=" -L $GLIBC_BASE/lib"
 
-LZ4_REV=065ec7e38fe83329031f6668c43bef83eff5808b
-LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/include"
-LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/lib/liblz4.a"
+# snappy
+SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
+if test -z $PIC_BUILD; then
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
+else
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
+fi
+CFLAGS+=" -DSNAPPY"
+
+if test -z $PIC_BUILD; then
+  # location of zlib headers and libraries
+  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
+  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
+  CFLAGS+=" -DZLIB"
+
+  # location of bzip headers and libraries
+  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
+  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
+  CFLAGS+=" -DBZIP2"
+
+  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
+  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
+  CFLAGS+=" -DLZ4"
+
+  ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+  CFLAGS+=" -DZSTD"
+fi
 
 # location of gflags headers and libraries
-GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/1ad047a6e6f6673991918ecadc670868205a243a/1.6/gcc-4.8.1-glibc-2.17/c3f970a/include/"
-GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/1ad047a6e6f6673991918ecadc670868205a243a/1.6/gcc-4.8.1-glibc-2.17/c3f970a/lib/libgflags.a"
+GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
+if test -z $PIC_BUILD; then
+  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
+else
+  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
+fi
+CFLAGS+=" -DGFLAGS=gflags"
 
 # location of jemalloc
-JEMALLOC_INCLUDE=" -I /mnt/gvfs/third-party2/jemalloc/c60d854f7824f334195fe7fd34b2bc9057e3c1f9/3.6.0/gcc-4.8.1-glibc-2.17/4d53c6f/include"
-JEMALLOC_LIB=" -Wl,--whole-archive /mnt/gvfs/third-party2/jemalloc/c60d854f7824f334195fe7fd34b2bc9057e3c1f9/3.6.0/gcc-4.8.1-glibc-2.17/4d53c6f/lib/libjemalloc.a"
+JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
+JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
 
-# location of numa
-NUMA_REV=829d10dac0230f99cd7e1778869d2adf3da24b65
-NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/include/"
-NUMA_LIB=" /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/lib/libnuma.a"
+if test -z $PIC_BUILD; then
+  # location of numa
+  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
+  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
+  CFLAGS+=" -DNUMA"
 
-# location of libunwind
-LIBUNWIND_REV=2c060e64064559905d46fd194000d61592087bdc
-LIBUNWIND="/mnt/gvfs/third-party2/libunwind/$LIBUNWIND_REV/1.1/gcc-4.8.1-glibc-2.17/675d945/lib/libunwind.a"
+  # location of libunwind
+  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
+fi
+
+# location of TBB
+TBB_INCLUDE=" -isystem $TBB_BASE/include/"
+if test -z $PIC_BUILD; then
+  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
+else
+  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
+fi
+CFLAGS+=" -DTBB"
 
 # use Intel SSE support for checksum calculations
 export USE_SSE=" -msse -msse4.2 "
 
-BINUTILS="/mnt/gvfs/third-party2/binutils/2aff2e7b474cd3e6ab23495ad1224b7d214b9f8e/2.21.1/centos6-native/da39a3e/bin"
+BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"
 
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
 
-GCC_BASE="/mnt/gvfs/third-party2/gcc/1ec615e23800f0815d474478ba476a0adc3fe788/4.8.1/centos6-native/cc6c9dc"
 STDLIBS="-L $GCC_BASE/lib64"
 
+CLANG_BIN="$CLANG_BASE/bin"
+CLANG_LIB="$CLANG_BASE/lib"
+CLANG_SRC="$CLANG_BASE/../../src"
+
+CLANG_ANALYZER="$CLANG_BIN/clang++"
+CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
+
 if [ -z "$USE_CLANG" ]; then
   # gcc
   CC="$GCC_BASE/bin/gcc"
   CXX="$GCC_BASE/bin/g++"
-  
-  CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic"
-  CFLAGS+=" -I $LIBGCC_INCLUDE -I $GLIBC_INCLUDE"
-  CFLAGS+=" $DEPS_INCLUDE"
-  CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_FALLOCATE_PRESENT"
-  CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DNUMA"
+
+  CFLAGS+=" -B$BINUTILS/gold"
+  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  JEMALLOC=1
 else
-  # clang 
-  CLANG_BASE="/mnt/gvfs/third-party2/clang/9ab68376f938992c4eb5946ca68f90c3185cffc8/3.4"
-  CLANG_INCLUDE="$CLANG_BASE/gcc-4.8.1-glibc-2.17/fb0f730/lib/clang/3.4/include"
-  CC="$CLANG_BASE/centos6-native/9cefd8a/bin/clang"
-  CXX="$CLANG_BASE/centos6-native/9cefd8a/bin/clang++"
+  # clang
+  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
+  CC="$CLANG_BIN/clang"
+  CXX="$CLANG_BIN/clang++"
 
-  KERNEL_HEADERS_INCLUDE="/mnt/gvfs/third-party2/kernel-headers/a683ed7135276731065a9d76d3016c9731f4e2f9/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.8.1-glibc-2.17/da39a3e/include/"
+  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
 
-  CFLAGS="-B$BINUTILS  -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1/x86_64-facebook-linux "
+  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
   CFLAGS+=" -isystem $LIBGCC_INCLUDE"
   CFLAGS+=" -isystem $CLANG_INCLUDE"
   CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
   CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
-  CFLAGS+=" -Wall -Wno-sign-compare -Wno-unused-variable -Winvalid-pch -Wno-deprecated -Woverloaded-virtual"
-  CFLAGS+=" $DEPS_INCLUDE"
-  CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_FALLOCATE_PRESENT"
-  CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DNUMA"
-  CXXFLAGS="$CFLAGS -nostdinc++"
+  CFLAGS+=" -Wno-expansion-to-defined "
+  CXXFLAGS="-nostdinc++"
 fi
 
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS $NUMA_LIB"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
-EXEC_LDFLAGS+=" -Wl,--no-whole-archive $LIBUNWIND"
+CFLAGS+=" $DEPS_INCLUDE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
+CXXFLAGS+=" $CFLAGS"
+
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
+EXEC_LDFLAGS+=" -B$BINUTILS/gold"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
+EXEC_LDFLAGS+=" $LIBUNWIND"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
+# required by libtbb
+EXEC_LDFLAGS+=" -ldl"
 
 PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
 
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS"
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
 
-VALGRIND_REV=b2a9f85e4b70cd03abc85a7f3027fbc4cef35bd0
-VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/$VALGRIND_REV/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a/bin/"
+VALGRIND_VER="$VALGRIND_BASE/bin/"
 
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD 

db/c.cc

@@ -385,11 +385,9 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
     unsigned char success;
     size_t new_value_len;
     char* tmp_new_value = (*full_merge_)(
-        state_,
-        key.data(), key.size(),
-        existing_value_data, existing_value_len,
-        &operand_pointers[0], &operand_sizes[0], n,
-        &success, &new_value_len);
+        state_, key.data(), key.size(), existing_value_data, existing_value_len,
+        &operand_pointers[0], &operand_sizes[0], static_cast<int>(n), &success,
+        &new_value_len);
     new_value->assign(tmp_new_value, new_value_len);
 
     if (delete_value_ != nullptr) {
@@ -417,7 +415,7 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
     size_t new_value_len;
     char* tmp_new_value = (*partial_merge_)(
         state_, key.data(), key.size(), &operand_pointers[0], &operand_sizes[0],
-        operand_count, &success, &new_value_len);
+        static_cast<int>(operand_count), &success, &new_value_len);
     new_value->assign(tmp_new_value, new_value_len);
 
     if (delete_value_ != nullptr) {
@@ -2041,7 +2039,7 @@ void rocksdb_options_set_min_level_to_compress(rocksdb_options_t* opt, int level
 
 int rocksdb_livefiles_count(
   const rocksdb_livefiles_t* lf) {
-  return lf->rep.size();
+  return static_cast<int>(lf->rep.size());
 }
 
 const char* rocksdb_livefiles_name(

db/c_test.c

@@ -132,7 +132,7 @@ static void CmpDestroy(void* arg) { }
 
 static int CmpCompare(void* arg, const char* a, size_t alen,
                       const char* b, size_t blen) {
-  int n = (alen < blen) ? alen : blen;
+  size_t n = (alen < blen) ? alen : blen;
   int r = memcmp(a, b, n);
   if (r == 0) {
     if (alen < blen) r = -1;

db/column_family.cc

@@ -87,6 +87,10 @@ ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
 
 uint32_t ColumnFamilyHandleImpl::GetID() const { return cfd()->GetID(); }
 
+const std::string& ColumnFamilyHandleImpl::GetName() const {
+  return cfd()->GetName();
+}
+
 const Comparator* ColumnFamilyHandleImpl::user_comparator() const {
   return cfd()->user_comparator();
 }
@@ -255,10 +259,23 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
     } else if (ioptions_.compaction_style == kCompactionStyleLevel) {
       compaction_picker_.reset(
           new LevelCompactionPicker(ioptions_, &internal_comparator_));
-    } else {
-      assert(ioptions_.compaction_style == kCompactionStyleFIFO);
+    } else if (ioptions_.compaction_style == kCompactionStyleFIFO) {
       compaction_picker_.reset(
           new FIFOCompactionPicker(ioptions_, &internal_comparator_));
+    } else if (ioptions_.compaction_style == kCompactionStyleNone) {
+      compaction_picker_.reset(new NullCompactionPicker(
+          ioptions_, &internal_comparator_));
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+          "Column family %s does not use any background compaction. "
+          "Compactions can only be done via CompactFiles\n",
+          GetName().c_str());
+    } else {
+      Log(InfoLogLevel::ERROR_LEVEL, ioptions_.info_log,
+          "Unable to recognize the specified compaction style %d. "
+          "Column family %s will use kCompactionStyleLevel.\n",
+          ioptions_.compaction_style, GetName().c_str());
+      compaction_picker_.reset(
+          new LevelCompactionPicker(ioptions_, &internal_comparator_));
     }
 
     Log(InfoLogLevel::INFO_LEVEL,
@@ -503,6 +520,22 @@ bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) {
   return false;
 }
 
+void ColumnFamilyData::NotifyOnFlushCompleted(
+    DB* db, const std::string& file_path,
+    bool triggered_flush_slowdown,
+    bool triggered_flush_stop) {
+
+#ifndef ROCKSDB_LITE
+  auto listeners = ioptions()->listeners;
+  for (auto listener : listeners) {
+    listener->OnFlushCompleted(
+        db, GetName(), file_path,
+        // Use path 0 as fulled memtables are first flushed into path 0.
+        triggered_flush_slowdown, triggered_flush_stop);
+  }
+#endif  // ROCKSDB_LITE
+}
+
 SuperVersion* ColumnFamilyData::InstallSuperVersion(
     SuperVersion* new_superversion, port::Mutex* db_mutex) {
   db_mutex->AssertHeld();
@@ -547,6 +580,7 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
   }
 }
 
+#ifndef ROCKSDB_LITE
 Status ColumnFamilyData::SetOptions(
       const std::unordered_map<std::string, std::string>& options_map) {
   MutableCFOptions new_mutable_cf_options;
@@ -558,6 +592,7 @@ Status ColumnFamilyData::SetOptions(
   }
   return s;
 }
+#endif  // ROCKSDB_LITE
 
 ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
                                  const DBOptions* db_options,

db/column_family.h

@@ -52,6 +52,7 @@ class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
   virtual const Comparator* user_comparator() const;
 
   virtual uint32_t GetID() const;
+  virtual const std::string& GetName() const override;
 
  private:
   ColumnFamilyData* cfd_;
@@ -186,9 +187,11 @@ class ColumnFamilyData {
   const MutableCFOptions* GetLatestMutableCFOptions() const {
     return &mutable_cf_options_;
   }
+#ifndef ROCKSDB_LITE
   // REQUIRES: DB mutex held
   Status SetOptions(
       const std::unordered_map<std::string, std::string>& options_map);
+#endif  // ROCKSDB_LITE
 
   InternalStats* internal_stats() { return internal_stats_.get(); }
 
@@ -250,6 +253,11 @@ class ColumnFamilyData {
 
   void ResetThreadLocalSuperVersions();
 
+  void NotifyOnFlushCompleted(
+      DB* db, const std::string& file_path,
+      bool triggered_flush_slowdown,
+      bool triggered_flush_stop);
+
  private:
   friend class ColumnFamilySet;
   ColumnFamilyData(uint32_t id, const std::string& name,

db/column_family_test.cc

@@ -133,7 +133,7 @@ class ColumnFamilyTest {
   void CreateColumnFamilies(
       const std::vector<std::string>& cfs,
       const std::vector<ColumnFamilyOptions> options = {}) {
-    int cfi = handles_.size();
+    int cfi = static_cast<int>(handles_.size());
     handles_.resize(cfi + cfs.size());
     names_.resize(cfi + cfs.size());
     for (size_t i = 0; i < cfs.size(); ++i) {
@@ -231,7 +231,7 @@ class ColumnFamilyTest {
       snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
       result += buf;
       if (f > 0) {
-        last_non_zero_offset = result.size();
+        last_non_zero_offset = static_cast<int>(result.size());
       }
     }
     result.resize(last_non_zero_offset);
@@ -287,8 +287,8 @@ class ColumnFamilyTest {
     assert(num_per_cf.size() == handles_.size());
 
     for (size_t i = 0; i < num_per_cf.size(); ++i) {
-      ASSERT_EQ(num_per_cf[i],
-                GetProperty(i, "rocksdb.num-immutable-mem-table"));
+      ASSERT_EQ(num_per_cf[i], GetProperty(static_cast<int>(i),
+                                           "rocksdb.num-immutable-mem-table"));
     }
   }
 
@@ -916,11 +916,11 @@ TEST(ColumnFamilyTest, DontRollEmptyLogs) {
   CreateColumnFamiliesAndReopen({"one", "two", "three", "four"});
 
   for (size_t i = 0; i < handles_.size(); ++i) {
-    PutRandomData(i, 10, 100);
+    PutRandomData(static_cast<int>(i), 10, 100);
   }
   int num_writable_file_start = env_->GetNumberOfNewWritableFileCalls();
   // this will trigger the flushes
-  for (size_t i = 0; i <= 4; ++i) {
+  for (int i = 0; i <= 4; ++i) {
     ASSERT_OK(Flush(i));
   }
 

db/compaction.cc

@@ -78,6 +78,38 @@ Compaction::Compaction(int number_levels, int start_level, int out_level,
   }
 }
 
+Compaction::Compaction(VersionStorageInfo* vstorage,
+    const autovector<CompactionInputFiles>& _inputs,
+    int _start_level, int _output_level,
+    uint64_t _max_grandparent_overlap_bytes,
+    const CompactionOptions& _options,
+    bool _deletion_compaction)
+    : start_level_(_start_level),
+      output_level_(_output_level),
+      max_output_file_size_(_options.output_file_size_limit),
+      max_grandparent_overlap_bytes_(_max_grandparent_overlap_bytes),
+      input_version_(nullptr),
+      number_levels_(vstorage->num_levels()),
+      cfd_(nullptr),
+      output_compression_(_options.compression),
+      seek_compaction_(false),
+      deletion_compaction_(_deletion_compaction),
+      inputs_(_inputs),
+      grandparent_index_(0),
+      seen_key_(false),
+      overlapped_bytes_(0),
+      base_index_(-1),
+      parent_index_(-1),
+      score_(0),
+      bottommost_level_(false),
+      is_full_compaction_(false),
+      is_manual_compaction_(false),
+      level_ptrs_(std::vector<size_t>(number_levels_)) {
+  for (int i = 0; i < number_levels_; i++) {
+    level_ptrs_[i] = 0;
+  }
+}
+
 Compaction::~Compaction() {
   delete edit_;
   if (input_version_ != nullptr) {
@@ -92,9 +124,9 @@ Compaction::~Compaction() {
 
 void Compaction::GenerateFileLevels() {
   input_levels_.resize(num_input_levels());
-  for (int which = 0; which < num_input_levels(); which++) {
-    DoGenerateLevelFilesBrief(
-        &input_levels_[which], inputs_[which].files, &arena_);
+  for (size_t which = 0; which < num_input_levels(); which++) {
+    DoGenerateLevelFilesBrief(&input_levels_[which], inputs_[which].files,
+                              &arena_);
   }
 }
 
@@ -112,7 +144,7 @@ bool Compaction::IsTrivialMove() const {
 }
 
 void Compaction::AddInputDeletions(VersionEdit* out_edit) {
-  for (int which = 0; which < num_input_levels(); which++) {
+  for (size_t which = 0; which < num_input_levels(); which++) {
     for (size_t i = 0; i < inputs_[which].size(); i++) {
       out_edit->DeleteFile(level(which), inputs_[which][i]->fd.GetNumber());
     }
@@ -175,7 +207,7 @@ bool Compaction::ShouldStopBefore(const Slice& internal_key) {
 
 // Mark (or clear) each file that is being compacted
 void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
-  for (int i = 0; i < num_input_levels(); i++) {
+  for (size_t i = 0; i < num_input_levels(); i++) {
     for (unsigned int j = 0; j < inputs_[i].size(); j++) {
       assert(mark_as_compacted ? !inputs_[i][j]->being_compacted :
                                   inputs_[i][j]->being_compacted);
@@ -261,7 +293,7 @@ void Compaction::Summary(char* output, int len) {
     return;
   }
 
-  for (int level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
+  for (size_t level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
     if (level_iter > 0) {
       write += snprintf(output + write, len - write, "], [");
       if (write < 0 || write >= len) {
@@ -285,7 +317,7 @@ uint64_t Compaction::OutputFilePreallocationSize(
   if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
     preallocation_size = mutable_options.MaxFileSizeForLevel(output_level());
   } else {
-    for (int level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
+    for (size_t level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
       for (const auto& f : inputs_[level_iter].files) {
         preallocation_size += f->fd.GetFileSize();
       }

db/compaction.h

@@ -23,7 +23,7 @@ struct CompactionInputFiles {
   inline bool empty() const { return files.empty(); }
   inline size_t size() const { return files.size(); }
   inline void clear() { files.clear(); }
-  inline FileMetaData* operator[](int i) const { return files[i]; }
+  inline FileMetaData* operator[](size_t i) const { return files[i]; }
 };
 
 class Version;
@@ -33,6 +33,13 @@ class VersionStorageInfo;
 // A Compaction encapsulates information about a compaction.
 class Compaction {
  public:
+  Compaction(VersionStorageInfo* input_version,
+    const autovector<CompactionInputFiles>& inputs,
+    int start_level, int output_level,
+    uint64_t max_grandparent_overlap_bytes,
+    const CompactionOptions& options,
+    bool deletion_compaction);
+
   // No copying allowed
   Compaction(const Compaction&) = delete;
   void operator=(const Compaction&) = delete;
@@ -41,7 +48,7 @@ class Compaction {
 
   // Returns the level associated to the specified compaction input level.
   // If compaction_input_level is not specified, then input_level is set to 0.
-  int level(int compaction_input_level = 0) const {
+  int level(size_t compaction_input_level = 0) const {
     return inputs_[compaction_input_level].level;
   }
 
@@ -49,7 +56,7 @@ class Compaction {
   int output_level() const { return output_level_; }
 
   // Returns the number of input levels in this compaction.
-  int num_input_levels() const { return inputs_.size(); }
+  size_t num_input_levels() const { return inputs_.size(); }
 
   // Return the object that holds the edits to the descriptor done
   // by this compaction.
@@ -59,7 +66,7 @@ class Compaction {
   // compaction input level.
   // The function will return 0 if when "compaction_input_level" < 0
   // or "compaction_input_level" >= "num_input_levels()".
-  int num_input_files(size_t compaction_input_level) const {
+  size_t num_input_files(size_t compaction_input_level) const {
     if (compaction_input_level < inputs_.size()) {
       return inputs_[compaction_input_level].size();
     }
@@ -76,7 +83,7 @@ class Compaction {
   // specified compaction input level.
   // REQUIREMENT: "compaction_input_level" must be >= 0 and
   //              < "input_levels()"
-  FileMetaData* input(size_t compaction_input_level, int i) const {
+  FileMetaData* input(size_t compaction_input_level, size_t i) const {
     assert(compaction_input_level < inputs_.size());
     return inputs_[compaction_input_level][i];
   }
@@ -91,7 +98,7 @@ class Compaction {
   }
 
   // Returns the LevelFilesBrief of the specified compaction input level.
-  LevelFilesBrief* input_levels(int compaction_input_level) {
+  LevelFilesBrief* input_levels(size_t compaction_input_level) {
     return &input_levels_[compaction_input_level];
   }
 
@@ -153,6 +160,8 @@ class Compaction {
   // Was this compaction triggered manually by the client?
   bool IsManualCompaction() { return is_manual_compaction_; }
 
+  void SetOutputPathId(uint32_t path_id) { output_path_id_ = path_id; }
+
   // Return the MutableCFOptions that should be used throughout the compaction
   // procedure
   const MutableCFOptions* mutable_cf_options() { return &mutable_cf_options_; }
@@ -164,6 +173,16 @@ class Compaction {
 
   void SetInputVersion(Version* input_version);
 
+  // mark (or clear) all files that are being compacted
+  void MarkFilesBeingCompacted(bool mark_as_compacted);
+
+  // Initialize whether the compaction is producing files at the
+  // bottommost level.
+  //
+  // @see BottomMostLevel()
+  void SetupBottomMostLevel(VersionStorageInfo* vstorage, bool is_manual,
+                            bool level0_only);
+
  private:
   friend class CompactionPicker;
   friend class UniversalCompactionPicker;
@@ -226,16 +245,6 @@ class Compaction {
   // records indices for all levels beyond "output_level_".
   std::vector<size_t> level_ptrs_;
 
-  // mark (or clear) all files that are being compacted
-  void MarkFilesBeingCompacted(bool mark_as_compacted);
-
-  // Initialize whether the compaction is producing files at the
-  // bottommost level.
-  //
-  // @see BottomMostLevel()
-  void SetupBottomMostLevel(VersionStorageInfo* vstorage, bool is_manual,
-                            bool level0_only);
-
   // In case of compaction error, reset the nextIndex that is used
   // to pick up the next file to be compacted from files_by_size_
   void ResetNextCompactionIndex();

db/compaction_job.cc

@@ -71,7 +71,6 @@ struct CompactionJob::CompactionState {
     SequenceNumber smallest_seqno, largest_seqno;
   };
   std::vector<Output> outputs;
-  std::list<uint64_t> allocated_file_numbers;
 
   // State kept for output being generated
   std::unique_ptr<WritableFile> outfile;
@@ -204,8 +203,7 @@ struct CompactionJob::CompactionState {
 CompactionJob::CompactionJob(
     Compaction* compaction, const DBOptions& db_options,
     const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
-    VersionSet* versions, port::Mutex* db_mutex,
-    std::atomic<bool>* shutting_down, FileNumToPathIdMap* pending_outputs,
+    VersionSet* versions, std::atomic<bool>* shutting_down,
     LogBuffer* log_buffer, Directory* db_directory, Statistics* stats,
     SnapshotList* snapshots, bool is_snapshot_supported,
     std::shared_ptr<Cache> table_cache,
@@ -217,9 +215,7 @@ CompactionJob::CompactionJob(
       env_options_(env_options),
       env_(db_options.env),
       versions_(versions),
-      db_mutex_(db_mutex),
       shutting_down_(shutting_down),
-      pending_outputs_(pending_outputs),
       log_buffer_(log_buffer),
       db_directory_(db_directory),
       stats_(stats),
@@ -229,7 +225,6 @@ CompactionJob::CompactionJob(
       yield_callback_(std::move(yield_callback)) {}
 
 void CompactionJob::Prepare() {
-  db_mutex_->AssertHeld();
   compact_->CleanupBatchBuffer();
   compact_->CleanupMergedBuffer();
 
@@ -269,9 +264,6 @@ void CompactionJob::Prepare() {
 
   // Is this compaction producing files at the bottommost level?
   bottommost_level_ = compact_->compaction->BottomMostLevel();
-
-  // Allocate the output file numbers before we release the lock
-  AllocateCompactionOutputFileNumbers();
 }
 
 Status CompactionJob::Run() {
@@ -423,32 +415,33 @@ Status CompactionJob::Run() {
   }
 
   compaction_stats_.micros = env_->NowMicros() - start_micros - imm_micros;
-  compaction_stats_.files_in_leveln = compact_->compaction->num_input_files(0);
+  compaction_stats_.files_in_leveln =
+      static_cast<int>(compact_->compaction->num_input_files(0));
   compaction_stats_.files_in_levelnp1 =
-      compact_->compaction->num_input_files(1);
+      static_cast<int>(compact_->compaction->num_input_files(1));
   MeasureTime(stats_, COMPACTION_TIME, compaction_stats_.micros);
 
-  int num_output_files = compact_->outputs.size();
+  size_t num_output_files = compact_->outputs.size();
   if (compact_->builder != nullptr) {
     // An error occurred so ignore the last output.
     assert(num_output_files > 0);
     --num_output_files;
   }
-  compaction_stats_.files_out_levelnp1 = num_output_files;
+  compaction_stats_.files_out_levelnp1 = static_cast<int>(num_output_files);
 
-  for (int i = 0; i < compact_->compaction->num_input_files(0); i++) {
+  for (size_t i = 0; i < compact_->compaction->num_input_files(0); i++) {
     compaction_stats_.bytes_readn +=
         compact_->compaction->input(0, i)->fd.GetFileSize();
     compaction_stats_.num_input_records +=
         static_cast<uint64_t>(compact_->compaction->input(0, i)->num_entries);
   }
 
-  for (int i = 0; i < compact_->compaction->num_input_files(1); i++) {
+  for (size_t i = 0; i < compact_->compaction->num_input_files(1); i++) {
     compaction_stats_.bytes_readnp1 +=
         compact_->compaction->input(1, i)->fd.GetFileSize();
   }
 
-  for (int i = 0; i < num_output_files; i++) {
+  for (size_t i = 0; i < num_output_files; i++) {
     compaction_stats_.bytes_written += compact_->outputs[i].file_size;
   }
   if (compact_->num_input_records > compact_->num_output_records) {
@@ -463,18 +456,14 @@ Status CompactionJob::Run() {
   return status;
 }
 
-Status CompactionJob::Install(Status status) {
-  db_mutex_->AssertHeld();
+void CompactionJob::Install(Status* status, port::Mutex* db_mutex) {
+  db_mutex->AssertHeld();
   ColumnFamilyData* cfd = compact_->compaction->column_family_data();
   cfd->internal_stats()->AddCompactionStats(
       compact_->compaction->output_level(), compaction_stats_);
 
-  // if there were any unused file number (mostly in case of
-  // compaction error), free up the entry from pending_putputs
-  ReleaseCompactionUnusedFileNumbers();
-
-  if (status.ok()) {
-    status = InstallCompactionResults();
+  if (status->ok()) {
+    *status = InstallCompactionResults(db_mutex);
   }
   VersionStorageInfo::LevelSummaryStorage tmp;
   const auto& stats = compaction_stats_;
@@ -495,26 +484,10 @@ Status CompactionJob::Install(Status status) {
               (stats.bytes_written + stats.bytes_readnp1 + stats.bytes_readn) /
                   static_cast<double>(stats.bytes_readn),
               stats.bytes_written / static_cast<double>(stats.bytes_readn),
-              status.ToString().c_str(), stats.num_input_records,
+              status->ToString().c_str(), stats.num_input_records,
               stats.num_dropped_records);
 
-  CleanupCompaction(status);
-  return status;
-}
-
-// Allocate the file numbers for the output file. We allocate as
-// many output file numbers as there are files in level+1 (at least one)
-// Insert them into pending_outputs so that they do not get deleted.
-void CompactionJob::AllocateCompactionOutputFileNumbers() {
-  db_mutex_->AssertHeld();
-  assert(compact_->builder == nullptr);
-  int filesNeeded = compact_->compaction->num_input_files(1);
-  for (int i = 0; i < std::max(filesNeeded, 1); i++) {
-    uint64_t file_number = versions_->NewFileNumber();
-    pending_outputs_->insert(
-        {file_number, compact_->compaction->GetOutputPathId()});
-    compact_->allocated_file_numbers.push_back(file_number);
-  }
+  CleanupCompaction(*status);
 }
 
 Status CompactionJob::ProcessKeyValueCompaction(int64_t* imm_micros,
@@ -889,7 +862,8 @@ void CompactionJob::CallCompactionFilterV2(
   assert(compact_->to_delete_buf_.size() == compact_->key_str_buf_.size());
   assert(compact_->to_delete_buf_.size() ==
          compact_->existing_value_str_buf_.size());
-  assert(compact_->to_delete_buf_.size() ==
+  assert(compact_->value_changed_buf_.empty() ||
+         compact_->to_delete_buf_.size() ==
          compact_->value_changed_buf_.size());
 
   int new_value_idx = 0;
@@ -904,7 +878,8 @@ void CompactionJob::CallCompactionFilterV2(
       // no value associated with delete
       compact_->existing_value_str_buf_[i].clear();
       RecordTick(stats_, COMPACTION_KEY_DROP_USER);
-    } else if (compact_->value_changed_buf_[i]) {
+    } else if (!compact_->value_changed_buf_.empty() &&
+        compact_->value_changed_buf_[i]) {
       compact_->existing_value_str_buf_[i] =
           compact_->new_value_buf_[new_value_idx++];
     }
@@ -966,8 +941,8 @@ Status CompactionJob::FinishCompactionOutputFile(Iterator* input) {
   return s;
 }
 
-Status CompactionJob::InstallCompactionResults() {
-  db_mutex_->AssertHeld();
+Status CompactionJob::InstallCompactionResults(port::Mutex* db_mutex) {
+  db_mutex->AssertHeld();
 
   // paranoia: verify that the files that we started with
   // still exist in the current version and in the same original level.
@@ -1003,7 +978,7 @@ Status CompactionJob::InstallCompactionResults() {
   }
   return versions_->LogAndApply(
       compact_->compaction->column_family_data(), mutable_cf_options_,
-      compact_->compaction->edit(), db_mutex_, db_directory_);
+      compact_->compaction->edit(), db_mutex, db_directory_);
 }
 
 // Given a sequence number, return the sequence number of the
@@ -1041,31 +1016,11 @@ void CompactionJob::RecordCompactionIOStats() {
   IOSTATS_RESET(bytes_written);
 }
 
-// Frees up unused file number.
-void CompactionJob::ReleaseCompactionUnusedFileNumbers() {
-  db_mutex_->AssertHeld();
-  for (const auto file_number : compact_->allocated_file_numbers) {
-    pending_outputs_->erase(file_number);
-  }
-}
-
 Status CompactionJob::OpenCompactionOutputFile() {
   assert(compact_ != nullptr);
   assert(compact_->builder == nullptr);
-  uint64_t file_number;
-  // If we have not yet exhausted the pre-allocated file numbers,
-  // then use the one from the front. Otherwise, we have to acquire
-  // the heavyweight lock and allocate a new file number.
-  if (!compact_->allocated_file_numbers.empty()) {
-    file_number = compact_->allocated_file_numbers.front();
-    compact_->allocated_file_numbers.pop_front();
-  } else {
-    db_mutex_->Lock();
-    file_number = versions_->NewFileNumber();
-    pending_outputs_->insert(
-        {file_number, compact_->compaction->GetOutputPathId()});
-    db_mutex_->Unlock();
-  }
+  // no need to lock because VersionSet::next_file_number_ is atomic
+  uint64_t file_number = versions_->NewFileNumber();
   // Make the output file
   std::string fname = TableFileName(db_options_.db_paths, file_number,
                                     compact_->compaction->GetOutputPathId());
@@ -1089,8 +1044,8 @@ Status CompactionJob::OpenCompactionOutputFile() {
 
   compact_->outputs.push_back(out);
   compact_->outfile->SetIOPriority(Env::IO_LOW);
-  compact_->outfile->SetPreallocationBlockSize(
-      compact_->compaction->OutputFilePreallocationSize(mutable_cf_options_));
+  compact_->outfile->SetPreallocationBlockSize(static_cast<size_t>(
+      compact_->compaction->OutputFilePreallocationSize(mutable_cf_options_)));
 
   ColumnFamilyData* cfd = compact_->compaction->column_family_data();
   compact_->builder.reset(NewTableBuilder(
@@ -1101,8 +1056,7 @@ Status CompactionJob::OpenCompactionOutputFile() {
   return s;
 }
 
-void CompactionJob::CleanupCompaction(Status status) {
-  db_mutex_->AssertHeld();
+void CompactionJob::CleanupCompaction(const Status& status) {
   if (compact_->builder != nullptr) {
     // May happen if we get a shutdown call in the middle of compaction
     compact_->builder->Abandon();
@@ -1112,7 +1066,6 @@ void CompactionJob::CleanupCompaction(Status status) {
   }
   for (size_t i = 0; i < compact_->outputs.size(); i++) {
     const CompactionState::Output& out = compact_->outputs[i];
-    pending_outputs_->erase(out.number);
 
     // If this file was inserted into the table cache then remove
     // them here because this compaction was not committed.

db/compaction_job.h

@@ -56,8 +56,7 @@ class CompactionJob {
   CompactionJob(Compaction* compaction, const DBOptions& db_options,
                 const MutableCFOptions& mutable_cf_options,
                 const EnvOptions& env_options, VersionSet* versions,
-                port::Mutex* db_mutex, std::atomic<bool>* shutting_down,
-                FileNumToPathIdMap* pending_outputs, LogBuffer* log_buffer,
+                std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
                 Directory* db_directory, Statistics* stats,
                 SnapshotList* snapshot_list, bool is_snapshot_supported,
                 std::shared_ptr<Cache> table_cache,
@@ -76,7 +75,7 @@ class CompactionJob {
   Status Run();
   // REQUIRED: mutex held
   // status is the return of Run()
-  Status Install(Status status);
+  void Install(Status* status, port::Mutex* db_mutex);
 
  private:
   void AllocateCompactionOutputFileNumbers();
@@ -87,14 +86,13 @@ class CompactionJob {
   // Call compaction_filter_v2->Filter() on kv-pairs in compact
   void CallCompactionFilterV2(CompactionFilterV2* compaction_filter_v2);
   Status FinishCompactionOutputFile(Iterator* input);
-  Status InstallCompactionResults();
+  Status InstallCompactionResults(port::Mutex* db_mutex);
   SequenceNumber findEarliestVisibleSnapshot(
       SequenceNumber in, const std::vector<SequenceNumber>& snapshots,
       SequenceNumber* prev_snapshot);
   void RecordCompactionIOStats();
-  void ReleaseCompactionUnusedFileNumbers();
   Status OpenCompactionOutputFile();
-  void CleanupCompaction(Status status);
+  void CleanupCompaction(const Status& status);
 
   // CompactionJob state
   struct CompactionState;
@@ -113,9 +111,7 @@ class CompactionJob {
   const EnvOptions& env_options_;
   Env* env_;
   VersionSet* versions_;
-  port::Mutex* db_mutex_;
   std::atomic<bool>* shutting_down_;
-  FileNumToPathIdMap* pending_outputs_;
   LogBuffer* log_buffer_;
   Directory* db_directory_;
   Statistics* stats_;

db/compaction_picker.cc

@@ -46,7 +46,7 @@ CompressionType GetCompressionType(
   // If the use has specified a different compression level for each level,
   // then pick the compression for that level.
   if (!ioptions.compression_per_level.empty()) {
-    const int n = ioptions.compression_per_level.size() - 1;
+    const int n = static_cast<int>(ioptions.compression_per_level.size()) - 1;
     // It is possible for level_ to be -1; in that case, we use level
     // 0's compression.  This occurs mostly in backwards compatibility
     // situations when the builder doesn't know what level the file
@@ -75,7 +75,7 @@ void CompactionPicker::SizeBeingCompacted(std::vector<uint64_t>& sizes) {
     uint64_t total = 0;
     for (auto c : compactions_in_progress_[level]) {
       assert(c->level() == level);
-      for (int i = 0; i < c->num_input_files(0); i++) {
+      for (size_t i = 0; i < c->num_input_files(0); i++) {
         total += c->input(0, i)->compensated_file_size;
       }
     }
@@ -185,7 +185,8 @@ bool CompactionPicker::ExpandWhileOverlapping(const std::string& cf_name,
 }
 
 // Returns true if any one of specified files are being compacted
-bool CompactionPicker::FilesInCompaction(std::vector<FileMetaData*>& files) {
+bool CompactionPicker::FilesInCompaction(
+    const std::vector<FileMetaData*>& files) {
   for (unsigned int i = 0; i < files.size(); i++) {
     if (files[i]->being_compacted) {
       return true;
@@ -194,6 +195,89 @@ bool CompactionPicker::FilesInCompaction(std::vector<FileMetaData*>& files) {
   return false;
 }
 
+Compaction* CompactionPicker::FormCompaction(
+      const CompactionOptions& compact_options,
+      const autovector<CompactionInputFiles>& input_files,
+      int output_level, VersionStorageInfo* vstorage,
+      const MutableCFOptions& mutable_cf_options) const {
+  uint64_t max_grandparent_overlap_bytes =
+      output_level + 1 < vstorage->num_levels() ?
+          mutable_cf_options.MaxGrandParentOverlapBytes(output_level + 1) :
+          std::numeric_limits<uint64_t>::max();
+  assert(input_files.size());
+  auto c = new Compaction(vstorage, input_files,
+      input_files[0].level, output_level,
+      max_grandparent_overlap_bytes,
+      compact_options, false);
+  c->mutable_cf_options_ = mutable_cf_options;
+  c->MarkFilesBeingCompacted(true);
+
+  // TODO(yhchiang): complete the SetBottomMostLevel as follows
+  // If there is no any key of the range in DB that is older than the
+  // range to compact, it is bottom most.  For leveled compaction,
+  // if number-of_level-1 is empty, and output is going to number-of_level-2,
+  // it is also bottom-most.  On the other hand, if number of level=1 (
+  // something like universal), the compaction is only "bottom-most" if
+  // the oldest file is involved.
+  c->SetupBottomMostLevel(
+      vstorage,
+      (output_level == vstorage->num_levels() - 1),
+      (output_level == 0));
+  return c;
+}
+
+Status CompactionPicker::GetCompactionInputsFromFileNumbers(
+    autovector<CompactionInputFiles>* input_files,
+    std::unordered_set<uint64_t>* input_set,
+    const VersionStorageInfo* vstorage,
+    const CompactionOptions& compact_options) const {
+  if (input_set->size() == 0U) {
+    return Status::InvalidArgument(
+        "Compaction must include at least one file.");
+  }
+  assert(input_files);
+
+  autovector<CompactionInputFiles> matched_input_files;
+  matched_input_files.resize(vstorage->num_levels());
+  int first_non_empty_level = -1;
+  int last_non_empty_level = -1;
+  // TODO(yhchiang): use a lazy-initialized mapping from
+  //                 file_number to FileMetaData in Version.
+  for (int level = 0; level < vstorage->num_levels(); ++level) {
+    for (auto file : vstorage->LevelFiles(level)) {
+      auto iter = input_set->find(file->fd.GetNumber());
+      if (iter != input_set->end()) {
+        matched_input_files[level].files.push_back(file);
+        input_set->erase(iter);
+        last_non_empty_level = level;
+        if (first_non_empty_level == -1) {
+          first_non_empty_level = level;
+        }
+      }
+    }
+  }
+
+  if (!input_set->empty()) {
+    std::string message(
+        "Cannot find matched SST files for the following file numbers:");
+    for (auto fn : *input_set) {
+      message += " ";
+      message += std::to_string(fn);
+    }
+    return Status::InvalidArgument(message);
+  }
+
+  for (int level = first_non_empty_level;
+       level <= last_non_empty_level; ++level) {
+    matched_input_files[level].level = level;
+    input_files->emplace_back(std::move(matched_input_files[level]));
+  }
+
+  return Status::OK();
+}
+
+
+
 // Returns true if any one of the parent files are being compacted
 bool CompactionPicker::ParentRangeInCompaction(VersionStorageInfo* vstorage,
                                                const InternalKey* smallest,
@@ -362,6 +446,247 @@ Compaction* CompactionPicker::CompactRange(
   return c;
 }
 
+#ifndef ROCKSDB_LITE
+namespace {
+// Test whether two files have overlapping key-ranges.
+bool HaveOverlappingKeyRanges(
+    const Comparator* c,
+    const SstFileMetaData& a, const SstFileMetaData& b) {
+  if (c->Compare(a.smallestkey, b.smallestkey) >= 0) {
+    if (c->Compare(a.smallestkey, b.largestkey) <= 0) {
+      // b.smallestkey <= a.smallestkey <= b.largestkey
+      return true;
+    }
+  } else if (c->Compare(a.largestkey, b.smallestkey) >= 0) {
+    // a.smallestkey < b.smallestkey <= a.largestkey
+    return true;
+  }
+  if (c->Compare(a.largestkey, b.largestkey) <= 0) {
+    if (c->Compare(a.largestkey, b.smallestkey) >= 0) {
+      // b.smallestkey <= a.largestkey <= b.largestkey
+      return true;
+    }
+  } else if (c->Compare(a.smallestkey, b.largestkey) <= 0) {
+    // a.smallestkey <= b.largestkey < a.largestkey
+    return true;
+  }
+  return false;
+}
+}  // namespace
+
+Status CompactionPicker::SanitizeCompactionInputFilesForAllLevels(
+      std::unordered_set<uint64_t>* input_files,
+      const ColumnFamilyMetaData& cf_meta,
+      const int output_level) const {
+  auto& levels = cf_meta.levels;
+  auto comparator = icmp_->user_comparator();
+
+  // TODO(yhchiang): If there is any input files of L1 or up and there
+  // is at least one L0 files. All L0 files older than the L0 file needs
+  // to be included. Otherwise, it is a false conditoin
+
+  // TODO(yhchiang): add is_adjustable to CompactionOptions
+
+  // the smallest and largest key of the current compaction input
+  std::string smallestkey;
+  std::string largestkey;
+  // a flag for initializing smallest and largest key
+  bool is_first = false;
+  const int kNotFound = -1;
+
+  // For each level, it does the following things:
+  // 1. Find the first and the last compaction input files
+  //    in the current level.
+  // 2. Include all files between the first and the last
+  //    compaction input files.
+  // 3. Update the compaction key-range.
+  // 4. For all remaining levels, include files that have
+  //    overlapping key-range with the compaction key-range.
+  for (int l = 0; l <= output_level; ++l) {
+    auto& current_files = levels[l].files;
+    int first_included = static_cast<int>(current_files.size());
+    int last_included = kNotFound;
+
+    // identify the first and the last compaction input files
+    // in the current level.
+    for (size_t f = 0; f < current_files.size(); ++f) {
+      if (input_files->find(TableFileNameToNumber(current_files[f].name)) !=
+          input_files->end()) {
+        first_included = std::min(first_included, static_cast<int>(f));
+        last_included = std::max(last_included, static_cast<int>(f));
+        if (is_first == false) {
+          smallestkey = current_files[f].smallestkey;
+          largestkey = current_files[f].largestkey;
+          is_first = true;
+        }
+      }
+    }
+    if (last_included == kNotFound) {
+      continue;
+    }
+
+    if (l != 0) {
+      // expend the compaction input of the current level if it
+      // has overlapping key-range with other non-compaction input
+      // files in the same level.
+      while (first_included > 0) {
+        if (comparator->Compare(
+                current_files[first_included - 1].largestkey,
+                current_files[first_included].smallestkey) < 0) {
+          break;
+        }
+        first_included--;
+      }
+
+      while (last_included < static_cast<int>(current_files.size()) - 1) {
+        if (comparator->Compare(
+                current_files[last_included + 1].smallestkey,
+                current_files[last_included].largestkey) > 0) {
+          break;
+        }
+        last_included++;
+      }
+    }
+
+    // include all files between the first and the last compaction input files.
+    for (int f = first_included; f <= last_included; ++f) {
+      if (current_files[f].being_compacted) {
+        return Status::Aborted(
+            "Necessary compaction input file " + current_files[f].name +
+            " is currently being compacted.");
+      }
+      input_files->insert(
+          TableFileNameToNumber(current_files[f].name));
+    }
+
+    // update smallest and largest key
+    if (l == 0) {
+      for (int f = first_included; f <= last_included; ++f) {
+        if (comparator->Compare(
+            smallestkey, current_files[f].smallestkey) > 0) {
+          smallestkey = current_files[f].smallestkey;
+        }
+        if (comparator->Compare(
+            largestkey, current_files[f].largestkey) < 0) {
+          largestkey = current_files[f].largestkey;
+        }
+      }
+    } else {
+      if (comparator->Compare(
+          smallestkey, current_files[first_included].smallestkey) > 0) {
+        smallestkey = current_files[first_included].smallestkey;
+      }
+      if (comparator->Compare(
+          largestkey, current_files[last_included].largestkey) < 0) {
+        largestkey = current_files[last_included].largestkey;
+      }
+    }
+
+    SstFileMetaData aggregated_file_meta;
+    aggregated_file_meta.smallestkey = smallestkey;
+    aggregated_file_meta.largestkey = largestkey;
+
+    // For all lower levels, include all overlapping files.
+    for (int m = l + 1; m <= output_level; ++m) {
+      for (auto& next_lv_file : levels[m].files) {
+        if (HaveOverlappingKeyRanges(
+            comparator, aggregated_file_meta, next_lv_file)) {
+          if (next_lv_file.being_compacted) {
+            return Status::Aborted(
+                "File " + next_lv_file.name +
+                " that has overlapping key range with one of the compaction "
+                " input file is currently being compacted.");
+          }
+          input_files->insert(
+              TableFileNameToNumber(next_lv_file.name));
+        }
+      }
+    }
+  }
+  return Status::OK();
+}
+
+Status CompactionPicker::SanitizeCompactionInputFiles(
+    std::unordered_set<uint64_t>* input_files,
+    const ColumnFamilyMetaData& cf_meta,
+    const int output_level) const {
+  assert(static_cast<int>(cf_meta.levels.size()) - 1 ==
+         cf_meta.levels[cf_meta.levels.size() - 1].level);
+  if (output_level >= static_cast<int>(cf_meta.levels.size())) {
+    return Status::InvalidArgument(
+        "Output level for column family " + cf_meta.name +
+        " must between [0, " +
+        std::to_string(cf_meta.levels[cf_meta.levels.size() - 1].level) +
+        "].");
+  }
+
+  if (output_level > MaxOutputLevel()) {
+    return Status::InvalidArgument(
+        "Exceed the maximum output level defined by "
+        "the current compaction algorithm --- " +
+            std::to_string(MaxOutputLevel()));
+  }
+
+  if (output_level < 0) {
+    return Status::InvalidArgument(
+        "Output level cannot be negative.");
+  }
+
+  if (input_files->size() == 0) {
+    return Status::InvalidArgument(
+        "A compaction must contain at least one file.");
+  }
+
+  Status s = SanitizeCompactionInputFilesForAllLevels(
+      input_files, cf_meta, output_level);
+
+  if (!s.ok()) {
+    return s;
+  }
+
+  // for all input files, check whether the file number matches
+  // any currently-existing files.
+  for (auto file_num : *input_files) {
+    bool found = false;
+    for (auto level_meta : cf_meta.levels) {
+      for (auto file_meta : level_meta.files) {
+        if (file_num == TableFileNameToNumber(file_meta.name)) {
+          if (file_meta.being_compacted) {
+            return Status::Aborted(
+                "Specified compaction input file " +
+                MakeTableFileName("", file_num) +
+                " is already being compacted.");
+          }
+          found = true;
+          break;
+        }
+      }
+      if (found) {
+        break;
+      }
+    }
+    if (!found) {
+      return Status::InvalidArgument(
+          "Specified compaction input file " +
+          MakeTableFileName("", file_num) +
+          " does not exist in column family " + cf_meta.name + ".");
+    }
+  }
+
+  return Status::OK();
+}
+#endif  // ROCKSDB_LITE
+
+bool LevelCompactionPicker::NeedsCompaction(const VersionStorageInfo* vstorage)
+    const {
+  for (int i = 0; i <= vstorage->MaxInputLevel(); i++) {
+    if (vstorage->CompactionScore(i) >= 1) {
+      return true;
+    }
+  }
+  return false;
+}
+
 Compaction* LevelCompactionPicker::PickCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
@@ -516,6 +841,12 @@ Compaction* LevelCompactionPicker::PickCompactionBySize(
   return c;
 }
 
+bool UniversalCompactionPicker::NeedsCompaction(
+    const VersionStorageInfo* vstorage) const {
+  const int kLevel0 = 0;
+  return vstorage->CompactionScore(kLevel0) >= 1;
+}
+
 // Universal style of compaction. Pick files that are contiguous in
 // time-range to compact.
 //
@@ -557,7 +888,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(
       // If max read amplification is exceeding configured limits, then force
       // compaction without looking at filesize ratios and try to reduce
       // the number of files to fewer than level0_file_num_compaction_trigger.
-      unsigned int num_files = level_files.size() -
+      unsigned int num_files =
+          static_cast<unsigned int>(level_files.size()) -
           mutable_cf_options.level0_file_num_compaction_trigger;
       if ((c = PickCompactionUniversalReadAmp(
                cf_name, mutable_cf_options, vstorage, score, UINT_MAX,
@@ -761,8 +1093,7 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp(
   if (ratio_to_compress >= 0) {
     uint64_t total_size = vstorage->NumLevelBytes(kLevel0);
     uint64_t older_file_size = 0;
-    for (unsigned int i = files.size() - 1;
-         i >= first_index_after; i--) {
+    for (size_t i = files.size() - 1; i >= first_index_after; i--) {
       older_file_size += files[i]->fd.GetFileSize();
       if (older_file_size * 100L >= total_size * (long) ratio_to_compress) {
         enable_compression = false;
@@ -915,6 +1246,12 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp(
   return c;
 }
 
+bool FIFOCompactionPicker::NeedsCompaction(const VersionStorageInfo* vstorage)
+    const {
+  const int kLevel0 = 0;
+  return vstorage->CompactionScore(kLevel0) >= 1;
+}
+
 Compaction* FIFOCompactionPicker::PickCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
@@ -923,7 +1260,7 @@ Compaction* FIFOCompactionPicker::PickCompaction(
   const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
   uint64_t total_size = 0;
   for (const auto& file : level_files) {
-    total_size += file->compensated_file_size;
+    total_size += file->fd.file_size;
   }
 
   if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size ||

db/compaction_picker.h

@@ -8,6 +8,11 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #pragma once
+#include <vector>
+#include <memory>
+#include <set>
+#include <unordered_set>
+
 #include "db/version_set.h"
 #include "db/compaction.h"
 #include "rocksdb/status.h"
@@ -25,6 +30,7 @@ namespace rocksdb {
 class LogBuffer;
 class Compaction;
 class VersionStorageInfo;
+struct CompactionInputFiles;
 
 class CompactionPicker {
  public:
@@ -62,6 +68,26 @@ class CompactionPicker {
   // for compaction input.
   virtual int MaxInputLevel(int current_num_levels) const = 0;
 
+  // The maximum allowed output level.  Default value is NumberLevels() - 1.
+  virtual int MaxOutputLevel() const {
+    return NumberLevels() - 1;
+  }
+
+  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const = 0;
+
+  // Sanitize the input set of compaction input files.
+  // When the input parameters do not describe a valid compaction, the
+  // function will try to fix the input_files by adding necessary
+  // files.  If it's not possible to conver an invalid input_files
+  // into a valid one by adding more files, the function will return a
+  // non-ok status with specific reason.
+#ifndef ROCKSDB_LITE
+  Status SanitizeCompactionInputFiles(
+      std::unordered_set<uint64_t>* input_files,
+      const ColumnFamilyMetaData& cf_meta,
+      const int output_level) const;
+#endif  // ROCKSDB_LITE
+
   // Free up the files that participated in a compaction
   void ReleaseCompactionFiles(Compaction* c, Status status);
 
@@ -69,6 +95,24 @@ class CompactionPicker {
   // compactions per level
   void SizeBeingCompacted(std::vector<uint64_t>& sizes);
 
+  // Returns true if any one of the specified files are being compacted
+  bool FilesInCompaction(const std::vector<FileMetaData*>& files);
+
+  // Takes a list of CompactionInputFiles and returns a Compaction object.
+  Compaction* FormCompaction(
+      const CompactionOptions& compact_options,
+      const autovector<CompactionInputFiles>& input_files,
+      int output_level, VersionStorageInfo* vstorage,
+      const MutableCFOptions& mutable_cf_options) const;
+
+  // Converts a set of compaction input file numbers into
+  // a list of CompactionInputFiles.
+  Status GetCompactionInputsFromFileNumbers(
+      autovector<CompactionInputFiles>* input_files,
+      std::unordered_set<uint64_t>* input_set,
+      const VersionStorageInfo* vstorage,
+      const CompactionOptions& compact_options) const;
+
  protected:
   int NumberLevels() const { return ioptions_.num_levels; }
 
@@ -98,9 +142,6 @@ class CompactionPicker {
   bool ExpandWhileOverlapping(const std::string& cf_name,
                               VersionStorageInfo* vstorage, Compaction* c);
 
-  // Returns true if any one of the specified files are being compacted
-  bool FilesInCompaction(std::vector<FileMetaData*>& files);
-
   // Returns true if any one of the parent files are being compacted
   bool ParentRangeInCompaction(VersionStorageInfo* vstorage,
                                const InternalKey* smallest,
@@ -113,11 +154,18 @@ class CompactionPicker {
 
   const ImmutableCFOptions& ioptions_;
 
+  // A helper function to SanitizeCompactionInputFiles() that
+  // sanitizes "input_files" by adding necessary files.
+#ifndef ROCKSDB_LITE
+  virtual Status SanitizeCompactionInputFilesForAllLevels(
+      std::unordered_set<uint64_t>* input_files,
+      const ColumnFamilyMetaData& cf_meta,
+      const int output_level) const;
+#endif  // ROCKSDB_LITE
+
   // record all the ongoing compactions for all levels
   std::vector<std::set<Compaction*>> compactions_in_progress_;
 
-
- private:
   const InternalKeyComparator* const icmp_;
 };
 
@@ -131,11 +179,19 @@ class UniversalCompactionPicker : public CompactionPicker {
                                      VersionStorageInfo* vstorage,
                                      LogBuffer* log_buffer) override;
 
-  // The maxinum allowed input level.  Always return 0.
+  // The maxinum allowed input level.  Always returns 0.
   virtual int MaxInputLevel(int current_num_levels) const override {
     return 0;
   }
 
+  // The maximum allowed output level.  Always returns 0.
+  virtual int MaxOutputLevel() const override {
+    return 0;
+  }
+
+  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
+      override;
+
  private:
   // Pick Universal compaction to limit read amplification
   Compaction* PickCompactionUniversalReadAmp(
@@ -170,6 +226,9 @@ class LevelCompactionPicker : public CompactionPicker {
     return current_num_levels - 2;
   }
 
+  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
+      override;
+
  private:
   // For the specfied level, pick a compaction.
   // Returns nullptr if there is no compaction to be done.
@@ -197,10 +256,55 @@ class FIFOCompactionPicker : public CompactionPicker {
       uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
       InternalKey** compaction_end) override;
 
-  // The maxinum allowed input level.  Always return 0.
+  // The maxinum allowed input level.  Always returns 0.
   virtual int MaxInputLevel(int current_num_levels) const override {
     return 0;
   }
+
+  // The maximum allowed output level.  Always returns 0.
+  virtual int MaxOutputLevel() const override {
+    return 0;
+  }
+
+  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
+      override;
+};
+
+class NullCompactionPicker : public CompactionPicker {
+ public:
+  NullCompactionPicker(const ImmutableCFOptions& ioptions,
+                       const InternalKeyComparator* icmp) :
+      CompactionPicker(ioptions, icmp) {}
+  virtual ~NullCompactionPicker() {}
+
+  // Always return "nullptr"
+  Compaction* PickCompaction(const std::string& cf_name,
+                             const MutableCFOptions& mutable_cf_options,
+                             VersionStorageInfo* vstorage,
+                             LogBuffer* log_buffer) override {
+    return nullptr;
+  }
+
+  // Always return "nullptr"
+  Compaction* CompactRange(
+      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+      VersionStorageInfo* vstorage, int input_level, int output_level,
+      uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
+      InternalKey** compaction_end) override {
+    return nullptr;
+  }
+
+  // Given the current number of levels, returns the highest allowed level
+  // for compaction input.
+  virtual int MaxInputLevel(int current_num_levels) const {
+    return current_num_levels - 2;
+  }
+
+  // Always returns false.
+  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
+      override {
+    return false;
+  }
 };
 
 // Utility function

db/compaction_picker_test.cc

@@ -19,113 +19,126 @@ class CountingLogger : public Logger {
 
 class CompactionPickerTest {
  public:
-  const Comparator* ucmp;
-  InternalKeyComparator icmp;
-  Options options;
-  ImmutableCFOptions ioptions;
-  MutableCFOptions mutable_cf_options;
+  const Comparator* ucmp_;
+  InternalKeyComparator icmp_;
+  Options options_;
+  ImmutableCFOptions ioptions_;
+  MutableCFOptions mutable_cf_options_;
   LevelCompactionPicker level_compaction_picker;
-  std::string cf_name;
-  CountingLogger logger;
-  LogBuffer log_buffer;
-  VersionStorageInfo vstorage;
-  uint32_t file_num;
-  CompactionOptionsFIFO fifo_options;
-  std::vector<uint64_t> size_being_compacted;
+  std::string cf_name_;
+  CountingLogger logger_;
+  LogBuffer log_buffer_;
+  uint32_t file_num_;
+  CompactionOptionsFIFO fifo_options_;
+  std::vector<uint64_t> size_being_compacted_;
+  std::unique_ptr<VersionStorageInfo> vstorage_;
+  std::vector<std::unique_ptr<FileMetaData>> files_;
 
   CompactionPickerTest()
-      : ucmp(BytewiseComparator()),
-        icmp(ucmp),
-        ioptions(options),
-        mutable_cf_options(options, ioptions),
-        level_compaction_picker(ioptions, &icmp),
-        cf_name("dummy"),
-        log_buffer(InfoLogLevel::INFO_LEVEL, &logger),
-        vstorage(&icmp, ucmp, options.num_levels, kCompactionStyleLevel,
-                 nullptr),
-        file_num(1) {
-    fifo_options.max_table_files_size = 1;
-    mutable_cf_options.RefreshDerivedOptions(ioptions);
-    size_being_compacted.resize(options.num_levels);
+      : ucmp_(BytewiseComparator()),
+        icmp_(ucmp_),
+        ioptions_(options_),
+        mutable_cf_options_(options_, ioptions_),
+        level_compaction_picker(ioptions_, &icmp_),
+        cf_name_("dummy"),
+        log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_),
+        file_num_(1),
+        vstorage_(nullptr) {
+    fifo_options_.max_table_files_size = 1;
+    mutable_cf_options_.RefreshDerivedOptions(ioptions_);
+    size_being_compacted_.resize(options_.num_levels);
   }
 
   ~CompactionPickerTest() {
-    for (int i = 0; i < vstorage.num_levels(); i++) {
-      for (auto* f : vstorage.LevelFiles(i)) {
-        delete f;
-      }
-    }
+  }
+
+  void NewVersionStorage(int num_levels, CompactionStyle style) {
+    DeleteVersionStorage();
+    options_.num_levels = num_levels;
+    vstorage_.reset(new VersionStorageInfo(
+        &icmp_, ucmp_, options_.num_levels, style, nullptr));
+  }
+
+  void DeleteVersionStorage() {
+    vstorage_.reset();
+    files_.clear();
   }
 
   void Add(int level, uint32_t file_number, const char* smallest,
            const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
            SequenceNumber smallest_seq = 100,
            SequenceNumber largest_seq = 100) {
-    assert(level < vstorage.num_levels());
+    assert(level < vstorage_->num_levels());
     FileMetaData* f = new FileMetaData;
     f->fd = FileDescriptor(file_number, path_id, file_size);
     f->smallest = InternalKey(smallest, smallest_seq, kTypeValue);
     f->largest = InternalKey(largest, largest_seq, kTypeValue);
     f->compensated_file_size = file_size;
     f->refs = 0;
-    vstorage.MaybeAddFile(level, f);
+    vstorage_->AddFile(level, f);
+    files_.emplace_back(f);
   }
 
   void UpdateVersionStorageInfo() {
-    vstorage.ComputeCompactionScore(mutable_cf_options, fifo_options,
-                                    size_being_compacted);
-    vstorage.UpdateFilesBySize();
-    vstorage.UpdateNumNonEmptyLevels();
-    vstorage.GenerateFileIndexer();
-    vstorage.GenerateLevelFilesBrief();
-    vstorage.SetFinalized();
+    vstorage_->ComputeCompactionScore(mutable_cf_options_, fifo_options_,
+                                    size_being_compacted_);
+    vstorage_->UpdateFilesBySize();
+    vstorage_->UpdateNumNonEmptyLevels();
+    vstorage_->GenerateFileIndexer();
+    vstorage_->GenerateLevelFilesBrief();
+    vstorage_->SetFinalized();
   }
 };
 
 TEST(CompactionPickerTest, Empty) {
+  NewVersionStorage(6, kCompactionStyleLevel);
   UpdateVersionStorageInfo();
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name, mutable_cf_options, &vstorage, &log_buffer));
+      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
   ASSERT_TRUE(compaction.get() == nullptr);
 }
 
 TEST(CompactionPickerTest, Single) {
-  mutable_cf_options.level0_file_num_compaction_trigger = 2;
+  NewVersionStorage(6, kCompactionStyleLevel);
+  mutable_cf_options_.level0_file_num_compaction_trigger = 2;
   Add(0, 1U, "p", "q");
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name, mutable_cf_options, &vstorage, &log_buffer));
+      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
   ASSERT_TRUE(compaction.get() == nullptr);
 }
 
 TEST(CompactionPickerTest, Level0Trigger) {
-  mutable_cf_options.level0_file_num_compaction_trigger = 2;
+  NewVersionStorage(6, kCompactionStyleLevel);
+  mutable_cf_options_.level0_file_num_compaction_trigger = 2;
   Add(0, 1U, "150", "200");
   Add(0, 2U, "200", "250");
 
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name, mutable_cf_options, &vstorage, &log_buffer));
+      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
   ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2, compaction->num_input_files(0));
+  ASSERT_EQ(2U, compaction->num_input_files(0));
   ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
   ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
 }
 
 TEST(CompactionPickerTest, Level1Trigger) {
+  NewVersionStorage(6, kCompactionStyleLevel);
   Add(1, 66U, "150", "200", 1000000000U);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name, mutable_cf_options, &vstorage, &log_buffer));
+      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
   ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1, compaction->num_input_files(0));
+  ASSERT_EQ(1U, compaction->num_input_files(0));
   ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
 }
 
 TEST(CompactionPickerTest, Level1Trigger2) {
+  NewVersionStorage(6, kCompactionStyleLevel);
   Add(1, 66U, "150", "200", 1000000001U);
   Add(1, 88U, "201", "300", 1000000000U);
   Add(2, 6U, "150", "179", 1000000000U);
@@ -134,18 +147,19 @@ TEST(CompactionPickerTest, Level1Trigger2) {
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name, mutable_cf_options, &vstorage, &log_buffer));
+      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
   ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1, compaction->num_input_files(0));
-  ASSERT_EQ(2, compaction->num_input_files(1));
+  ASSERT_EQ(1U, compaction->num_input_files(0));
+  ASSERT_EQ(2U, compaction->num_input_files(1));
   ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
   ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
   ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
 }
 
 TEST(CompactionPickerTest, LevelMaxScore) {
-  mutable_cf_options.target_file_size_base = 10000000;
-  mutable_cf_options.target_file_size_multiplier = 10;
+  NewVersionStorage(6, kCompactionStyleLevel);
+  mutable_cf_options_.target_file_size_base = 10000000;
+  mutable_cf_options_.target_file_size_multiplier = 10;
   Add(0, 1U, "150", "200", 1000000000U);
   // Level 1 score 1.2
   Add(1, 66U, "150", "200", 6000000U);
@@ -162,12 +176,83 @@ TEST(CompactionPickerTest, LevelMaxScore) {
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name, mutable_cf_options, &vstorage, &log_buffer));
+      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
   ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1, compaction->num_input_files(0));
+  ASSERT_EQ(1U, compaction->num_input_files(0));
   ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
 }
 
+TEST(CompactionPickerTest, NeedsCompactionLevel) {
+  const int kLevels = 6;
+  const int kFileCount = 20;
+  for (int level = 0; level < kLevels - 1; ++level) {
+    uint64_t file_size =
+        mutable_cf_options_.MaxBytesForLevel(level) * 2 / kFileCount;
+    for (int file_count = 1; file_count <= kFileCount; ++file_count) {
+      // start a brand new version in each test.
+      NewVersionStorage(kLevels, kCompactionStyleLevel);
+      for (int i = 0; i < file_count; ++i) {
+        Add(level, i, std::to_string((i + 100) * 1000).c_str(),
+            std::to_string((i + 100) * 1000 + 999).c_str(),
+            file_size, 0, i * 100, i * 100 + 99);
+      }
+      UpdateVersionStorageInfo();
+      ASSERT_EQ(vstorage_->CompactionScoreLevel(0), level);
+      ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
+                vstorage_->CompactionScore(0) >= 1);
+      // release the version storage
+      DeleteVersionStorage();
+    }
+  }
+}
+
+TEST(CompactionPickerTest, NeedsCompactionUniversal) {
+  NewVersionStorage(1, kCompactionStyleUniversal);
+  UniversalCompactionPicker universal_compaction_picker(
+      ioptions_, &icmp_);
+  // must return false when there's no files.
+  ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
+            false);
+
+  // verify the trigger given different number of L0 files.
+  for (int i = 1;
+       i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) {
+    Add(0, i, std::to_string((i + 100) * 1000).c_str(),
+        std::to_string((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100,
+        i * 100 + 99);
+    ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
+              vstorage_->CompactionScore(0) >= 1);
+  }
+}
+
+TEST(CompactionPickerTest, NeedsCompactionFIFO) {
+  NewVersionStorage(1, kCompactionStyleFIFO);
+  const int kFileCount =
+      mutable_cf_options_.level0_file_num_compaction_trigger * 3;
+  const uint64_t kFileSize = 100000;
+  const uint64_t kMaxSize = kFileSize * kFileCount / 2;
+
+  fifo_options_.max_table_files_size = kMaxSize;
+  ioptions_.compaction_options_fifo = fifo_options_;
+  FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
+
+  // must return false when there's no files.
+  ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), false);
+
+  // verify whether compaction is needed based on the current
+  // size of L0 files.
+  uint64_t current_size = 0;
+  for (int i = 1; i <= kFileCount; ++i) {
+    Add(0, i, std::to_string((i + 100) * 1000).c_str(),
+        std::to_string((i + 100) * 1000 + 999).c_str(),
+        kFileSize, 0, i * 100, i * 100 + 99);
+    current_size += kFileSize;
+    ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
+              vstorage_->CompactionScore(0) >= 1);
+  }
+}
+
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

db/comparator_db_test.cc

@@ -82,7 +82,7 @@ void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
     }
 
     int type = rnd->Uniform(2);
-    int index = rnd->Uniform(source_strings.size());
+    int index = rnd->Uniform(static_cast<int>(source_strings.size()));
     auto& key = source_strings[index];
     switch (type) {
       case 0:
@@ -124,7 +124,7 @@ void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
         break;
       case 2: {
         // Seek to random key
-        auto key_idx = rnd->Uniform(source_strings.size());
+        auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size()));
         auto key = source_strings[key_idx];
         iter->Seek(key);
         result_iter->Seek(key);
@@ -150,7 +150,7 @@ void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
         break;
       default: {
         assert(type == 5);
-        auto key_idx = rnd->Uniform(source_strings.size());
+        auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size()));
         auto key = source_strings[key_idx];
         std::string result;
         auto status = db->Get(ReadOptions(), key, &result);
@@ -325,7 +325,7 @@ TEST(ComparatorDBTest, SimpleSuffixReverseComparator) {
       source_prefixes.push_back(test::RandomHumanReadableString(&rnd, 8));
     }
     for (int j = 0; j < 20; j++) {
-      int prefix_index = rnd.Uniform(source_prefixes.size());
+      int prefix_index = rnd.Uniform(static_cast<int>(source_prefixes.size()));
       std::string key = source_prefixes[prefix_index] +
                         test::RandomHumanReadableString(&rnd, rnd.Uniform(8));
       source_strings.push_back(key);

db/corruption_test.cc

@@ -115,8 +115,8 @@ class CorruptionTest {
         continue;
       }
       missed += (key - next_expected);
-      next_expected = key + 1;
-      if (iter->value() != Value(key, &value_space)) {
+      next_expected = static_cast<unsigned int>(key + 1);
+      if (iter->value() != Value(static_cast<int>(key), &value_space)) {
         bad_values++;
       } else {
         correct++;
@@ -143,14 +143,14 @@ class CorruptionTest {
       if (-offset > sbuf.st_size) {
         offset = 0;
       } else {
-        offset = sbuf.st_size + offset;
+        offset = static_cast<int>(sbuf.st_size + offset);
       }
     }
     if (offset > sbuf.st_size) {
-      offset = sbuf.st_size;
+      offset = static_cast<int>(sbuf.st_size);
     }
     if (offset + bytes_to_corrupt > sbuf.st_size) {
-      bytes_to_corrupt = sbuf.st_size - offset;
+      bytes_to_corrupt = static_cast<int>(sbuf.st_size - offset);
     }
 
     // Do it
@@ -177,7 +177,7 @@ class CorruptionTest {
           type == filetype &&
           static_cast<int>(number) > picked_number) {  // Pick latest file
         fname = dbname_ + "/" + filenames[i];
-        picked_number = number;
+        picked_number = static_cast<int>(number);
       }
     }
     ASSERT_TRUE(!fname.empty()) << filetype;
@@ -246,7 +246,8 @@ TEST(CorruptionTest, RecoverWriteError) {
 TEST(CorruptionTest, NewFileErrorDuringWrite) {
   // Do enough writing to force minor compaction
   env_.writable_file_error_ = true;
-  const int num = 3 + (Options().write_buffer_size / kValueSize);
+  const int num =
+      static_cast<int>(3 + (Options().write_buffer_size / kValueSize));
   std::string value_storage;
   Status s;
   bool failed = false;

db/cuckoo_table_db_test.cc

@@ -92,7 +92,7 @@ class CuckooTableDBTest {
   // Return spread of files per level
   std::string FilesPerLevel() {
     std::string result;
-    int last_non_zero_offset = 0;
+    size_t last_non_zero_offset = 0;
     for (int level = 0; level < db_->NumberLevels(); level++) {
       int f = NumTableFilesAtLevel(level);
       char buf[100];

db/db_bench.cc

@@ -251,7 +251,8 @@ DEFINE_int32(universal_compression_size_percent, -1,
 DEFINE_int64(cache_size, -1, "Number of bytes to use as a cache of uncompressed"
              "data. Negative means use default settings.");
 
-DEFINE_int32(block_size, rocksdb::BlockBasedTableOptions().block_size,
+DEFINE_int32(block_size,
+             static_cast<int32_t>(rocksdb::BlockBasedTableOptions().block_size),
              "Number of bytes in a block.");
 
 DEFINE_int32(block_restart_interval,
@@ -1877,14 +1878,15 @@ class Benchmark {
       exit(1);
     }
     switch (FLAGS_rep_factory) {
-      case kPrefixHash:
-        options.memtable_factory.reset(NewHashSkipListRepFactory(
-            FLAGS_hash_bucket_count));
-        break;
       case kSkipList:
         options.memtable_factory.reset(new SkipListFactory(
             FLAGS_skip_list_lookahead));
         break;
+#ifndef ROCKSDB_LITE
+      case kPrefixHash:
+        options.memtable_factory.reset(
+            NewHashSkipListRepFactory(FLAGS_hash_bucket_count));
+        break;
       case kHashLinkedList:
         options.memtable_factory.reset(NewHashLinkListRepFactory(
             FLAGS_hash_bucket_count));
@@ -1898,8 +1900,14 @@ class Benchmark {
         options.memtable_factory.reset(NewHashCuckooRepFactory(
             options.write_buffer_size, FLAGS_key_size + FLAGS_value_size));
         break;
+#else
+      default:
+        fprintf(stderr, "Only skip list is supported in lite mode\n");
+        exit(1);
+#endif  // ROCKSDB_LITE
     }
     if (FLAGS_use_plain_table) {
+#ifndef ROCKSDB_LITE
       if (FLAGS_rep_factory != kPrefixHash &&
           FLAGS_rep_factory != kHashLinkedList) {
         fprintf(stderr, "Waring: plain table is used with skipList\n");
@@ -1920,7 +1928,12 @@ class Benchmark {
       plain_table_options.hash_table_ratio = 0.75;
       options.table_factory = std::shared_ptr<TableFactory>(
           NewPlainTableFactory(plain_table_options));
+#else
+      fprintf(stderr, "Plain table is not supported in lite mode\n");
+      exit(1);
+#endif  // ROCKSDB_LITE
     } else if (FLAGS_use_cuckoo_table) {
+#ifndef ROCKSDB_LITE
       if (FLAGS_cuckoo_hash_ratio > 1 || FLAGS_cuckoo_hash_ratio < 0) {
         fprintf(stderr, "Invalid cuckoo_hash_ratio\n");
         exit(1);
@@ -1930,6 +1943,10 @@ class Benchmark {
       table_options.identity_as_first_hash = FLAGS_identity_as_first_hash;
       options.table_factory = std::shared_ptr<TableFactory>(
           NewCuckooTableFactory(table_options));
+#else
+      fprintf(stderr, "Cuckoo table is not supported in lite mode\n");
+      exit(1);
+#endif  // ROCKSDB_LITE
     } else {
       BlockBasedTableOptions block_based_options;
       if (FLAGS_use_hash_search) {
@@ -2111,8 +2128,9 @@ class Benchmark {
         for (uint64_t i = 0; i < num_; ++i) {
           values_[i] = i;
         }
-        std::shuffle(values_.begin(), values_.end(),
-            std::default_random_engine(FLAGS_seed));
+        std::shuffle(
+            values_.begin(), values_.end(),
+            std::default_random_engine(static_cast<unsigned int>(FLAGS_seed)));
       }
     }
 
@@ -2220,7 +2238,10 @@ class Benchmark {
   }
 
   void ReadSequential(ThreadState* thread, DB* db) {
-    Iterator* iter = db->NewIterator(ReadOptions(FLAGS_verify_checksum, true));
+    ReadOptions options(FLAGS_verify_checksum, true);
+    options.tailing = FLAGS_use_tailing_iterator;
+
+    Iterator* iter = db->NewIterator(options);
     int64_t i = 0;
     int64_t bytes = 0;
     for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) {

db/db_impl.cc

@@ -64,6 +64,7 @@
 #include "util/autovector.h"
 #include "util/build_version.h"
 #include "util/coding.h"
+#include "util/db_info_dumper.h"
 #include "util/hash_skiplist_rep.h"
 #include "util/hash_linklist_rep.h"
 #include "util/logging.h"
@@ -213,7 +214,8 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
 #endif  // ROCKSDB_LITE
       bg_work_gate_closed_(false),
       refitting_level_(false),
-      opened_successfully_(false) {
+      opened_successfully_(false),
+      notifying_events_(0) {
   env_->GetAbsolutePath(dbname, &db_absolute_path_);
 
   // Reserve ten files or so for other uses and give the rest to TableCache.
@@ -239,6 +241,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
 
 DBImpl::~DBImpl() {
   mutex_.Lock();
+
   if (flush_on_destroy_) {
     for (auto cfd : *versions_->GetColumnFamilySet()) {
       if (!cfd->mem()->IsEmpty()) {
@@ -254,10 +257,10 @@ DBImpl::~DBImpl() {
 
   // Wait for background work to finish
   shutting_down_.store(true, std::memory_order_release);
-  while (bg_compaction_scheduled_ || bg_flush_scheduled_) {
+  while (bg_compaction_scheduled_ || bg_flush_scheduled_ || notifying_events_) {
     bg_cv_.Wait();
   }
-
+  listeners_.clear();
   flush_scheduler_.Clear();
 
   if (default_cf_handle_ != nullptr) {
@@ -443,8 +446,11 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
   }
 
   // don't delete live files
-  for (auto pair : pending_outputs_) {
-    job_context->sst_live.emplace_back(pair.first, pair.second, 0);
+  if (pending_outputs_.size()) {
+    job_context->min_pending_output = *pending_outputs_.begin();
+  } else {
+    // delete all of them
+    job_context->min_pending_output = std::numeric_limits<uint64_t>::max();
   }
   versions_->AddLiveFiles(&job_context->sst_live);
 
@@ -567,7 +573,10 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
         keep = (number >= state.manifest_file_number);
         break;
       case kTableFile:
-        keep = (sst_live_map.find(number) != sst_live_map.end());
+        // If the second condition is not there, this makes
+        // DontDeletePendingOutputs fail
+        keep = (sst_live_map.find(number) != sst_live_map.end()) ||
+               number >= state.min_pending_output;
         break;
       case kTempFile:
         // Any temp files that are currently being written to must
@@ -828,7 +837,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
     // The previous incarnation may not have written any MANIFEST
     // records after allocating this log number.  So we manually
     // update the file number allocation counter in VersionSet.
-    versions_->MarkFileNumberUsed(log_number);
+    versions_->MarkFileNumberUsedDuringRecovery(log_number);
     // Open the log file
     std::string fname = LogFileName(db_options_.wal_dir, log_number);
     unique_ptr<SequentialFile> file;
@@ -962,7 +971,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
       // not actually used. that is because VersionSet assumes
       // VersionSet::next_file_number_ always to be strictly greater than any
       // log number
-      versions_->MarkFileNumberUsed(max_log_number + 1);
+      versions_->MarkFileNumberUsedDuringRecovery(max_log_number + 1);
       status = versions_->LogAndApply(
           cfd, *cfd->GetLatestMutableCFOptions(), edit, &mutex_);
       if (!status.ok()) {
@@ -981,7 +990,8 @@ Status DBImpl::WriteLevel0TableForRecovery(ColumnFamilyData* cfd, MemTable* mem,
   const uint64_t start_micros = env_->NowMicros();
   FileMetaData meta;
   meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0);
-  pending_outputs_[meta.fd.GetNumber()] = 0;  // path 0 for level 0 file.
+  auto pending_outputs_inserted_elem =
+      CaptureCurrentFileNumberInPendingOutputs();
   ReadOptions ro;
   ro.total_order_seek = true;
   Arena arena;
@@ -1013,7 +1023,7 @@ Status DBImpl::WriteLevel0TableForRecovery(ColumnFamilyData* cfd, MemTable* mem,
         cfd->GetName().c_str(), meta.fd.GetNumber(), meta.fd.GetFileSize(),
         s.ToString().c_str());
   }
-  pending_outputs_.erase(meta.fd.GetNumber());
+  ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
 
   // Note that if file_size is zero, the file has been deleted and
   // should not be added to the manifest.
@@ -1044,11 +1054,12 @@ Status DBImpl::FlushMemTableToOutputFile(
 
   FlushJob flush_job(dbname_, cfd, db_options_, mutable_cf_options,
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     &pending_outputs_, snapshots_.GetNewest(), job_context,
-                     log_buffer, db_directory_.get(),
-                     GetCompressionFlush(*cfd->ioptions()), stats_);
+                     snapshots_.GetNewest(), job_context, log_buffer,
+                     db_directory_.get(), GetCompressionFlush(*cfd->ioptions()),
+                     stats_);
 
-  Status s = flush_job.Run();
+  uint64_t file_number;
+  Status s = flush_job.Run(&file_number);
 
   if (s.ok()) {
     InstallSuperVersionBackground(cfd, job_context, mutable_cf_options);
@@ -1078,9 +1089,42 @@ Status DBImpl::FlushMemTableToOutputFile(
     bg_error_ = s;
   }
   RecordFlushIOStats();
+#ifndef ROCKSDB_LITE
+  if (s.ok()) {
+    // may temporarily unlock and lock the mutex.
+    NotifyOnFlushCompleted(cfd, file_number);
+  }
+#endif  // ROCKSDB_LITE
   return s;
 }
 
+void DBImpl::NotifyOnFlushCompleted(
+    ColumnFamilyData* cfd, uint64_t file_number) {
+  mutex_.AssertHeld();
+  if (shutting_down_.load(std::memory_order_acquire)) {
+    return;
+  }
+  bool triggered_flush_slowdown =
+      (cfd->current()->storage_info()->NumLevelFiles(0) >=
+       cfd->options()->level0_slowdown_writes_trigger);
+  bool triggered_flush_stop =
+      (cfd->current()->storage_info()->NumLevelFiles(0) >=
+       cfd->options()->level0_stop_writes_trigger);
+  notifying_events_++;
+  // release lock while notifying events
+  mutex_.Unlock();
+  // TODO(yhchiang): make db_paths dynamic.
+  cfd->NotifyOnFlushCompleted(
+        this, MakeTableFileName(db_options_.db_paths[0].path, file_number),
+        triggered_flush_slowdown,
+        triggered_flush_stop);
+  mutex_.Lock();
+  notifying_events_--;
+  assert(notifying_events_ >= 0);
+  // no need to signal bg_cv_ as it will be signaled at the end of the
+  // flush process.
+}
+
 Status DBImpl::CompactRange(ColumnFamilyHandle* column_family,
                             const Slice* begin, const Slice* end,
                             bool reduce_level, int target_level,
@@ -1142,8 +1186,177 @@ Status DBImpl::CompactRange(ColumnFamilyHandle* column_family,
   return s;
 }
 
+Status DBImpl::CompactFiles(
+    const CompactionOptions& compact_options,
+    ColumnFamilyHandle* column_family,
+    const std::vector<std::string>& input_file_names,
+    const int output_level, const int output_path_id) {
+#ifdef ROCKSDB_LITE
+    // not supported in lite version
+  return Status::NotSupported("Not supported in ROCKSDB LITE");
+#else
+  MutexLock l(&mutex_);
+  if (column_family == nullptr) {
+    return Status::InvalidArgument("ColumnFamilyHandle must be non-null.");
+  }
+
+  auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
+  assert(cfd);
+  // TODO(yhchiang): use superversion
+  cfd->Ref();
+  auto version = cfd->current();
+  version->Ref();
+  auto s = CompactFilesImpl(compact_options, cfd, version,
+                            input_file_names, output_level, output_path_id);
+  // TODO(yhchiang): unref could move into CompactFilesImpl().  Otherwise,
+  // FindObsoleteFiles might never able to find any file to delete.
+  version->Unref();
+  // TODO(yhchiang): cfd should be deleted after its last reference.
+  cfd->Unref();
+  return s;
+#endif  // ROCKSDB_LITE
+}
+
+#ifndef ROCKSDB_LITE
+Status DBImpl::CompactFilesImpl(
+    const CompactionOptions& compact_options, ColumnFamilyData* cfd,
+    Version* version, const std::vector<std::string>& input_file_names,
+    const int output_level, int output_path_id) {
+  mutex_.AssertHeld();
+
+  LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
+
+  if (shutting_down_.load(std::memory_order_acquire)) {
+    return Status::ShutdownInProgress();
+  }
+
+  std::unordered_set<uint64_t> input_set;
+  for (auto file_name : input_file_names) {
+    input_set.insert(TableFileNameToNumber(file_name));
+  }
+
+  ColumnFamilyMetaData cf_meta;
+  // TODO(yhchiang): can directly use version here if none of the
+  // following functions call is pluggable to external developers.
+  version->GetColumnFamilyMetaData(&cf_meta);
+
+  if (output_path_id < 0) {
+    if (db_options_.db_paths.size() == 1U) {
+      output_path_id = 0;
+    } else {
+      return Status::NotSupported(
+          "Automatic output path selection is not "
+          "yet supported in CompactFiles()");
+    }
+  }
+
+  Status s = cfd->compaction_picker()->SanitizeCompactionInputFiles(
+      &input_set, cf_meta, output_level);
+  if (!s.ok()) {
+    return s;
+  }
+
+  autovector<CompactionInputFiles> input_files;
+  s = cfd->compaction_picker()->GetCompactionInputsFromFileNumbers(
+      &input_files, &input_set, version->storage_info(), compact_options);
+  if (!s.ok()) {
+    return s;
+  }
+
+  for (auto inputs : input_files) {
+    if (cfd->compaction_picker()->FilesInCompaction(inputs.files)) {
+      return Status::Aborted(
+          "Some of the necessary compaction input "
+          "files are already being compacted");
+    }
+  }
+
+  // At this point, CompactFiles will be run.
+  bg_compaction_scheduled_++;
+
+  unique_ptr<Compaction> c;
+  assert(cfd->compaction_picker());
+  c.reset(cfd->compaction_picker()->FormCompaction(
+        compact_options, input_files,
+        output_level, version->storage_info(),
+        *cfd->GetLatestMutableCFOptions()));
+  assert(c);
+  c->SetInputVersion(version);
+  c->SetOutputPathId(static_cast<uint32_t>(output_path_id));
+  // deletion compaction currently not allowed in CompactFiles.
+  assert(!c->IsDeletionCompaction());
+
+  JobContext job_context(true);
+  auto yield_callback = [&]() {
+    return CallFlushDuringCompaction(c->column_family_data(),
+                                     *c->mutable_cf_options(), &job_context,
+                                     &log_buffer);
+  };
+  CompactionJob compaction_job(c.get(), db_options_, *c->mutable_cf_options(),
+                               env_options_, versions_.get(), &shutting_down_,
+                               &log_buffer, db_directory_.get(), stats_,
+                               &snapshots_, IsSnapshotSupported(), table_cache_,
+                               std::move(yield_callback));
+  compaction_job.Prepare();
+
+  mutex_.Unlock();
+  Status status = compaction_job.Run();
+  mutex_.Lock();
+  compaction_job.Install(&status, &mutex_);
+  if (status.ok()) {
+    InstallSuperVersionBackground(c->column_family_data(), &job_context,
+                                  *c->mutable_cf_options());
+  }
+  c->ReleaseCompactionFiles(s);
+  c->ReleaseInputs();
+  c.reset();
+
+  if (status.ok()) {
+    // Done
+  } else if (status.IsShutdownInProgress()) {
+    // Ignore compaction errors found during shutting down
+  } else {
+    Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log, "Compaction error: %s",
+        status.ToString().c_str());
+    if (db_options_.paranoid_checks && bg_error_.ok()) {
+      bg_error_ = status;
+    }
+  }
+
+  // If !s.ok(), this means that Compaction failed. In that case, we want
+  // to delete all obsolete files we might have created and we force
+  // FindObsoleteFiles(). This is because job_context does not
+  // catch all created files if compaction failed.
+  // TODO(yhchiang): write an unit-test to make sure files are actually
+  //                 deleted after CompactFiles.
+  FindObsoleteFiles(&job_context, !s.ok());
+
+  // delete unnecessary files if any, this is done outside the mutex
+  if (job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) {
+    mutex_.Unlock();
+    // Have to flush the info logs before bg_compaction_scheduled_--
+    // because if bg_flush_scheduled_ becomes 0 and the lock is
+    // released, the deconstructor of DB can kick in and destroy all the
+    // states of DB so info_log might not be available after that point.
+    // It also applies to access other states that DB owns.
+    log_buffer.FlushBufferToLog();
+    if (job_context.HaveSomethingToDelete()) {
+      PurgeObsoleteFiles(job_context);
+    }
+    mutex_.Lock();
+  }
+
+  bg_compaction_scheduled_--;
+
+  return status;
+}
+#endif  // ROCKSDB_LITE
+
 Status DBImpl::SetOptions(ColumnFamilyHandle* column_family,
     const std::unordered_map<std::string, std::string>& options_map) {
+#ifdef ROCKSDB_LITE
+  return Status::NotSupported("Not supported in ROCKSDB LITE");
+#else
   auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
   if (options_map.empty()) {
     Log(InfoLogLevel::WARN_LEVEL,
@@ -1179,6 +1392,7 @@ Status DBImpl::SetOptions(ColumnFamilyHandle* column_family,
         "[%s] SetOptions failed", cfd->GetName().c_str());
   }
   return s;
+#endif  // ROCKSDB_LITE
 }
 
 // return the same level if it cannot be moved
@@ -1466,7 +1680,8 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
     bool is_compaction_needed = false;
     // no need to refcount since we're under a mutex
     for (auto cfd : *versions_->GetColumnFamilySet()) {
-      if (cfd->current()->storage_info()->NeedsCompaction()) {
+      if (cfd->compaction_picker()->NeedsCompaction(
+              cfd->current()->storage_info())) {
         is_compaction_needed = true;
         break;
       }
@@ -1550,6 +1765,9 @@ void DBImpl::BackgroundCallFlush() {
   {
     MutexLock l(&mutex_);
 
+    auto pending_outputs_inserted_elem =
+        CaptureCurrentFileNumberInPendingOutputs();
+
     Status s;
     if (!shutting_down_.load(std::memory_order_acquire)) {
       s = BackgroundFlush(&madeProgress, &job_context, &log_buffer);
@@ -1573,6 +1791,8 @@ void DBImpl::BackgroundCallFlush() {
       }
     }
 
+    ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
+
     // If !s.ok(), this means that Flush failed. In that case, we want
     // to delete all obsolete files and we force FindObsoleteFiles()
     FindObsoleteFiles(&job_context, !s.ok());
@@ -1616,6 +1836,10 @@ void DBImpl::BackgroundCallCompaction() {
   LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
   {
     MutexLock l(&mutex_);
+
+    auto pending_outputs_inserted_elem =
+        CaptureCurrentFileNumberInPendingOutputs();
+
     assert(bg_compaction_scheduled_);
     Status s;
     if (!shutting_down_.load(std::memory_order_acquire)) {
@@ -1640,6 +1864,8 @@ void DBImpl::BackgroundCallCompaction() {
       }
     }
 
+    ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
+
     // If !s.ok(), this means that Compaction failed. In that case, we want
     // to delete all obsolete files we might have created and we force
     // FindObsoleteFiles(). This is because job_context does not
@@ -1846,16 +2072,16 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
                                        *c->mutable_cf_options(), job_context,
                                        log_buffer);
     };
-    CompactionJob compaction_job(
-        c.get(), db_options_, *c->mutable_cf_options(), env_options_,
-        versions_.get(), &mutex_, &shutting_down_, &pending_outputs_,
-        log_buffer, db_directory_.get(), stats_, &snapshots_,
-        IsSnapshotSupported(), table_cache_, std::move(yield_callback));
+    CompactionJob compaction_job(c.get(), db_options_, *c->mutable_cf_options(),
+                                 env_options_, versions_.get(), &shutting_down_,
+                                 log_buffer, db_directory_.get(), stats_,
+                                 &snapshots_, IsSnapshotSupported(),
+                                 table_cache_, std::move(yield_callback));
     compaction_job.Prepare();
     mutex_.Unlock();
     status = compaction_job.Run();
     mutex_.Lock();
-    status = compaction_job.Install(status);
+    compaction_job.Install(&status, &mutex_);
     if (status.ok()) {
       InstallSuperVersionBackground(c->column_family_data(), job_context,
                                     *c->mutable_cf_options());
@@ -2039,7 +2265,7 @@ SuperVersion* DBImpl::InstallSuperVersion(
   MaybeScheduleFlushOrCompaction();
 
   // Update max_total_in_memory_state_
-  auto old_memtable_size = 0;
+  size_t old_memtable_size = 0;
   if (old) {
     old_memtable_size = old->mutable_cf_options.write_buffer_size *
                         old->mutable_cf_options.max_write_buffer_number;
@@ -2707,7 +2933,8 @@ Status DBImpl::DelayWrite(uint64_t expiration_time) {
   auto delay = write_controller_.GetDelay();
   if (write_controller_.IsStopped() == false && delay > 0) {
     mutex_.Unlock();
-    env_->SleepForMicroseconds(delay);
+    // hopefully we don't have to sleep more than 2 billion microseconds
+    env_->SleepForMicroseconds(static_cast<int>(delay));
     mutex_.Lock();
   }
 
@@ -2790,8 +3017,6 @@ Status DBImpl::SetNewMemtableAndNewLogFile(ColumnFamilyData* cfd,
   if (!s.ok()) {
     // how do we fail if we're not creating new log?
     assert(creating_new_log);
-    // Avoid chewing through file number space in a tight loop.
-    versions_->ReuseLogFileNumber(new_log_number);
     assert(!new_mem);
     assert(!new_log);
     return s;
@@ -2968,6 +3193,22 @@ void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,
   }
 }
 
+std::list<uint64_t>::iterator
+DBImpl::CaptureCurrentFileNumberInPendingOutputs() {
+  // We need to remember the iterator of our insert, because after the
+  // background job is done, we need to remove that element from
+  // pending_outputs_.
+  pending_outputs_.push_back(versions_->current_next_file_number());
+  auto pending_outputs_inserted_elem = pending_outputs_.end();
+  --pending_outputs_inserted_elem;
+  return pending_outputs_inserted_elem;
+}
+
+void DBImpl::ReleaseFileNumberFromPendingOutputs(
+    std::list<uint64_t>::iterator v) {
+  pending_outputs_.erase(v);
+}
+
 #ifndef ROCKSDB_LITE
 Status DBImpl::GetUpdatesSince(
     SequenceNumber seq, unique_ptr<TransactionLogIterator>* iter,
@@ -3078,6 +3319,17 @@ void DBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
   MutexLock l(&mutex_);
   versions_->GetLiveFilesMetaData(metadata);
 }
+
+void DBImpl::GetColumnFamilyMetaData(
+    ColumnFamilyHandle* column_family,
+    ColumnFamilyMetaData* cf_meta) {
+  assert(column_family);
+  auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
+  auto* sv = GetAndRefSuperVersion(cfd);
+  sv->current->GetColumnFamilyMetaData(cf_meta);
+  ReturnAndCleanupSuperVersion(cfd, sv);
+}
+
 #endif  // ROCKSDB_LITE
 
 Status DBImpl::CheckConsistency() {
@@ -3123,7 +3375,7 @@ Status DBImpl::GetDbIdentity(std::string& identity) {
   }
   char buffer[file_size];
   Slice id;
-  s = idfile->Read(file_size, &id, buffer);
+  s = idfile->Read(static_cast<size_t>(file_size), &id, buffer);
   if (!s.ok()) {
     return s;
   }
@@ -3328,6 +3580,15 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
   if (s.ok()) {
     impl->opened_successfully_ = true;
     *dbptr = impl;
+    // TODO(yhchiang): Add NotifyOnDatabaseOpen() here.
+    // Since the column-family handles are only available after DB::Open(),
+    // typically developers will need to pass the returned ColumnFamilyHandles
+    // to their EventListeners in order to maintain the mapping between
+    // column-family-name to ColumnFamilyHandle.  However, some database
+    // events might happen before the user passing those ColumnFamilyHandle to
+    // their Listeners.  To address this, we should have NotifyOnDatabaseOpen()
+    // here which passes the created ColumnFamilyHandle to the Listeners
+    // as the first event after DB::Open().
   } else {
     for (auto h : *handles) {
       delete h;

db/db_impl.h

@@ -12,7 +12,9 @@
 #include <deque>
 #include <limits>
 #include <set>
+#include <list>
 #include <utility>
+#include <list>
 #include <vector>
 #include <string>
 
@@ -114,6 +116,13 @@ class DBImpl : public DB {
                               bool reduce_level = false, int target_level = -1,
                               uint32_t target_path_id = 0);
 
+  using DB::CompactFiles;
+  virtual Status CompactFiles(
+      const CompactionOptions& compact_options,
+      ColumnFamilyHandle* column_family,
+      const std::vector<std::string>& input_file_names,
+      const int output_level, const int output_path_id = -1);
+
   using DB::SetOptions;
   Status SetOptions(ColumnFamilyHandle* column_family,
       const std::unordered_map<std::string, std::string>& options_map);
@@ -151,6 +160,15 @@ class DBImpl : public DB {
   virtual Status DeleteFile(std::string name);
 
   virtual void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
+
+  // Obtains the meta data of the specified column family of the DB.
+  // Status::NotFound() will be returned if the current DB does not have
+  // any column family match the specified name.
+  // TODO(yhchiang): output parameter is placed in the end in this codebase.
+  virtual void GetColumnFamilyMetaData(
+      ColumnFamilyHandle* column_family,
+      ColumnFamilyMetaData* metadata) override;
+
 #endif  // ROCKSDB_LITE
 
   // checks if all live files exist on file system and that their file sizes
@@ -210,7 +228,7 @@ class DBImpl : public DB {
   // REQUIRES: mutex locked
   // pass the pointer that you got from TEST_BeginWrite()
   void TEST_EndWrite(void* w);
-#endif  // NDEBUG
+#endif  // ROCKSDB_LITE
 
   // Returns the list of live files in 'live' and the list
   // of all files in the filesystem in 'candidate_files'.
@@ -238,6 +256,8 @@ class DBImpl : public DB {
   Iterator* NewInternalIterator(const ReadOptions&, ColumnFamilyData* cfd,
                                 SuperVersion* super_version, Arena* arena);
 
+  void NotifyOnFlushCompleted(ColumnFamilyData* cfd, uint64_t file_number);
+
  private:
   friend class DB;
   friend class InternalStats;
@@ -265,6 +285,24 @@ class DBImpl : public DB {
   // Delete any unneeded files and stale in-memory entries.
   void DeleteObsoleteFiles();
 
+  // Background process needs to call
+  //     auto x = CaptureCurrentFileNumberInPendingOutputs()
+  //     <do something>
+  //     ReleaseFileNumberFromPendingOutputs(x)
+  // This will protect any temporary files created while <do something> is
+  // executing from being deleted.
+  // -----------
+  // This function will capture current file number and append it to
+  // pending_outputs_. This will prevent any background process to delete any
+  // file created after this point.
+  std::list<uint64_t>::iterator CaptureCurrentFileNumberInPendingOutputs();
+  // This function should be called with the result of
+  // CaptureCurrentFileNumberInPendingOutputs(). It then marks that any file
+  // created between the calls CaptureCurrentFileNumberInPendingOutputs() and
+  // ReleaseFileNumberFromPendingOutputs() can now be deleted (if it's not live
+  // and blocked by any other pending_outputs_ calls)
+  void ReleaseFileNumberFromPendingOutputs(std::list<uint64_t>::iterator v);
+
   // Flush the in-memory write buffer to storage.  Switches to a new
   // log-file/memtable and writes a new descriptor iff successful.
   Status FlushMemTableToOutputFile(ColumnFamilyData* cfd,
@@ -299,6 +337,15 @@ class DBImpl : public DB {
   void RecordFlushIOStats();
   void RecordCompactionIOStats();
 
+#ifndef ROCKSDB_LITE
+  Status CompactFilesImpl(
+      const CompactionOptions& compact_options, ColumnFamilyData* cfd,
+      Version* version, const std::vector<std::string>& input_file_names,
+      const int output_level, int output_path_id);
+#endif  // ROCKSDB_LITE
+
+  ColumnFamilyData* GetColumnFamilyDataByName(const std::string& cf_name);
+
   void MaybeScheduleFlushOrCompaction();
   static void BGWorkCompaction(void* db);
   static void BGWorkFlush(void* db);
@@ -390,10 +437,16 @@ class DBImpl : public DB {
 
   SnapshotList snapshots_;
 
-  // Set of table files to protect from deletion because they are
-  // part of ongoing compactions.
-  // map from pending file number ID to their path IDs.
-  FileNumToPathIdMap pending_outputs_;
+  // For each background job, pending_outputs_ keeps the current file number at
+  // the time that background job started.
+  // FindObsoleteFiles()/PurgeObsoleteFiles() never deletes any file that has
+  // number bigger than any of the file number in pending_outputs_. Since file
+  // numbers grow monotonically, this also means that pending_outputs_ is always
+  // sorted. After a background job is done executing, its file number is
+  // deleted from pending_outputs_, which allows PurgeObsoleteFiles() to clean
+  // it up.
+  // State is protected with db mutex.
+  std::list<uint64_t> pending_outputs_;
 
   // At least one compaction or flush job is pending but not yet scheduled
   // because of the max background thread limit.
@@ -463,6 +516,12 @@ class DBImpl : public DB {
   // Indicate DB was opened successfully
   bool opened_successfully_;
 
+  // The list of registered event listeners.
+  std::list<EventListener*> listeners_;
+
+  // count how many events are currently being notified.
+  int notifying_events_;
+
   // No copying allowed
   DBImpl(const DBImpl&);
   void operator=(const DBImpl&);
@@ -528,8 +587,4 @@ static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
   if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
 }
 
-// Dump db file summary, implemented in util/
-extern void DumpDBFileSummary(const DBOptions& options,
-                              const std::string& dbname);
-
 }  // namespace rocksdb

db/db_impl_readonly.h

@@ -62,10 +62,20 @@ class DBImplReadOnly : public DBImpl {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
 
+  using DBImpl::CompactFiles;
+  virtual Status CompactFiles(
+      const CompactionOptions& compact_options,
+      ColumnFamilyHandle* column_family,
+      const std::vector<std::string>& input_file_names,
+      const int output_level, const int output_path_id = -1) override {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }
+
 #ifndef ROCKSDB_LITE
   virtual Status DisableFileDeletions() override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
+
   virtual Status EnableFileDeletions(bool force) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }

db/db_iter_test.cc

@@ -19,7 +19,7 @@
 
 namespace rocksdb {
 
-static uint32_t TestGetTickerCount(const Options& options,
+static uint64_t TestGetTickerCount(const Options& options,
                                    Tickers ticker_type) {
   return options.statistics->getTickerCount(ticker_type);
 }

db/db_test.cc

@@ -15,6 +15,7 @@
 #include <unordered_set>
 #include <utility>
 
+#include "db/filename.h"
 #include "db/dbformat.h"
 #include "db/db_impl.h"
 #include "db/filename.h"
@@ -165,6 +166,8 @@ class SpecialEnv : public EnvWrapper {
 
   std::atomic<uint32_t> non_writable_count_;
 
+  std::function<void()>* table_write_callback_;
+
   explicit SpecialEnv(Env* base) : EnvWrapper(base), rnd_(301) {
     delay_sstable_sync_.store(false, std::memory_order_release);
     drop_writes_.store(false, std::memory_order_release);
@@ -181,6 +184,7 @@ class SpecialEnv : public EnvWrapper {
     non_writeable_rate_ = 0;
     new_writable_count_ = 0;
     non_writable_count_ = 0;
+    table_write_callback_ = nullptr;
   }
 
   Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
@@ -196,6 +200,9 @@ class SpecialEnv : public EnvWrapper {
             base_(std::move(base)) {
       }
       Status Append(const Slice& data) {
+        if (env_->table_write_callback_) {
+          (*env_->table_write_callback_)();
+        }
         if (env_->drop_writes_.load(std::memory_order_acquire)) {
           // Drop writes on the floor
           return Status::OK();
@@ -661,7 +668,7 @@ class DBTest {
   void CreateColumnFamilies(const std::vector<std::string>& cfs,
                             const Options& options) {
     ColumnFamilyOptions cf_opts(options);
-    int cfi = handles_.size();
+    size_t cfi = handles_.size();
     handles_.resize(cfi + cfs.size());
     for (auto cf : cfs) {
       ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++]));
@@ -926,7 +933,7 @@ class DBTest {
     int num_levels =
         (cf == 0) ? db_->NumberLevels() : db_->NumberLevels(handles_[1]);
     std::string result;
-    int last_non_zero_offset = 0;
+    size_t last_non_zero_offset = 0;
     for (int level = 0; level < num_levels; level++) {
       int f = NumTableFilesAtLevel(level, cf);
       char buf[100];
@@ -940,7 +947,7 @@ class DBTest {
     return result;
   }
 
-  int CountFiles() {
+  size_t CountFiles() {
     std::vector<std::string> files;
     env_->GetChildren(dbname_, &files);
 
@@ -949,10 +956,10 @@ class DBTest {
       env_->GetChildren(last_options_.wal_dir, &logfiles);
     }
 
-    return static_cast<int>(files.size() + logfiles.size());
+    return files.size() + logfiles.size();
   }
 
-  int CountLiveFiles() {
+  size_t CountLiveFiles() {
     std::vector<LiveFileMetaData> metadata;
     db_->GetLiveFilesMetaData(&metadata);
     return metadata.size();
@@ -3157,10 +3164,15 @@ Options DeletionTriggerOptions() {
 }  // anonymous namespace
 
 TEST(DBTest, CompactionDeletionTrigger) {
-  Options options = CurrentOptions(DeletionTriggerOptions());
-
   for (int tid = 0; tid < 2; ++tid) {
     uint64_t db_size[2];
+    Options options = CurrentOptions(DeletionTriggerOptions());
+
+    if (tid == 1) {
+      // second pass with universal compaction
+      options.compaction_style = kCompactionStyleUniversal;
+      options.num_levels = 1;
+    }
 
     DestroyAndReopen(options);
     Random rnd(301);
@@ -3184,10 +3196,6 @@ TEST(DBTest, CompactionDeletionTrigger) {
 
     // must have much smaller db size.
     ASSERT_GT(db_size[0] / 3, db_size[1]);
-
-    // repeat the test with universal compaction
-    options.compaction_style = kCompactionStyleUniversal;
-    options.num_levels = 1;
   }
 }
 
@@ -3196,6 +3204,12 @@ TEST(DBTest, CompactionDeletionTriggerReopen) {
     uint64_t db_size[3];
     Options options = CurrentOptions(DeletionTriggerOptions());
 
+    if (tid == 1) {
+      // second pass with universal compaction
+      options.compaction_style = kCompactionStyleUniversal;
+      options.num_levels = 1;
+    }
+
     DestroyAndReopen(options);
     Random rnd(301);
 
@@ -3238,10 +3252,6 @@ TEST(DBTest, CompactionDeletionTriggerReopen) {
     db_size[2] = Size(Key(0), Key(kTestSize - 1));
     // this time we're expecting significant drop in size.
     ASSERT_GT(db_size[0] / 3, db_size[2]);
-
-    // repeat the test with universal compaction
-    options.compaction_style = kCompactionStyleUniversal;
-    options.num_levels = 1;
   }
 }
 
@@ -3274,6 +3284,22 @@ class DeleteFilter : public CompactionFilter {
   virtual const char* Name() const override { return "DeleteFilter"; }
 };
 
+class ConditionalFilter : public CompactionFilter {
+ public:
+  explicit ConditionalFilter(const std::string* filtered_value)
+      : filtered_value_(filtered_value) {}
+  virtual bool Filter(int level, const Slice& key, const Slice& value,
+                      std::string* new_value,
+                      bool* value_changed) const override {
+    return value.ToString() == *filtered_value_;
+  }
+
+  virtual const char* Name() const override { return "ConditionalFilter"; }
+
+ private:
+  const std::string* filtered_value_;
+};
+
 class ChangeFilter : public CompactionFilter {
  public:
   explicit ChangeFilter() {}
@@ -3324,6 +3350,25 @@ class DeleteFilterFactory : public CompactionFilterFactory {
   virtual const char* Name() const override { return "DeleteFilterFactory"; }
 };
 
+class ConditionalFilterFactory : public CompactionFilterFactory {
+ public:
+  explicit ConditionalFilterFactory(const Slice& filtered_value)
+      : filtered_value_(filtered_value.ToString()) {}
+
+  virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
+      const CompactionFilter::Context& context) override {
+    return std::unique_ptr<CompactionFilter>(
+        new ConditionalFilter(&filtered_value_));
+  }
+
+  virtual const char* Name() const override {
+    return "ConditionalFilterFactory";
+  }
+
+ private:
+  std::string filtered_value_;
+};
+
 class ChangeFilterFactory : public CompactionFilterFactory {
  public:
   explicit ChangeFilterFactory() {}
@@ -4051,8 +4096,43 @@ TEST(DBTest, UniversalCompactionFourPaths) {
 
   Destroy(options);
 }
+
 #endif
 
+void CheckColumnFamilyMeta(const ColumnFamilyMetaData& cf_meta) {
+  uint64_t cf_size = 0;
+  uint64_t cf_csize = 0;
+  size_t file_count = 0;
+  for (auto level_meta : cf_meta.levels) {
+    uint64_t level_size = 0;
+    uint64_t level_csize = 0;
+    file_count += level_meta.files.size();
+    for (auto file_meta : level_meta.files) {
+      level_size += file_meta.size;
+    }
+    ASSERT_EQ(level_meta.size, level_size);
+    cf_size += level_size;
+    cf_csize += level_csize;
+  }
+  ASSERT_EQ(cf_meta.file_count, file_count);
+  ASSERT_EQ(cf_meta.size, cf_size);
+}
+
+TEST(DBTest, ColumnFamilyMetaDataTest) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  int key_index = 0;
+  ColumnFamilyMetaData cf_meta;
+  for (int i = 0; i < 100; ++i) {
+    GenerateNewFile(&rnd, &key_index);
+    db_->GetColumnFamilyMetaData(&cf_meta);
+    CheckColumnFamilyMeta(cf_meta);
+  }
+}
+
 TEST(DBTest, ConvertCompactionStyle) {
   Random rnd(301);
   int max_key_level_insert = 200;
@@ -4229,7 +4309,7 @@ bool MinLevelToCompress(CompressionType& type, Options& options, int wbits,
 
 TEST(DBTest, MinLevelToCompress1) {
   Options options = CurrentOptions();
-  CompressionType type;
+  CompressionType type = kSnappyCompression;
   if (!MinLevelToCompress(type, options, -14, -1, 0)) {
     return;
   }
@@ -4249,7 +4329,7 @@ TEST(DBTest, MinLevelToCompress1) {
 
 TEST(DBTest, MinLevelToCompress2) {
   Options options = CurrentOptions();
-  CompressionType type;
+  CompressionType type = kSnappyCompression;
   if (!MinLevelToCompress(type, options, 15, -1, 0)) {
     return;
   }
@@ -4281,7 +4361,8 @@ TEST(DBTest, RepeatedWritesToSameKey) {
         options.num_levels + options.level0_stop_writes_trigger;
 
     Random rnd(301);
-    std::string value = RandomString(&rnd, 2 * options.write_buffer_size);
+    std::string value =
+        RandomString(&rnd, static_cast<int>(2 * options.write_buffer_size));
     for (int i = 0; i < 5 * kMaxFiles; i++) {
       ASSERT_OK(Put(1, "key", value));
       ASSERT_LE(TotalTableFiles(1), kMaxFiles);
@@ -4612,7 +4693,7 @@ TEST(DBTest, CompactionFilterDeletesAll) {
 
   // this will produce empty file (delete compaction filter)
   ASSERT_OK(db_->CompactRange(nullptr, nullptr));
-  ASSERT_EQ(0, CountLiveFiles());
+  ASSERT_EQ(0U, CountLiveFiles());
 
   Reopen(options);
 
@@ -4675,6 +4756,75 @@ TEST(DBTest, CompactionFilterWithValueChange) {
   } while (ChangeCompactOptions());
 }
 
+TEST(DBTest, CompactionFilterWithMergeOperator) {
+  std::string one, two, three, four;
+  PutFixed64(&one, 1);
+  PutFixed64(&two, 2);
+  PutFixed64(&three, 3);
+  PutFixed64(&four, 4);
+
+  Options options;
+  options = CurrentOptions(options);
+  options.create_if_missing = true;
+  options.merge_operator = MergeOperators::CreateUInt64AddOperator();
+  options.num_levels = 3;
+  options.max_mem_compaction_level = 0;
+  // Filter out keys with value is 2.
+  options.compaction_filter_factory =
+      std::make_shared<ConditionalFilterFactory>(two);
+  DestroyAndReopen(options);
+
+  // In the same compaction, a value type needs to be deleted based on
+  // compaction filter, and there is a merge type for the key. compaction
+  // filter result is ignored.
+  ASSERT_OK(db_->Put(WriteOptions(), "foo", two));
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->Merge(WriteOptions(), "foo", one));
+  ASSERT_OK(Flush());
+  std::string newvalue = Get("foo");
+  ASSERT_EQ(newvalue, three);
+  dbfull()->CompactRange(nullptr, nullptr);
+  newvalue = Get("foo");
+  ASSERT_EQ(newvalue, three);
+
+  // value key can be deleted based on compaction filter, leaving only
+  // merge keys.
+  ASSERT_OK(db_->Put(WriteOptions(), "bar", two));
+  ASSERT_OK(Flush());
+  dbfull()->CompactRange(nullptr, nullptr);
+  newvalue = Get("bar");
+  ASSERT_EQ("NOT_FOUND", newvalue);
+  ASSERT_OK(db_->Merge(WriteOptions(), "bar", two));
+  ASSERT_OK(Flush());
+  dbfull()->CompactRange(nullptr, nullptr);
+  newvalue = Get("bar");
+  ASSERT_EQ(two, two);
+
+  // Compaction filter never applies to merge keys.
+  ASSERT_OK(db_->Put(WriteOptions(), "foobar", one));
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->Merge(WriteOptions(), "foobar", two));
+  ASSERT_OK(Flush());
+  newvalue = Get("foobar");
+  ASSERT_EQ(newvalue, three);
+  dbfull()->CompactRange(nullptr, nullptr);
+  newvalue = Get("foobar");
+  ASSERT_EQ(newvalue, three);
+
+  // In the same compaction, both of value type and merge type keys need to be
+  // deleted based on compaction filter, and there is a merge type for the key.
+  // For both keys, compaction filter results are ignored.
+  ASSERT_OK(db_->Put(WriteOptions(), "barfoo", two));
+  ASSERT_OK(Flush());
+  ASSERT_OK(db_->Merge(WriteOptions(), "barfoo", two));
+  ASSERT_OK(Flush());
+  newvalue = Get("barfoo");
+  ASSERT_EQ(newvalue, four);
+  dbfull()->CompactRange(nullptr, nullptr);
+  newvalue = Get("barfoo");
+  ASSERT_EQ(newvalue, four);
+}
+
 TEST(DBTest, CompactionFilterContextManual) {
   KeepFilterFactory* filter = new KeepFilterFactory();
 
@@ -5800,7 +5950,7 @@ TEST(DBTest, DropWrites) {
     ASSERT_OK(Put("foo", "v1"));
     ASSERT_EQ("v1", Get("foo"));
     Compact("a", "z");
-    const int num_files = CountFiles();
+    const size_t num_files = CountFiles();
     // Force out-of-space errors
     env_->drop_writes_.store(true, std::memory_order_release);
     env_->sleep_counter_.Reset();
@@ -5986,7 +6136,7 @@ TEST(DBTest, FilesDeletedAfterCompaction) {
     CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
     ASSERT_OK(Put(1, "foo", "v2"));
     Compact(1, "a", "z");
-    const int num_files = CountLiveFiles();
+    const size_t num_files = CountLiveFiles();
     for (int i = 0; i < 10; i++) {
       ASSERT_OK(Put(1, "foo", "v2"));
       Compact(1, "a", "z");
@@ -6459,7 +6609,7 @@ TEST(DBTest, FlushOneColumnFamily) {
   ASSERT_OK(Put(6, "alyosha", "alyosha"));
   ASSERT_OK(Put(7, "popovich", "popovich"));
 
-  for (size_t i = 0; i < 8; ++i) {
+  for (int i = 0; i < 8; ++i) {
     Flush(i);
     auto tables = ListTableFiles(env_, dbname_);
     ASSERT_EQ(tables.size(), i + 1U);
@@ -6803,8 +6953,8 @@ TEST(DBTest, TransactionLogIteratorCorruptedLog) {
     // than 1025 entries
     auto iter = OpenTransactionLogIter(0);
     int count;
-    int last_sequence_read = ReadRecords(iter, count);
-    ASSERT_LT(last_sequence_read, 1025);
+    SequenceNumber last_sequence_read = ReadRecords(iter, count);
+    ASSERT_LT(last_sequence_read, 1025U);
     // Try to read past the gap, should be able to seek to key1025
     auto iter2 = OpenTransactionLogIter(last_sequence_read + 1);
     ExpectRecords(1, iter2);
@@ -7237,6 +7387,15 @@ class ModelDB: public DB {
     return Status::NotSupported("Not supported operation.");
   }
 
+  using DB::CompactFiles;
+  virtual Status CompactFiles(
+      const CompactionOptions& compact_options,
+      ColumnFamilyHandle* column_family,
+      const std::vector<std::string>& input_file_names,
+      const int output_level, const int output_path_id = -1) override {
+    return Status::NotSupported("Not supported operation.");
+  }
+
   using DB::NumberLevels;
   virtual int NumberLevels(ColumnFamilyHandle* column_family) { return 1; }
 
@@ -7305,6 +7464,10 @@ class ModelDB: public DB {
 
   virtual ColumnFamilyHandle* DefaultColumnFamily() const { return nullptr; }
 
+  virtual void GetColumnFamilyMetaData(
+      ColumnFamilyHandle* column_family,
+      ColumnFamilyMetaData* metadata) {}
+
  private:
   class ModelIter: public Iterator {
    public:
@@ -8193,6 +8356,211 @@ TEST(DBTest, RateLimitingTest) {
   ASSERT_TRUE(ratio < 0.6);
 }
 
+namespace {
+  bool HaveOverlappingKeyRanges(
+      const Comparator* c,
+      const SstFileMetaData& a, const SstFileMetaData& b) {
+    if (c->Compare(a.smallestkey, b.smallestkey) >= 0) {
+      if (c->Compare(a.smallestkey, b.largestkey) <= 0) {
+        // b.smallestkey <= a.smallestkey <= b.largestkey
+        return true;
+      }
+    } else if (c->Compare(a.largestkey, b.smallestkey) >= 0) {
+      // a.smallestkey < b.smallestkey <= a.largestkey
+      return true;
+    }
+    if (c->Compare(a.largestkey, b.largestkey) <= 0) {
+      if (c->Compare(a.largestkey, b.smallestkey) >= 0) {
+        // b.smallestkey <= a.largestkey <= b.largestkey
+        return true;
+      }
+    } else if (c->Compare(a.smallestkey, b.largestkey) <= 0) {
+      // a.smallestkey <= b.largestkey < a.largestkey
+      return true;
+    }
+    return false;
+  }
+
+  // Identifies all files between level "min_level" and "max_level"
+  // which has overlapping key range with "input_file_meta".
+  void GetOverlappingFileNumbersForLevelCompaction(
+      const ColumnFamilyMetaData& cf_meta,
+      const Comparator* comparator,
+      int min_level, int max_level,
+      const SstFileMetaData* input_file_meta,
+      std::set<std::string>* overlapping_file_names) {
+    std::set<const SstFileMetaData*> overlapping_files;
+    overlapping_files.insert(input_file_meta);
+    for (int m = min_level; m <= max_level; ++m) {
+      for (auto& file : cf_meta.levels[m].files) {
+        for (auto* included_file : overlapping_files) {
+          if (HaveOverlappingKeyRanges(
+                  comparator, *included_file, file)) {
+            overlapping_files.insert(&file);
+            overlapping_file_names->insert(file.name);
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  void VerifyCompactionResult(
+      const ColumnFamilyMetaData& cf_meta,
+      const std::set<std::string>& overlapping_file_numbers) {
+    for (auto& level : cf_meta.levels) {
+      for (auto& file : level.files) {
+        assert(overlapping_file_numbers.find(file.name) ==
+               overlapping_file_numbers.end());
+      }
+    }
+  }
+
+  const SstFileMetaData* PickFileRandomly(
+      const ColumnFamilyMetaData& cf_meta,
+      Random* rand,
+      int* level = nullptr) {
+    auto file_id = rand->Uniform(static_cast<int>(
+        cf_meta.file_count)) + 1;
+    for (auto& level_meta : cf_meta.levels) {
+      if (file_id <= level_meta.files.size()) {
+        if (level != nullptr) {
+          *level = level_meta.level;
+        }
+        auto result = rand->Uniform(file_id);
+        return &(level_meta.files[result]);
+      }
+      file_id -= level_meta.files.size();
+    }
+    assert(false);
+    return nullptr;
+  }
+}  // namespace
+
+TEST(DBTest, CompactFilesOnLevelCompaction) {
+  const int kTestKeySize = 16;
+  const int kTestValueSize = 984;
+  const int kEntrySize = kTestKeySize + kTestValueSize;
+  const int kEntriesPerBuffer = 100;
+  Options options;
+  options.create_if_missing = true;
+  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
+  options.compaction_style = kCompactionStyleLevel;
+  options.target_file_size_base = options.write_buffer_size;
+  options.max_bytes_for_level_base = options.target_file_size_base * 2;
+  options.level0_stop_writes_trigger = 2;
+  options.max_bytes_for_level_multiplier = 2;
+  options.compression = kNoCompression;
+  options = CurrentOptions(options);
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  Random rnd(301);
+  for (int key = 64 * kEntriesPerBuffer; key >= 0; --key) {
+    ASSERT_OK(Put(1, std::to_string(key), RandomString(&rnd, kTestValueSize)));
+  }
+  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
+  dbfull()->TEST_WaitForCompact();
+
+  ColumnFamilyMetaData cf_meta;
+  dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
+  int output_level = static_cast<int>(cf_meta.levels.size()) - 1;
+  for (int file_picked = 5; file_picked > 0; --file_picked) {
+    std::set<std::string> overlapping_file_names;
+    std::vector<std::string> compaction_input_file_names;
+    for (int f = 0; f < file_picked; ++f) {
+      int level;
+      auto file_meta = PickFileRandomly(cf_meta, &rnd, &level);
+      compaction_input_file_names.push_back(file_meta->name);
+      GetOverlappingFileNumbersForLevelCompaction(
+          cf_meta, options.comparator, level, output_level,
+          file_meta, &overlapping_file_names);
+    }
+
+    ASSERT_OK(dbfull()->CompactFiles(
+        CompactionOptions(), handles_[1],
+        compaction_input_file_names,
+        output_level));
+
+    // Make sure all overlapping files do not exist after compaction
+    dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
+    VerifyCompactionResult(cf_meta, overlapping_file_names);
+  }
+
+  // make sure all key-values are still there.
+  for (int key = 64 * kEntriesPerBuffer; key >= 0; --key) {
+    ASSERT_NE(Get(1, std::to_string(key)), "NOT_FOUND");
+  }
+}
+
+TEST(DBTest, CompactFilesOnUniversalCompaction) {
+  const int kTestKeySize = 16;
+  const int kTestValueSize = 984;
+  const int kEntrySize = kTestKeySize + kTestValueSize;
+  const int kEntriesPerBuffer = 10;
+
+  ChangeCompactOptions();
+  Options options;
+  options.create_if_missing = true;
+  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
+  options.compaction_style = kCompactionStyleLevel;
+  options.target_file_size_base = options.write_buffer_size;
+  options.compression = kNoCompression;
+  options = CurrentOptions(options);
+  CreateAndReopenWithCF({"pikachu"}, options);
+  ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
+  Random rnd(301);
+  for (int key = 1024 * kEntriesPerBuffer; key >= 0; --key) {
+    ASSERT_OK(Put(1, std::to_string(key), RandomString(&rnd, kTestValueSize)));
+  }
+  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
+  dbfull()->TEST_WaitForCompact();
+  ColumnFamilyMetaData cf_meta;
+  dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
+  std::vector<std::string> compaction_input_file_names;
+  for (auto file : cf_meta.levels[0].files) {
+    if (rnd.OneIn(2)) {
+      compaction_input_file_names.push_back(file.name);
+    }
+  }
+
+  if (compaction_input_file_names.size() == 0) {
+    compaction_input_file_names.push_back(
+        cf_meta.levels[0].files[0].name);
+  }
+
+  // expect fail since universal compaction only allow L0 output
+  ASSERT_TRUE(!dbfull()->CompactFiles(
+      CompactionOptions(), handles_[1],
+      compaction_input_file_names, 1).ok());
+
+  // expect ok and verify the compacted files no longer exist.
+  ASSERT_OK(dbfull()->CompactFiles(
+      CompactionOptions(), handles_[1],
+      compaction_input_file_names, 0));
+
+  dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
+  VerifyCompactionResult(
+      cf_meta,
+      std::set<std::string>(compaction_input_file_names.begin(),
+          compaction_input_file_names.end()));
+
+  compaction_input_file_names.clear();
+
+  // Pick the first and the last file, expect everything is
+  // compacted into one single file.
+  compaction_input_file_names.push_back(
+      cf_meta.levels[0].files[0].name);
+  compaction_input_file_names.push_back(
+      cf_meta.levels[0].files[
+          cf_meta.levels[0].files.size() - 1].name);
+  ASSERT_OK(dbfull()->CompactFiles(
+      CompactionOptions(), handles_[1],
+      compaction_input_file_names, 0));
+
+  dbfull()->GetColumnFamilyMetaData(handles_[1], &cf_meta);
+  ASSERT_EQ(cf_meta.levels[0].files.size(), 1U);
+}
+
 TEST(DBTest, TableOptionsSanitizeTest) {
   Options options = CurrentOptions();
   options.create_if_missing = true;
@@ -9039,6 +9407,37 @@ TEST(DBTest, DynamicMiscOptions) {
   assert_reseek_count(300, 1);
 }
 
+TEST(DBTest, DontDeletePendingOutputs) {
+  Options options;
+  options.env = env_;
+  options.create_if_missing = true;
+  DestroyAndReopen(options);
+
+  // Every time we write to a table file, call FOF/POF with full DB scan. This
+  // will make sure our pending_outputs_ protection work correctly
+  std::function<void()> purge_obsolete_files_function = [&]() {
+    JobContext job_context;
+    dbfull()->TEST_LockMutex();
+    dbfull()->FindObsoleteFiles(&job_context, true /*force*/);
+    dbfull()->TEST_UnlockMutex();
+    dbfull()->PurgeObsoleteFiles(job_context);
+  };
+
+  env_->table_write_callback_ = &purge_obsolete_files_function;
+
+  for (int i = 0; i < 2; ++i) {
+    ASSERT_OK(Put("a", "begin"));
+    ASSERT_OK(Put("z", "end"));
+    ASSERT_OK(Flush());
+  }
+
+  // If pending output guard does not work correctly, PurgeObsoleteFiles() will
+  // delete the file that Compaction is trying to create, causing this: error
+  // db/db_test.cc:975: IO error:
+  // /tmp/rocksdbtest-1552237650/db_test/000009.sst: No such file or directory
+  Compact("a", "b");
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/dbformat.cc

@@ -137,7 +137,8 @@ LookupKey::LookupKey(const Slice& _user_key, SequenceNumber s) {
     dst = new char[needed];
   }
   start_ = dst;
-  dst = EncodeVarint32(dst, usize + 8);
+  // NOTE: We don't support users keys of more than 2GB :)
+  dst = EncodeVarint32(dst, static_cast<uint32_t>(usize + 8));
   kstart_ = dst;
   memcpy(dst, _user_key.data(), usize);
   dst += usize;

db/dbformat.h

@@ -206,13 +206,19 @@ class LookupKey {
   ~LookupKey();
 
   // Return a key suitable for lookup in a MemTable.
-  Slice memtable_key() const { return Slice(start_, end_ - start_); }
+  Slice memtable_key() const {
+    return Slice(start_, static_cast<size_t>(end_ - start_));
+  }
 
   // Return an internal key (suitable for passing to an internal iterator)
-  Slice internal_key() const { return Slice(kstart_, end_ - kstart_); }
+  Slice internal_key() const {
+    return Slice(kstart_, static_cast<size_t>(end_ - kstart_));
+  }
 
   // Return the user key
-  Slice user_key() const { return Slice(kstart_, end_ - kstart_ - 8); }
+  Slice user_key() const {
+    return Slice(kstart_, static_cast<size_t>(end_ - kstart_ - 8));
+  }
 
  private:
   // We construct a char array of the form:
@@ -319,8 +325,8 @@ class IterKey {
 
   void EncodeLengthPrefixedKey(const Slice& key) {
     auto size = key.size();
-    EnlargeBufferIfNeeded(size + VarintLength(size));
-    char* ptr = EncodeVarint32(key_, size);
+    EnlargeBufferIfNeeded(size + static_cast<size_t>(VarintLength(size)));
+    char* ptr = EncodeVarint32(key_, static_cast<uint32_t>(size));
     memcpy(ptr, key.data(), size);
   }
 

db/file_indexer.cc

@@ -17,17 +17,16 @@ namespace rocksdb {
 FileIndexer::FileIndexer(const Comparator* ucmp)
     : num_levels_(0), ucmp_(ucmp), level_rb_(nullptr) {}
 
-uint32_t FileIndexer::NumLevelIndex() const {
-  return next_level_index_.size();
-}
+size_t FileIndexer::NumLevelIndex() const { return next_level_index_.size(); }
 
-uint32_t FileIndexer::LevelIndexSize(uint32_t level) const {
+size_t FileIndexer::LevelIndexSize(size_t level) const {
   return next_level_index_[level].num_index;
 }
 
-void FileIndexer::GetNextLevelIndex(
-    const uint32_t level, const uint32_t file_index, const int cmp_smallest,
-    const int cmp_largest, int32_t* left_bound, int32_t* right_bound) const {
+void FileIndexer::GetNextLevelIndex(const size_t level, const size_t file_index,
+                                    const int cmp_smallest,
+                                    const int cmp_largest, int32_t* left_bound,
+                                    int32_t* right_bound) const {
   assert(level > 0);
 
   // Last level, no hint
@@ -69,7 +68,7 @@ void FileIndexer::GetNextLevelIndex(
   assert(*right_bound <= level_rb_[level + 1]);
 }
 
-void FileIndexer::UpdateIndex(Arena* arena, const uint32_t num_levels,
+void FileIndexer::UpdateIndex(Arena* arena, const size_t num_levels,
                               std::vector<FileMetaData*>* const files) {
   if (files == nullptr) {
     return;
@@ -90,11 +89,11 @@ void FileIndexer::UpdateIndex(Arena* arena, const uint32_t num_levels,
   }
 
   // L1 - Ln-1
-  for (uint32_t level = 1; level < num_levels_ - 1; ++level) {
+  for (size_t level = 1; level < num_levels_ - 1; ++level) {
     const auto& upper_files = files[level];
-    const int32_t upper_size = upper_files.size();
+    const int32_t upper_size = static_cast<int32_t>(upper_files.size());
     const auto& lower_files = files[level + 1];
-    level_rb_[level] = upper_files.size() - 1;
+    level_rb_[level] = static_cast<int32_t>(upper_files.size()) - 1;
     if (upper_size == 0) {
       continue;
     }
@@ -129,7 +128,8 @@ void FileIndexer::UpdateIndex(Arena* arena, const uint32_t num_levels,
         [](IndexUnit* index, int32_t f_idx) { index->largest_rb = f_idx; });
   }
 
-  level_rb_[num_levels_ - 1] = files[num_levels_ - 1].size() - 1;
+  level_rb_[num_levels_ - 1] =
+      static_cast<int32_t>(files[num_levels_ - 1].size()) - 1;
 }
 
 void FileIndexer::CalculateLB(
@@ -137,8 +137,8 @@ void FileIndexer::CalculateLB(
     const std::vector<FileMetaData*>& lower_files, IndexLevel* index_level,
     std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
     std::function<void(IndexUnit*, int32_t)> set_index) {
-  const int32_t upper_size = upper_files.size();
-  const int32_t lower_size = lower_files.size();
+  const int32_t upper_size = static_cast<int32_t>(upper_files.size());
+  const int32_t lower_size = static_cast<int32_t>(lower_files.size());
   int32_t upper_idx = 0;
   int32_t lower_idx = 0;
 
@@ -175,8 +175,8 @@ void FileIndexer::CalculateRB(
     const std::vector<FileMetaData*>& lower_files, IndexLevel* index_level,
     std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
     std::function<void(IndexUnit*, int32_t)> set_index) {
-  const int32_t upper_size = upper_files.size();
-  const int32_t lower_size = lower_files.size();
+  const int32_t upper_size = static_cast<int32_t>(upper_files.size());
+  const int32_t lower_size = static_cast<int32_t>(lower_files.size());
   int32_t upper_idx = upper_size - 1;
   int32_t lower_idx = lower_size - 1;
 

db/file_indexer.h

@@ -42,19 +42,19 @@ class FileIndexer {
  public:
   explicit FileIndexer(const Comparator* ucmp);
 
-  uint32_t NumLevelIndex() const;
+  size_t NumLevelIndex() const;
 
-  uint32_t LevelIndexSize(uint32_t level) const;
+  size_t LevelIndexSize(size_t level) const;
 
   // Return a file index range in the next level to search for a key based on
   // smallest and largest key comparision for the current file specified by
   // level and file_index. When *left_index < *right_index, both index should
   // be valid and fit in the vector size.
-  void GetNextLevelIndex(
-    const uint32_t level, const uint32_t file_index, const int cmp_smallest,
-    const int cmp_largest, int32_t* left_bound, int32_t* right_bound) const;
+  void GetNextLevelIndex(const size_t level, const size_t file_index,
+                         const int cmp_smallest, const int cmp_largest,
+                         int32_t* left_bound, int32_t* right_bound) const;
 
-  void UpdateIndex(Arena* arena, const uint32_t num_levels,
+  void UpdateIndex(Arena* arena, const size_t num_levels,
                    std::vector<FileMetaData*>* const files);
 
   enum {
@@ -62,7 +62,7 @@ class FileIndexer {
   };
 
  private:
-  uint32_t num_levels_;
+  size_t num_levels_;
   const Comparator* ucmp_;
 
   struct IndexUnit {

db/file_indexer_test.cc

@@ -22,8 +22,15 @@ class IntComparator : public Comparator {
   int Compare(const Slice& a, const Slice& b) const {
     assert(a.size() == 8);
     assert(b.size() == 8);
-    return *reinterpret_cast<const int64_t*>(a.data()) -
-      *reinterpret_cast<const int64_t*>(b.data());
+    int64_t diff = *reinterpret_cast<const int64_t*>(a.data()) -
+                   *reinterpret_cast<const int64_t*>(b.data());
+    if (diff < 0) {
+      return -1;
+    } else if (diff == 0) {
+      return 0;
+    } else {
+      return 1;
+    }
   }
 
   const char* Name() const {

db/filename.cc

@@ -79,6 +79,17 @@ std::string MakeTableFileName(const std::string& path, uint64_t number) {
   return MakeFileName(path, number, "sst");
 }
 
+uint64_t TableFileNameToNumber(const std::string& name) {
+  uint64_t number = 0;
+  uint64_t base = 1;
+  int pos = static_cast<int>(name.find_last_of('.'));
+  while (--pos >= 0 && name[pos] >= '0' && name[pos] <= '9') {
+    number += (name[pos] - '0') * base;
+    base *= 10;
+  }
+  return number;
+}
+
 std::string TableFileName(const std::vector<DbPath>& db_paths, uint64_t number,
                           uint32_t path_id) {
   assert(number > 0);

db/filename.h

@@ -36,9 +36,6 @@ enum FileType {
   kIdentityFile
 };
 
-// map from file number to path ID.
-typedef std::unordered_map<uint64_t, uint32_t> FileNumToPathIdMap;
-
 // Return the name of the log file with the specified number
 // in the db named by "dbname".  The result will be prefixed with
 // "dbname".
@@ -55,6 +52,10 @@ extern std::string ArchivedLogFileName(const std::string& dbname,
 
 extern std::string MakeTableFileName(const std::string& name, uint64_t number);
 
+// the reverse function of MakeTableFileName
+// TODO(yhchiang): could merge this function with ParseFileName()
+extern uint64_t TableFileNameToNumber(const std::string& name);
+
 // Return the name of the sstable with the specified number
 // in the db named by "dbname".  The result will be prefixed with
 // "dbname".

db/flush_job.cc

@@ -55,7 +55,6 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
                    const MutableCFOptions& mutable_cf_options,
                    const EnvOptions& env_options, VersionSet* versions,
                    port::Mutex* db_mutex, std::atomic<bool>* shutting_down,
-                   FileNumToPathIdMap* pending_outputs,
                    SequenceNumber newest_snapshot, JobContext* job_context,
                    LogBuffer* log_buffer, Directory* db_directory,
                    CompressionType output_compression, Statistics* stats)
@@ -67,7 +66,6 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
       versions_(versions),
       db_mutex_(db_mutex),
       shutting_down_(shutting_down),
-      pending_outputs_(pending_outputs),
       newest_snapshot_(newest_snapshot),
       job_context_(job_context),
       log_buffer_(log_buffer),
@@ -75,9 +73,9 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
       output_compression_(output_compression),
       stats_(stats) {}
 
-Status FlushJob::Run() {
+Status FlushJob::Run(uint64_t* file_number) {
   // Save the contents of the earliest memtable as a new Table
-  uint64_t file_number;
+  uint64_t fn;
   autovector<MemTable*> mems;
   cfd_->imm()->PickMemtablesToFlush(&mems);
   if (mems.empty()) {
@@ -98,7 +96,7 @@ Status FlushJob::Run() {
   edit->SetColumnFamily(cfd_->GetID());
 
   // This will release and re-acquire the mutex.
-  Status s = WriteLevel0Table(mems, edit, &file_number);
+  Status s = WriteLevel0Table(mems, edit, &fn);
 
   if (s.ok() &&
       (shutting_down_->load(std::memory_order_acquire) || cfd_->IsDropped())) {
@@ -107,15 +105,17 @@ Status FlushJob::Run() {
   }
 
   if (!s.ok()) {
-    cfd_->imm()->RollbackMemtableFlush(mems, file_number, pending_outputs_);
+    cfd_->imm()->RollbackMemtableFlush(mems, fn);
   } else {
     // Replace immutable memtable with the generated Table
     s = cfd_->imm()->InstallMemtableFlushResults(
-        cfd_, mutable_cf_options_, mems, versions_, db_mutex_, file_number,
-        pending_outputs_, &job_context_->memtables_to_free, db_directory_,
-        log_buffer_);
+        cfd_, mutable_cf_options_, mems, versions_, db_mutex_, fn,
+        &job_context_->memtables_to_free, db_directory_, log_buffer_);
   }
 
+  if (s.ok() && file_number != nullptr) {
+    *file_number = fn;
+  }
   return s;
 }
 
@@ -128,7 +128,6 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
   meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0);
   *filenumber = meta.fd.GetNumber();
   // path 0 for level 0 file.
-  pending_outputs_->insert({meta.fd.GetNumber(), 0});
 
   const SequenceNumber earliest_seqno_in_memtable =
       mems[0]->GetFirstSequenceNumber();
@@ -151,9 +150,9 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
       memtables.push_back(m->NewIterator(ro, &arena));
     }
     {
-      ScopedArenaIterator iter(NewMergingIterator(&cfd_->internal_comparator(),
-                                                  &memtables[0],
-                                                  memtables.size(), &arena));
+      ScopedArenaIterator iter(
+          NewMergingIterator(&cfd_->internal_comparator(), &memtables[0],
+                             static_cast<int>(memtables.size()), &arena));
       Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
           "[%s] Level-0 flush table #%" PRIu64 ": started",
           cfd_->GetName().c_str(), meta.fd.GetNumber());
@@ -180,15 +179,6 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
   // re-acquire the most current version
   base = cfd_->current();
 
-  // There could be multiple threads writing to its own level-0 file.
-  // The pending_outputs cannot be cleared here, otherwise this newly
-  // created file might not be considered as a live-file by another
-  // compaction thread that is concurrently deleting obselete files.
-  // The pending_outputs can be cleared only after the new version is
-  // committed so that other threads can recognize this file as a
-  // valid one.
-  // pending_outputs_.erase(meta.number);
-
   // Note that if file_size is zero, the file has been deleted and
   // should not be added to the manifest.
   int level = 0;

db/flush_job.h

@@ -55,13 +55,12 @@ class FlushJob {
            const MutableCFOptions& mutable_cf_options,
            const EnvOptions& env_options, VersionSet* versions,
            port::Mutex* db_mutex, std::atomic<bool>* shutting_down,
-           FileNumToPathIdMap* pending_outputs, SequenceNumber newest_snapshot,
-           JobContext* job_context, LogBuffer* log_buffer,
-           Directory* db_directory, CompressionType output_compression,
-           Statistics* stats);
+           SequenceNumber newest_snapshot, JobContext* job_context,
+           LogBuffer* log_buffer, Directory* db_directory,
+           CompressionType output_compression, Statistics* stats);
   ~FlushJob() {}
 
-  Status Run();
+  Status Run(uint64_t* file_number = nullptr);
 
  private:
   Status WriteLevel0Table(const autovector<MemTable*>& mems, VersionEdit* edit,
@@ -74,7 +73,6 @@ class FlushJob {
   VersionSet* versions_;
   port::Mutex* db_mutex_;
   std::atomic<bool>* shutting_down_;
-  FileNumToPathIdMap* pending_outputs_;
   SequenceNumber newest_snapshot_;
   JobContext* job_context_;
   LogBuffer* log_buffer_;

db/flush_job_test.cc

@@ -73,7 +73,6 @@ class FlushJobTest {
   std::unique_ptr<VersionSet> versions_;
   port::Mutex mutex_;
   std::atomic<bool> shutting_down_;
-  FileNumToPathIdMap pending_outputs_;
   std::shared_ptr<MockTableFactory> mock_table_factory_;
 };
 
@@ -83,8 +82,8 @@ TEST(FlushJobTest, Empty) {
   FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
                      db_options_, *cfd->GetLatestMutableCFOptions(),
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     &pending_outputs_, SequenceNumber(), &job_context, nullptr,
-                     nullptr, kNoCompression, nullptr);
+                     SequenceNumber(), &job_context, nullptr, nullptr,
+                     kNoCompression, nullptr);
   ASSERT_OK(flush_job.Run());
 }
 
@@ -108,8 +107,8 @@ TEST(FlushJobTest, NonEmpty) {
   FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
                      db_options_, *cfd->GetLatestMutableCFOptions(),
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     &pending_outputs_, SequenceNumber(), &job_context, nullptr,
-                     nullptr, kNoCompression, nullptr);
+                     SequenceNumber(), &job_context, nullptr, nullptr,
+                     kNoCompression, nullptr);
   mutex_.Lock();
   ASSERT_OK(flush_job.Run());
   mutex_.Unlock();

db/forward_iterator.cc

@@ -125,6 +125,8 @@ ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
       sv_(current_sv),
       mutable_iter_(nullptr),
       current_(nullptr),
+      status_(Status::OK()),
+      immutable_status_(Status::OK()),
       valid_(false),
       is_prev_set_(false),
       is_prev_inclusive_(false) {
@@ -177,7 +179,7 @@ void ForwardIterator::SeekToFirst() {
   if (sv_ == nullptr ||
       sv_ ->version_number != cfd_->GetSuperVersionNumber()) {
     RebuildIterators(true);
-  } else if (status_.IsIncomplete()) {
+  } else if (immutable_status_.IsIncomplete()) {
     ResetIncompleteIterators();
   }
   SeekInternal(Slice(), true);
@@ -187,7 +189,7 @@ void ForwardIterator::Seek(const Slice& internal_key) {
   if (sv_ == nullptr ||
       sv_ ->version_number != cfd_->GetSuperVersionNumber()) {
     RebuildIterators(true);
-  } else if (status_.IsIncomplete()) {
+  } else if (immutable_status_.IsIncomplete()) {
     ResetIncompleteIterators();
   }
   SeekInternal(internal_key, false);
@@ -205,13 +207,16 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
   // if it turns to need to seek immutable often. We probably want to have
   // an option to turn it off.
   if (seek_to_first || NeedToSeekImmutable(internal_key)) {
+    immutable_status_ = Status::OK();
     {
       auto tmp = MinIterHeap(MinIterComparator(&cfd_->internal_comparator()));
       immutable_min_heap_.swap(tmp);
     }
     for (auto* m : imm_iters_) {
       seek_to_first ? m->SeekToFirst() : m->Seek(internal_key);
-      if (m->Valid()) {
+      if (!m->status().ok()) {
+        immutable_status_ = m->status();
+      } else if (m->Valid()) {
         immutable_min_heap_.push(m);
       }
     }
@@ -235,13 +240,8 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
         l0_iters_[i]->Seek(internal_key);
       }
 
-      if (l0_iters_[i]->status().IsIncomplete()) {
-        // if any of the immutable iterators is incomplete (no-io option was
-        // used), we are unable to reliably find the smallest key
-        assert(read_options_.read_tier == kBlockCacheTier);
-        status_ = l0_iters_[i]->status();
-        valid_ = false;
-        return;
+      if (!l0_iters_[i]->status().ok()) {
+        immutable_status_ = l0_iters_[i]->status();
       } else if (l0_iters_[i]->Valid()) {
         immutable_min_heap_.push(l0_iters_[i]);
       }
@@ -264,10 +264,11 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
         if (search_left_bound == search_right_bound) {
           f_idx = search_left_bound;
         } else if (search_left_bound < search_right_bound) {
-          f_idx = FindFileInRange(
-              level_files, internal_key, search_left_bound,
-              search_right_bound == FileIndexer::kLevelMaxIndex ?
-                level_files.size() : search_right_bound);
+          f_idx =
+              FindFileInRange(level_files, internal_key, search_left_bound,
+                              search_right_bound == FileIndexer::kLevelMaxIndex
+                                  ? static_cast<uint32_t>(level_files.size())
+                                  : search_right_bound);
         } else {
           // search_left_bound > search_right_bound
           // There are only 2 cases this can happen:
@@ -311,12 +312,8 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
         seek_to_first ? level_iters_[level - 1]->SeekToFirst() :
                         level_iters_[level - 1]->Seek(internal_key);
 
-        if (level_iters_[level - 1]->status().IsIncomplete()) {
-          // see above
-          assert(read_options_.read_tier == kBlockCacheTier);
-          status_ = level_iters_[level - 1]->status();
-          valid_ = false;
-          return;
+        if (!level_iters_[level - 1]->status().ok()) {
+          immutable_status_ = level_iters_[level - 1]->status();
         } else if (level_iters_[level - 1]->Valid()) {
           immutable_min_heap_.push(level_iters_[level - 1]);
         }
@@ -371,11 +368,8 @@ void ForwardIterator::Next() {
 
   current_->Next();
   if (current_ != mutable_iter_) {
-    if (current_->status().IsIncomplete()) {
-      assert(read_options_.read_tier == kBlockCacheTier);
-      status_ = current_->status();
-      valid_ = false;
-      return;
+    if (!current_->status().ok()) {
+      immutable_status_ = current_->status();
     } else if (current_->Valid()) {
       immutable_min_heap_.push(current_);
     }
@@ -401,23 +395,7 @@ Status ForwardIterator::status() const {
     return mutable_iter_->status();
   }
 
-  for (auto *it : imm_iters_) {
-    if (it && !it->status().ok()) {
-      return it->status();
-    }
-  }
-  for (auto *it : l0_iters_) {
-    if (it && !it->status().ok()) {
-      return it->status();
-    }
-  }
-  for (auto *it : level_iters_) {
-    if (it && !it->status().ok()) {
-      return it->status();
-    }
-  }
-
-  return Status::OK();
+  return immutable_status_;
 }
 
 void ForwardIterator::RebuildIterators(bool refresh_sv) {
@@ -511,7 +489,7 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
   // 'target' belongs to that interval (immutable_min_heap_.top() is already
   // at the correct position).
 
-  if (!valid_ || !current_ || !is_prev_set_) {
+  if (!valid_ || !current_ || !is_prev_set_ || !immutable_status_.ok()) {
     return true;
   }
   Slice prev_key = prev_key_.GetKey();

db/forward_iterator.h

@@ -97,6 +97,7 @@ class ForwardIterator : public Iterator {
   Iterator* current_;
   // internal iterator status
   Status status_;
+  Status immutable_status_;
   bool valid_;
 
   IterKey prev_key_;

db/internal_stats.cc

@@ -235,7 +235,7 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
     case kCompactionPending:
       // 1 if the system already determines at least one compacdtion is needed.
       // 0 otherwise,
-      *value = (vstorage->NeedsCompaction() ? 1 : 0);
+      *value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0);
       return true;
     case kBackgroundErrors:
       // Accumulated number of  errors in background flushes or compactions.

db/job_context.h

@@ -58,8 +58,12 @@ struct JobContext {
 
   // the current manifest_file_number, log_number and prev_log_number
   // that corresponds to the set of files in 'live'.
-  uint64_t manifest_file_number, pending_manifest_file_number, log_number,
-      prev_log_number;
+  uint64_t manifest_file_number;
+  uint64_t pending_manifest_file_number;
+  uint64_t log_number;
+  uint64_t prev_log_number;
+
+  uint64_t min_pending_output = 0;
 
   explicit JobContext(bool create_superversion = false) {
     manifest_file_number = 0;

db/listener_test.cc

@@ -0,0 +1,344 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#include "db/dbformat.h"
+#include "db/db_impl.h"
+#include "db/filename.h"
+#include "db/version_set.h"
+#include "db/write_batch_internal.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/perf_context.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/slice_transform.h"
+#include "rocksdb/table.h"
+#include "rocksdb/options.h"
+#include "rocksdb/table_properties.h"
+#include "table/block_based_table_factory.h"
+#include "table/plain_table_factory.h"
+#include "util/hash.h"
+#include "util/hash_linklist_rep.h"
+#include "utilities/merge_operators.h"
+#include "util/logging.h"
+#include "util/mutexlock.h"
+#include "util/rate_limiter.h"
+#include "util/statistics.h"
+#include "util/testharness.h"
+#include "util/sync_point.h"
+#include "util/testutil.h"
+
+#ifndef ROCKSDB_LITE
+
+namespace rocksdb {
+
+class EventListenerTest {
+ public:
+  EventListenerTest() {
+    dbname_ = test::TmpDir() + "/listener_test";
+    ASSERT_OK(DestroyDB(dbname_, Options()));
+    db_ = nullptr;
+    Reopen();
+  }
+
+  ~EventListenerTest() {
+    Close();
+    Options options;
+    options.db_paths.emplace_back(dbname_, 0);
+    options.db_paths.emplace_back(dbname_ + "_2", 0);
+    options.db_paths.emplace_back(dbname_ + "_3", 0);
+    options.db_paths.emplace_back(dbname_ + "_4", 0);
+    ASSERT_OK(DestroyDB(dbname_, options));
+  }
+
+  void CreateColumnFamilies(const std::vector<std::string>& cfs,
+                            const ColumnFamilyOptions* options = nullptr) {
+    ColumnFamilyOptions cf_opts;
+    cf_opts = ColumnFamilyOptions(Options());
+    size_t cfi = handles_.size();
+    handles_.resize(cfi + cfs.size());
+    for (auto cf : cfs) {
+      ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++]));
+    }
+  }
+
+  void Close() {
+    for (auto h : handles_) {
+      delete h;
+    }
+    handles_.clear();
+    delete db_;
+    db_ = nullptr;
+  }
+
+  void ReopenWithColumnFamilies(const std::vector<std::string>& cfs,
+                                const Options* options = nullptr) {
+    ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
+  }
+
+  Status TryReopenWithColumnFamilies(const std::vector<std::string>& cfs,
+                                     const Options* options = nullptr) {
+    Close();
+    Options opts = (options == nullptr) ? Options() : *options;
+    std::vector<const Options*> v_opts(cfs.size(), &opts);
+    return TryReopenWithColumnFamilies(cfs, v_opts);
+  }
+
+  Status TryReopenWithColumnFamilies(
+      const std::vector<std::string>& cfs,
+      const std::vector<const Options*>& options) {
+    Close();
+    ASSERT_EQ(cfs.size(), options.size());
+    std::vector<ColumnFamilyDescriptor> column_families;
+    for (size_t i = 0; i < cfs.size(); ++i) {
+      column_families.push_back(ColumnFamilyDescriptor(cfs[i], *options[i]));
+    }
+    DBOptions db_opts = DBOptions(*options[0]);
+    return DB::Open(db_opts, dbname_, column_families, &handles_, &db_);
+  }
+
+  Status TryReopen(Options* options = nullptr) {
+    Close();
+    Options opts;
+    if (options != nullptr) {
+      opts = *options;
+    } else {
+      opts.create_if_missing = true;
+    }
+
+    return DB::Open(opts, dbname_, &db_);
+  }
+
+  void Reopen(Options* options = nullptr) {
+    ASSERT_OK(TryReopen(options));
+  }
+
+  void CreateAndReopenWithCF(const std::vector<std::string>& cfs,
+                             const Options* options = nullptr) {
+    CreateColumnFamilies(cfs, options);
+    std::vector<std::string> cfs_plus_default = cfs;
+    cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName);
+    ReopenWithColumnFamilies(cfs_plus_default, options);
+  }
+
+  DBImpl* dbfull() {
+    return reinterpret_cast<DBImpl*>(db_);
+  }
+
+  Status Put(int cf, const Slice& k, const Slice& v,
+             WriteOptions wo = WriteOptions()) {
+    return db_->Put(wo, handles_[cf], k, v);
+  }
+
+  Status Flush(int cf = 0) {
+    if (cf == 0) {
+      return db_->Flush(FlushOptions());
+    } else {
+      return db_->Flush(FlushOptions(), handles_[cf]);
+    }
+  }
+
+
+  DB* db_;
+  std::string dbname_;
+  std::vector<ColumnFamilyHandle*> handles_;
+};
+
+class TestFlushListener : public EventListener {
+ public:
+  void OnFlushCompleted(
+      DB* db, const std::string& name,
+      const std::string& file_path,
+      bool triggered_writes_slowdown,
+      bool triggered_writes_stop) override {
+    flushed_dbs_.push_back(db);
+    flushed_column_family_names_.push_back(name);
+    if (triggered_writes_slowdown) {
+      slowdown_count++;
+    }
+    if (triggered_writes_stop) {
+      stop_count++;
+    }
+  }
+
+  std::vector<std::string> flushed_column_family_names_;
+  std::vector<DB*> flushed_dbs_;
+  int slowdown_count;
+  int stop_count;
+};
+
+TEST(EventListenerTest, OnSingleDBFlushTest) {
+  Options options;
+  TestFlushListener* listener = new TestFlushListener();
+  options.listeners.emplace_back(listener);
+  std::vector<std::string> cf_names = {
+      "pikachu", "ilya", "muromec", "dobrynia",
+      "nikitich", "alyosha", "popovich"};
+  CreateAndReopenWithCF(cf_names, &options);
+
+  ASSERT_OK(Put(1, "pikachu", "pikachu"));
+  ASSERT_OK(Put(2, "ilya", "ilya"));
+  ASSERT_OK(Put(3, "muromec", "muromec"));
+  ASSERT_OK(Put(4, "dobrynia", "dobrynia"));
+  ASSERT_OK(Put(5, "nikitich", "nikitich"));
+  ASSERT_OK(Put(6, "alyosha", "alyosha"));
+  ASSERT_OK(Put(7, "popovich", "popovich"));
+  for (size_t i = 1; i < 8; ++i) {
+    Flush(static_cast<int>(i));
+    dbfull()->TEST_WaitForFlushMemTable();
+    ASSERT_EQ(listener->flushed_dbs_.size(), i);
+    ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
+  }
+
+  // make sure call-back functions are called in the right order
+  for (size_t i = 0; i < cf_names.size(); ++i) {
+    ASSERT_EQ(listener->flushed_dbs_[i], db_);
+    ASSERT_EQ(listener->flushed_column_family_names_[i], cf_names[i]);
+  }
+}
+
+TEST(EventListenerTest, MultiCF) {
+  Options options;
+  TestFlushListener* listener = new TestFlushListener();
+  options.listeners.emplace_back(listener);
+  std::vector<std::string> cf_names = {
+      "pikachu", "ilya", "muromec", "dobrynia",
+      "nikitich", "alyosha", "popovich"};
+  CreateAndReopenWithCF(cf_names, &options);
+
+  ASSERT_OK(Put(1, "pikachu", "pikachu"));
+  ASSERT_OK(Put(2, "ilya", "ilya"));
+  ASSERT_OK(Put(3, "muromec", "muromec"));
+  ASSERT_OK(Put(4, "dobrynia", "dobrynia"));
+  ASSERT_OK(Put(5, "nikitich", "nikitich"));
+  ASSERT_OK(Put(6, "alyosha", "alyosha"));
+  ASSERT_OK(Put(7, "popovich", "popovich"));
+  for (size_t i = 1; i < 8; ++i) {
+    Flush(static_cast<int>(i));
+    ASSERT_EQ(listener->flushed_dbs_.size(), i);
+    ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
+  }
+
+  // make sure call-back functions are called in the right order
+  for (size_t i = 0; i < cf_names.size(); i++) {
+    ASSERT_EQ(listener->flushed_dbs_[i], db_);
+    ASSERT_EQ(listener->flushed_column_family_names_[i], cf_names[i]);
+  }
+}
+
+TEST(EventListenerTest, MultiDBMultiListeners) {
+  std::vector<TestFlushListener*> listeners;
+  const int kNumDBs = 5;
+  const int kNumListeners = 10;
+  for (int i = 0; i < kNumListeners; ++i) {
+    listeners.emplace_back(new TestFlushListener());
+  }
+
+  std::vector<std::string> cf_names = {
+      "pikachu", "ilya", "muromec", "dobrynia",
+      "nikitich", "alyosha", "popovich"};
+
+  Options options;
+  options.create_if_missing = true;
+  for (int i = 0; i < kNumListeners; ++i) {
+    options.listeners.emplace_back(listeners[i]);
+  }
+  DBOptions db_opts(options);
+  ColumnFamilyOptions cf_opts(options);
+
+  std::vector<DB*> dbs;
+  std::vector<std::vector<ColumnFamilyHandle *>> vec_handles;
+
+  for (int d = 0; d < kNumDBs; ++d) {
+    ASSERT_OK(DestroyDB(dbname_ + std::to_string(d), options));
+    DB* db;
+    std::vector<ColumnFamilyHandle*> handles;
+    ASSERT_OK(DB::Open(options, dbname_ + std::to_string(d), &db));
+    for (size_t c = 0; c < cf_names.size(); ++c) {
+      ColumnFamilyHandle* handle;
+      db->CreateColumnFamily(cf_opts, cf_names[c], &handle);
+      handles.push_back(handle);
+    }
+
+    vec_handles.push_back(std::move(handles));
+    dbs.push_back(db);
+  }
+
+  for (int d = 0; d < kNumDBs; ++d) {
+    for (size_t c = 0; c < cf_names.size(); ++c) {
+      ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c],
+                cf_names[c], cf_names[c]));
+    }
+  }
+
+  for (size_t c = 0; c < cf_names.size(); ++c) {
+    for (int d = 0; d < kNumDBs; ++d) {
+      ASSERT_OK(dbs[d]->Flush(FlushOptions(), vec_handles[d][c]));
+      reinterpret_cast<DBImpl*>(dbs[d])->TEST_WaitForFlushMemTable();
+    }
+  }
+
+  for (auto* listener : listeners) {
+    int pos = 0;
+    for (size_t c = 0; c < cf_names.size(); ++c) {
+      for (int d = 0; d < kNumDBs; ++d) {
+        ASSERT_EQ(listener->flushed_dbs_[pos], dbs[d]);
+        ASSERT_EQ(listener->flushed_column_family_names_[pos], cf_names[c]);
+        pos++;
+      }
+    }
+  }
+
+  for (auto handles : vec_handles) {
+    for (auto h : handles) {
+      delete h;
+    }
+    handles.clear();
+  }
+  vec_handles.clear();
+
+  for (auto db : dbs) {
+    delete db;
+  }
+}
+
+TEST(EventListenerTest, DisableBGCompaction) {
+  Options options;
+  TestFlushListener* listener = new TestFlushListener();
+  const int kSlowdownTrigger = 5;
+  const int kStopTrigger = 10;
+  options.level0_slowdown_writes_trigger = kSlowdownTrigger;
+  options.level0_stop_writes_trigger = kStopTrigger;
+  options.listeners.emplace_back(listener);
+  // BG compaction is disabled.  Number of L0 files will simply keeps
+  // increasing in this test.
+  options.compaction_style = kCompactionStyleNone;
+  options.compression = kNoCompression;
+  options.write_buffer_size = 100000;  // Small write buffer
+
+  CreateAndReopenWithCF({"pikachu"}, &options);
+  WriteOptions wopts;
+  wopts.timeout_hint_us = 100000;
+  ColumnFamilyMetaData cf_meta;
+  db_->GetColumnFamilyMetaData(handles_[1], &cf_meta);
+  // keep writing until writes are forced to stop.
+  for (int i = 0; static_cast<int>(cf_meta.file_count) < kStopTrigger; ++i) {
+    Put(1, std::to_string(i), std::string(100000, 'x'), wopts);
+    db_->GetColumnFamilyMetaData(handles_[1], &cf_meta);
+  }
+  ASSERT_GE(listener->slowdown_count, kStopTrigger - kSlowdownTrigger);
+  ASSERT_GE(listener->stop_count, 1);
+}
+
+}  // namespace rocksdb
+
+#endif  // ROCKSDB_LITE
+
+int main(int argc, char** argv) {
+  return rocksdb::test::RunAllTests();
+}
+

db/log_and_apply_bench.cc

@@ -6,6 +6,11 @@
 
 #include <vector>
 
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <inttypes.h>
 #include "util/testharness.h"
 #include "util/benchharness.h"
 #include "db/version_set.h"
@@ -14,9 +19,9 @@
 
 namespace rocksdb {
 
-std::string MakeKey(unsigned int num) {
+std::string MakeKey(uint64_t num) {
   char buf[30];
-  snprintf(buf, sizeof(buf), "%016u", num);
+  snprintf(buf, sizeof(buf), "%016" PRIu64, num);
   return std::string(buf);
 }
 

db/log_reader.cc

@@ -54,7 +54,7 @@ bool Reader::SkipToInitialBlock() {
   if (block_start_location > 0) {
     Status skip_status = file_->Skip(block_start_location);
     if (!skip_status.ok()) {
-      ReportDrop(block_start_location, skip_status);
+      ReportDrop(static_cast<size_t>(block_start_location), skip_status);
       return false;
     }
   }

db/log_test.cc

@@ -558,9 +558,9 @@ TEST(LogTest, ErrorJoinsRecords) {
 
   ASSERT_EQ("correct", Read());
   ASSERT_EQ("EOF", Read());
-  const unsigned int dropped = DroppedBytes();
-  ASSERT_LE(dropped, 2*kBlockSize + 100);
-  ASSERT_GE(dropped, 2*kBlockSize);
+  size_t dropped = DroppedBytes();
+  ASSERT_LE(dropped, 2 * kBlockSize + 100);
+  ASSERT_GE(dropped, 2 * kBlockSize);
 }
 
 TEST(LogTest, ReadStart) {

db/memtable.cc

@@ -188,7 +188,7 @@ KeyHandle MemTableRep::Allocate(const size_t len, char** buf) {
 // into this scratch space.
 const char* EncodeKey(std::string* scratch, const Slice& target) {
   scratch->clear();
-  PutVarint32(scratch, target.size());
+  PutVarint32(scratch, static_cast<uint32_t>(target.size()));
   scratch->append(target.data(), target.size());
   return scratch->data();
 }
@@ -288,12 +288,12 @@ void MemTable::Add(SequenceNumber s, ValueType type,
   //  key bytes    : char[internal_key.size()]
   //  value_size   : varint32 of value.size()
   //  value bytes  : char[value.size()]
-  size_t key_size = key.size();
-  size_t val_size = value.size();
-  size_t internal_key_size = key_size + 8;
-  const size_t encoded_len =
-      VarintLength(internal_key_size) + internal_key_size +
-      VarintLength(val_size) + val_size;
+  uint32_t key_size = static_cast<uint32_t>(key.size());
+  uint32_t val_size = static_cast<uint32_t>(value.size());
+  uint32_t internal_key_size = key_size + 8;
+  const uint32_t encoded_len = VarintLength(internal_key_size) +
+                               internal_key_size + VarintLength(val_size) +
+                               val_size;
   char* buf = nullptr;
   KeyHandle handle = table_->Allocate(encoded_len, &buf);
   assert(buf != nullptr);
@@ -502,8 +502,8 @@ void MemTable::Update(SequenceNumber seq,
       switch (static_cast<ValueType>(tag & 0xff)) {
         case kTypeValue: {
           Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length);
-          uint32_t prev_size = prev_value.size();
-          uint32_t new_size = value.size();
+          uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
+          uint32_t new_size = static_cast<uint32_t>(value.size());
 
           // Update value, if new value size  <= previous value size
           if (new_size <= prev_size ) {
@@ -560,10 +560,10 @@ bool MemTable::UpdateCallback(SequenceNumber seq,
       switch (static_cast<ValueType>(tag & 0xff)) {
         case kTypeValue: {
           Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length);
-          uint32_t  prev_size = prev_value.size();
+          uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
 
           char* prev_buffer = const_cast<char*>(prev_value.data());
-          uint32_t  new_prev_size = prev_size;
+          uint32_t new_prev_size = prev_size;
 
           std::string str_value;
           WriteLock wl(GetLock(lkey.user_key()));

db/memtable_list.cc

@@ -144,8 +144,7 @@ void MemTableList::PickMemtablesToFlush(autovector<MemTable*>* ret) {
 }
 
 void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
-                                         uint64_t file_number,
-                                         FileNumToPathIdMap* pending_outputs) {
+                                         uint64_t file_number) {
   assert(!mems.empty());
 
   // If the flush was not successful, then just reset state.
@@ -159,7 +158,6 @@ void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
     m->edit_.Clear();
     num_flush_not_started_++;
   }
-  pending_outputs->erase(file_number);
   imm_flush_needed.store(true, std::memory_order_release);
 }
 
@@ -167,9 +165,8 @@ void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
 Status MemTableList::InstallMemtableFlushResults(
     ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
     const autovector<MemTable*>& mems, VersionSet* vset, port::Mutex* mu,
-    uint64_t file_number, FileNumToPathIdMap* pending_outputs,
-    autovector<MemTable*>* to_delete, Directory* db_directory,
-    LogBuffer* log_buffer) {
+    uint64_t file_number, autovector<MemTable*>* to_delete,
+    Directory* db_directory, LogBuffer* log_buffer) {
   mu->AssertHeld();
 
   // flush was sucessful
@@ -220,11 +217,6 @@ Status MemTableList::InstallMemtableFlushResults(
         current_->Remove(m);
         assert(m->file_number_ > 0);
 
-        // pending_outputs can be cleared only after the newly created file
-        // has been written to a committed version so that other concurrently
-        // executing compaction threads do not mistakenly assume that this
-        // file is not live.
-        pending_outputs->erase(m->file_number_);
         if (m->Unref() != nullptr) {
           to_delete->push_back(m);
         }
@@ -237,7 +229,6 @@ Status MemTableList::InstallMemtableFlushResults(
         m->flush_in_progress_ = false;
         m->edit_.Clear();
         num_flush_not_started_++;
-        pending_outputs->erase(m->file_number_);
         m->file_number_ = 0;
         imm_flush_needed.store(true, std::memory_order_release);
       }

db/memtable_list.h

@@ -108,16 +108,14 @@ class MemTableList {
   // Reset status of the given memtable list back to pending state so that
   // they can get picked up again on the next round of flush.
   void RollbackMemtableFlush(const autovector<MemTable*>& mems,
-                             uint64_t file_number,
-                             FileNumToPathIdMap* pending_outputs);
+                             uint64_t file_number);
 
   // Commit a successful flush in the manifest file
   Status InstallMemtableFlushResults(
       ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
       const autovector<MemTable*>& m, VersionSet* vset, port::Mutex* mu,
-      uint64_t file_number, FileNumToPathIdMap* pending_outputs,
-      autovector<MemTable*>* to_delete, Directory* db_directory,
-      LogBuffer* log_buffer);
+      uint64_t file_number, autovector<MemTable*>* to_delete,
+      Directory* db_directory, LogBuffer* log_buffer);
 
   // New memtables are inserted at the front of the list.
   // Takes ownership of the referenced held on *m by the caller of Add().

db/merge_test.cc

@@ -23,15 +23,11 @@ using namespace std;
 using namespace rocksdb;
 
 namespace {
-  int numMergeOperatorCalls;
-  void resetNumMergeOperatorCalls() {
-    numMergeOperatorCalls = 0;
-  }
+size_t num_merge_operator_calls;
+void resetNumMergeOperatorCalls() { num_merge_operator_calls = 0; }
 
-  int num_partial_merge_calls;
-  void resetNumPartialMergeCalls() {
-    num_partial_merge_calls = 0;
-  }
+size_t num_partial_merge_calls;
+void resetNumPartialMergeCalls() { num_partial_merge_calls = 0; }
 }
 
 class CountMergeOperator : public AssociativeMergeOperator {
@@ -45,7 +41,7 @@ class CountMergeOperator : public AssociativeMergeOperator {
                      const Slice& value,
                      std::string* new_value,
                      Logger* logger) const override {
-    ++numMergeOperatorCalls;
+    ++num_merge_operator_calls;
     if (existing_value == nullptr) {
       new_value->assign(value.data(), value.size());
       return true;
@@ -307,31 +303,31 @@ void testCounters(Counters& counters, DB* db, bool test_compaction) {
   }
 }
 
-void testSuccessiveMerge(
-    Counters& counters, int max_num_merges, int num_merges) {
+void testSuccessiveMerge(Counters& counters, size_t max_num_merges,
+                         size_t num_merges) {
 
   counters.assert_remove("z");
   uint64_t sum = 0;
 
-  for (int i = 1; i <= num_merges; ++i) {
+  for (size_t i = 1; i <= num_merges; ++i) {
     resetNumMergeOperatorCalls();
     counters.assert_add("z", i);
     sum += i;
 
     if (i % (max_num_merges + 1) == 0) {
-      assert(numMergeOperatorCalls == max_num_merges + 1);
+      assert(num_merge_operator_calls == max_num_merges + 1);
     } else {
-      assert(numMergeOperatorCalls == 0);
+      assert(num_merge_operator_calls == 0);
     }
 
     resetNumMergeOperatorCalls();
     assert(counters.assert_get("z") == sum);
-    assert(numMergeOperatorCalls == i % (max_num_merges + 1));
+    assert(num_merge_operator_calls == i % (max_num_merges + 1));
   }
 }
 
-void testPartialMerge(Counters* counters, DB* db, int max_merge, int min_merge,
-                      int count) {
+void testPartialMerge(Counters* counters, DB* db, size_t max_merge,
+                      size_t min_merge, size_t count) {
   FlushOptions o;
   o.wait = true;
 
@@ -339,7 +335,7 @@ void testPartialMerge(Counters* counters, DB* db, int max_merge, int min_merge,
   //              operands exceeds the threshold.
   uint64_t tmp_sum = 0;
   resetNumPartialMergeCalls();
-  for (int i = 1; i <= count; i++) {
+  for (size_t i = 1; i <= count; i++) {
     counters->assert_add("b", i);
     tmp_sum += i;
   }
@@ -348,7 +344,7 @@ void testPartialMerge(Counters* counters, DB* db, int max_merge, int min_merge,
   ASSERT_EQ(tmp_sum, counters->assert_get("b"));
   if (count > max_merge) {
     // in this case, FullMerge should be called instead.
-    ASSERT_EQ(num_partial_merge_calls, 0);
+    ASSERT_EQ(num_partial_merge_calls, 0U);
   } else {
     // if count >= min_merge, then partial merge should be called once.
     ASSERT_EQ((count >= min_merge), (num_partial_merge_calls == 1));
@@ -358,20 +354,18 @@ void testPartialMerge(Counters* counters, DB* db, int max_merge, int min_merge,
   resetNumPartialMergeCalls();
   tmp_sum = 0;
   db->Put(rocksdb::WriteOptions(), "c", "10");
-  for (int i = 1; i <= count; i++) {
+  for (size_t i = 1; i <= count; i++) {
     counters->assert_add("c", i);
     tmp_sum += i;
   }
   db->Flush(o);
   db->CompactRange(nullptr, nullptr);
   ASSERT_EQ(tmp_sum, counters->assert_get("c"));
-  ASSERT_EQ(num_partial_merge_calls, 0);
+  ASSERT_EQ(num_partial_merge_calls, 0U);
 }
 
-void testSingleBatchSuccessiveMerge(
-    DB* db,
-    int max_num_merges,
-    int num_merges) {
+void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges,
+                                    size_t num_merges) {
   assert(num_merges > max_num_merges);
 
   Slice key("BatchSuccessiveMerge");
@@ -380,7 +374,7 @@ void testSingleBatchSuccessiveMerge(
 
   // Create the batch
   WriteBatch batch;
-  for (int i = 0; i < num_merges; ++i) {
+  for (size_t i = 0; i < num_merges; ++i) {
     batch.Merge(key, merge_value_slice);
   }
 
@@ -390,8 +384,9 @@ void testSingleBatchSuccessiveMerge(
     Status s = db->Write(WriteOptions(), &batch);
     assert(s.ok());
   }
-  assert(numMergeOperatorCalls ==
-      num_merges - (num_merges % (max_num_merges + 1)));
+  ASSERT_EQ(
+      num_merge_operator_calls,
+      static_cast<size_t>(num_merges - (num_merges % (max_num_merges + 1))));
 
   // Get the value
   resetNumMergeOperatorCalls();
@@ -403,7 +398,8 @@ void testSingleBatchSuccessiveMerge(
   assert(get_value_str.size() == sizeof(uint64_t));
   uint64_t get_value = DecodeFixed64(&get_value_str[0]);
   ASSERT_EQ(get_value, num_merges * merge_value);
-  ASSERT_EQ(numMergeOperatorCalls, (num_merges % (max_num_merges + 1)));
+  ASSERT_EQ(num_merge_operator_calls,
+            static_cast<size_t>((num_merges % (max_num_merges + 1))));
 }
 
 void runTest(int argc, const string& dbname, const bool use_ttl = false) {

db/perf_context_test.cc

@@ -38,8 +38,13 @@ std::shared_ptr<DB> OpenDb(bool read_only = false) {
       FLAGS_min_write_buffer_number_to_merge;
 
     if (FLAGS_use_set_based_memetable) {
+#ifndef ROCKSDB_LITE
       options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(0));
       options.memtable_factory.reset(NewHashSkipListRepFactory());
+#else
+      fprintf(stderr, "Prefix hash is not supported in lite mode\n");
+      exit(1);
+#endif  // ROCKSDB_LITE
     }
 
     Status s;
@@ -88,22 +93,27 @@ TEST(PerfContextTest, SeekIntoDeletion) {
     hist_get_time.Add(elapsed_nanos);
   }
 
-  std::cout << "Get uesr key comparison: \n" << hist_get.ToString()
+  std::cout << "Get user key comparison: \n" << hist_get.ToString()
             << "Get time: \n" << hist_get_time.ToString();
 
-  HistogramImpl hist_seek_to_first;
-  std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
+  {
+    HistogramImpl hist_seek_to_first;
+    std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
 
-  perf_context.Reset();
-  StopWatchNano timer(Env::Default(), true);
-  iter->SeekToFirst();
-  hist_seek_to_first.Add(perf_context.user_key_comparison_count);
-  auto elapsed_nanos = timer.ElapsedNanos();
+    perf_context.Reset();
+    StopWatchNano timer(Env::Default(), true);
+    iter->SeekToFirst();
+    hist_seek_to_first.Add(perf_context.user_key_comparison_count);
+    auto elapsed_nanos = timer.ElapsedNanos();
 
-  std::cout << "SeekToFirst uesr key comparison: \n" << hist_seek_to_first.ToString()
-            << "ikey skipped: " << perf_context.internal_key_skipped_count << "\n"
-            << "idelete skipped: " << perf_context.internal_delete_skipped_count << "\n"
-            << "elapsed: " << elapsed_nanos << "\n";
+    std::cout << "SeekToFirst uesr key comparison: \n"
+              << hist_seek_to_first.ToString()
+              << "ikey skipped: " << perf_context.internal_key_skipped_count
+              << "\n"
+              << "idelete skipped: "
+              << perf_context.internal_delete_skipped_count << "\n"
+              << "elapsed: " << elapsed_nanos << "\n";
+  }
 
   HistogramImpl hist_seek;
   for (int i = 0; i < FLAGS_total_keys; ++i) {
@@ -224,7 +234,6 @@ void ProfileQueries(bool enabled_time = false) {
     std::string key = "k" + std::to_string(i);
     std::string value = "v" + std::to_string(i);
 
-    std::vector<Slice> keys = {Slice(key)};
     std::vector<std::string> values;
 
     perf_context.Reset();
@@ -239,7 +248,7 @@ void ProfileQueries(bool enabled_time = false) {
     std::string key = "k" + std::to_string(i);
     std::string value = "v" + std::to_string(i);
 
-    std::vector<Slice> keys = {Slice(key)};
+    std::vector<Slice> multiget_keys = {Slice(key)};
     std::vector<std::string> values;
 
     perf_context.Reset();
@@ -252,7 +261,7 @@ void ProfileQueries(bool enabled_time = false) {
     hist_get.Add(perf_context.user_key_comparison_count);
 
     perf_context.Reset();
-    db->MultiGet(read_options, keys, &values);
+    db->MultiGet(read_options, multiget_keys, &values);
     hist_mget_snapshot.Add(perf_context.get_snapshot_time);
     hist_mget_memtable.Add(perf_context.get_from_memtable_time);
     hist_mget_files.Add(perf_context.get_from_output_files_time);
@@ -329,7 +338,7 @@ void ProfileQueries(bool enabled_time = false) {
     std::string key = "k" + std::to_string(i);
     std::string value = "v" + std::to_string(i);
 
-    std::vector<Slice> keys = {Slice(key)};
+    std::vector<Slice> multiget_keys = {Slice(key)};
     std::vector<std::string> values;
 
     perf_context.Reset();
@@ -342,7 +351,7 @@ void ProfileQueries(bool enabled_time = false) {
     hist_get.Add(perf_context.user_key_comparison_count);
 
     perf_context.Reset();
-    db->MultiGet(read_options, keys, &values);
+    db->MultiGet(read_options, multiget_keys, &values);
     hist_mget_snapshot.Add(perf_context.get_snapshot_time);
     hist_mget_memtable.Add(perf_context.get_from_memtable_time);
     hist_mget_files.Add(perf_context.get_from_output_files_time);

db/plain_table_db_test.cc

@@ -158,7 +158,7 @@ class PlainTableDBTest {
   // Return spread of files per level
   std::string FilesPerLevel() {
     std::string result;
-    int last_non_zero_offset = 0;
+    size_t last_non_zero_offset = 0;
     for (int level = 0; level < db_->NumberLevels(); level++) {
       int f = NumTableFilesAtLevel(level);
       char buf[100];

db/prefix_test.cc

@@ -29,14 +29,14 @@ using GFLAGS::ParseCommandLineFlags;
 
 DEFINE_bool(trigger_deadlock, false,
             "issue delete in range scan to trigger PrefixHashMap deadlock");
-DEFINE_uint64(bucket_count, 100000, "number of buckets");
+DEFINE_int32(bucket_count, 100000, "number of buckets");
 DEFINE_uint64(num_locks, 10001, "number of locks");
 DEFINE_bool(random_prefix, false, "randomize prefix");
 DEFINE_uint64(total_prefixes, 100000, "total number of prefixes");
 DEFINE_uint64(items_per_prefix, 1, "total number of values per prefix");
 DEFINE_int64(write_buffer_size, 33554432, "");
-DEFINE_int64(max_write_buffer_number, 2, "");
-DEFINE_int64(min_write_buffer_number_to_merge, 1, "");
+DEFINE_int32(max_write_buffer_number, 2, "");
+DEFINE_int32(min_write_buffer_number_to_merge, 1, "");
 DEFINE_int32(skiplist_height, 4, "");
 DEFINE_int32(memtable_prefix_bloom_bits, 10000000, "");
 DEFINE_int32(memtable_prefix_bloom_probes, 10, "");

db/skiplist_test.cc

@@ -253,11 +253,10 @@ class ConcurrentTest {
         // Note that generation 0 is never inserted, so it is ok if
         // <*,0,*> is missing.
         ASSERT_TRUE((gen(pos) == 0U) ||
-                    (gen(pos) > (uint64_t)initial_state.Get(key(pos)))
-                    ) << "key: " << key(pos)
-                      << "; gen: " << gen(pos)
-                      << "; initgen: "
-                      << initial_state.Get(key(pos));
+                    (gen(pos) > static_cast<uint64_t>(initial_state.Get(
+                                    static_cast<int>(key(pos))))))
+            << "key: " << key(pos) << "; gen: " << gen(pos)
+            << "; initgen: " << initial_state.Get(static_cast<int>(key(pos)));
 
         // Advance to next key in the valid key space
         if (key(pos) < key(current)) {

db/version_builder.cc

@@ -284,7 +284,7 @@ class VersionBuilder::Rep {
     if (levels_[level].deleted_files.count(f->fd.GetNumber()) > 0) {
       // File is deleted: do nothing
     } else {
-      vstorage->MaybeAddFile(level, f);
+      vstorage->AddFile(level, f);
     }
   }
 };

db/version_builder_test.cc

@@ -14,31 +14,31 @@ namespace rocksdb {
 
 class VersionBuilderTest {
  public:
-  const Comparator* ucmp;
-  InternalKeyComparator icmp;
-  Options options;
-  ImmutableCFOptions ioptions;
-  MutableCFOptions mutable_cf_options;
-  VersionStorageInfo vstorage;
-  uint32_t file_num;
-  CompactionOptionsFIFO fifo_options;
-  std::vector<uint64_t> size_being_compacted;
+  const Comparator* ucmp_;
+  InternalKeyComparator icmp_;
+  Options options_;
+  ImmutableCFOptions ioptions_;
+  MutableCFOptions mutable_cf_options_;
+  VersionStorageInfo vstorage_;
+  uint32_t file_num_;
+  CompactionOptionsFIFO fifo_options_;
+  std::vector<uint64_t> size_being_compacted_;
 
   VersionBuilderTest()
-      : ucmp(BytewiseComparator()),
-        icmp(ucmp),
-        ioptions(options),
-        mutable_cf_options(options, ioptions),
-        vstorage(&icmp, ucmp, options.num_levels, kCompactionStyleLevel,
+      : ucmp_(BytewiseComparator()),
+        icmp_(ucmp_),
+        ioptions_(options_),
+        mutable_cf_options_(options_, ioptions_),
+        vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
                  nullptr),
-        file_num(1) {
-    mutable_cf_options.RefreshDerivedOptions(ioptions);
-    size_being_compacted.resize(options.num_levels);
+        file_num_(1) {
+    mutable_cf_options_.RefreshDerivedOptions(ioptions_);
+    size_being_compacted_.resize(options_.num_levels);
   }
 
   ~VersionBuilderTest() {
-    for (int i = 0; i < vstorage.num_levels(); i++) {
-      for (auto* f : vstorage.LevelFiles(i)) {
+    for (int i = 0; i < vstorage_.num_levels(); i++) {
+      for (auto* f : vstorage_.LevelFiles(i)) {
         if (--f->refs == 0) {
           delete f;
         }
@@ -54,25 +54,33 @@ class VersionBuilderTest {
   void Add(int level, uint32_t file_number, const char* smallest,
            const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
            SequenceNumber smallest_seq = 100,
-           SequenceNumber largest_seq = 100) {
-    assert(level < vstorage.num_levels());
+           SequenceNumber largest_seq = 100,
+           uint64_t num_entries = 0, uint64_t num_deletions = 0,
+           bool sampled = false) {
+    assert(level < vstorage_.num_levels());
     FileMetaData* f = new FileMetaData;
     f->fd = FileDescriptor(file_number, path_id, file_size);
     f->smallest = GetInternalKey(smallest, smallest_seq);
     f->largest = GetInternalKey(largest, largest_seq);
     f->compensated_file_size = file_size;
     f->refs = 0;
-    vstorage.MaybeAddFile(level, f);
+    f->num_entries = num_entries;
+    f->num_deletions = num_deletions;
+    vstorage_.AddFile(level, f);
+    if (sampled) {
+      f->init_stats_from_file = true;
+      vstorage_.UpdateAccumulatedStats(f);
+    }
   }
 
   void UpdateVersionStorageInfo() {
-    vstorage.ComputeCompactionScore(mutable_cf_options, fifo_options,
-                                    size_being_compacted);
-    vstorage.UpdateFilesBySize();
-    vstorage.UpdateNumNonEmptyLevels();
-    vstorage.GenerateFileIndexer();
-    vstorage.GenerateLevelFilesBrief();
-    vstorage.SetFinalized();
+    vstorage_.ComputeCompactionScore(mutable_cf_options_, fifo_options_,
+                                    size_being_compacted_);
+    vstorage_.UpdateFilesBySize();
+    vstorage_.UpdateNumNonEmptyLevels();
+    vstorage_.GenerateFileIndexer();
+    vstorage_.GenerateLevelFilesBrief();
+    vstorage_.SetFinalized();
   }
 };
 
@@ -99,9 +107,9 @@ TEST(VersionBuilderTest, ApplyAndSaveTo) {
 
   EnvOptions env_options;
 
-  VersionBuilder version_builder(env_options, nullptr, &vstorage);
+  VersionBuilder version_builder(env_options, nullptr, &vstorage_);
 
-  VersionStorageInfo new_vstorage(&icmp, ucmp, options.num_levels,
+  VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
                                   kCompactionStyleLevel, nullptr);
   version_builder.Apply(&version_edit);
   version_builder.SaveTo(&new_vstorage);
@@ -118,6 +126,28 @@ TEST(VersionBuilderTest, ApplyAndSaveTo) {
   }
 }
 
+TEST(VersionBuilderTest, EstimatedActiveKeys) {
+  const uint32_t kTotalSamples = 20;
+  const uint32_t kNumLevels = 5;
+  const uint32_t kFilesPerLevel = 8;
+  const uint32_t kNumFiles = kNumLevels * kFilesPerLevel;
+  const uint32_t kEntriesPerFile = 1000;
+  const uint32_t kDeletionsPerFile = 100;
+  for (uint32_t i = 0; i < kNumFiles; ++i) {
+    Add(static_cast<int>(i / kFilesPerLevel), i + 1,
+        std::to_string((i + 100) * 1000).c_str(),
+        std::to_string((i + 100) * 1000 + 999).c_str(),
+        100U,  0, 100, 100,
+        kEntriesPerFile, kDeletionsPerFile,
+        (i < kTotalSamples));
+  }
+  // minus 2X for the number of deletion entries because:
+  // 1x for deletion entry does not count as a data entry.
+  // 1x for each deletion entry will actually remove one data entry.
+  ASSERT_EQ(vstorage_.GetEstimatedActiveKeys(),
+            (kEntriesPerFile - 2 * kDeletionsPerFile) * kNumFiles);
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

db/version_edit.h

@@ -160,7 +160,7 @@ class VersionEdit {
   // Add the specified file at the specified number.
   // REQUIRES: This version has not been saved (see VersionSet::SaveTo)
   // REQUIRES: "smallest" and "largest" are smallest and largest keys in file
-  void AddFile(int level, uint64_t file, uint64_t file_path_id,
+  void AddFile(int level, uint64_t file, uint32_t file_path_id,
                uint64_t file_size, const InternalKey& smallest,
                const InternalKey& largest, const SequenceNumber& smallest_seqno,
                const SequenceNumber& largest_seqno) {
@@ -180,9 +180,7 @@ class VersionEdit {
   }
 
   // Number of edits
-  int NumEntries() {
-    return new_files_.size() + deleted_files_.size();
-  }
+  size_t NumEntries() { return new_files_.size() + deleted_files_.size(); }
 
   bool IsColumnFamilyManipulation() {
     return is_column_family_add_ || is_column_family_drop_;

db/version_edit_test.cc

@@ -26,11 +26,12 @@ class VersionEditTest { };
 
 TEST(VersionEditTest, EncodeDecode) {
   static const uint64_t kBig = 1ull << 50;
+  static const uint32_t kBig32Bit = 1ull << 30;
 
   VersionEdit edit;
   for (int i = 0; i < 4; i++) {
     TestEncodeDecode(edit);
-    edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, 0,
+    edit.AddFile(3, kBig + 300 + i, kBig32Bit + 400 + i, 0,
                  InternalKey("foo", kBig + 500 + i, kTypeValue),
                  InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
                  kBig + 500 + i, kBig + 600 + i);

db/version_set.cc

@@ -14,13 +14,14 @@
 #endif
 
 #include <inttypes.h>
+#include <stdio.h>
 #include <algorithm>
 #include <map>
 #include <set>
 #include <climits>
 #include <unordered_map>
 #include <vector>
-#include <stdio.h>
+#include <string>
 
 #include "db/filename.h"
 #include "db/log_reader.h"
@@ -200,8 +201,8 @@ class FilePicker {
  private:
   unsigned int num_levels_;
   unsigned int curr_level_;
-  int search_left_bound_;
-  int search_right_bound_;
+  int32_t search_left_bound_;
+  int32_t search_right_bound_;
 #ifndef NDEBUG
   std::vector<FileMetaData*>* files_;
 #endif
@@ -257,11 +258,13 @@ class FilePicker {
           start_index = search_left_bound_;
         } else if (search_left_bound_ < search_right_bound_) {
           if (search_right_bound_ == FileIndexer::kLevelMaxIndex) {
-            search_right_bound_ = curr_file_level_->num_files - 1;
+            search_right_bound_ =
+                static_cast<int32_t>(curr_file_level_->num_files) - 1;
           }
-          start_index = FindFileInRange(*internal_comparator_,
-              *curr_file_level_, ikey_,
-              search_left_bound_, search_right_bound_);
+          start_index =
+              FindFileInRange(*internal_comparator_, *curr_file_level_, ikey_,
+                              static_cast<uint32_t>(search_left_bound_),
+                              static_cast<uint32_t>(search_right_bound_));
         } else {
           // search_left_bound > search_right_bound, key does not exist in
           // this level. Since no comparision is done in this level, it will
@@ -314,7 +317,8 @@ Version::~Version() {
 int FindFile(const InternalKeyComparator& icmp,
              const LevelFilesBrief& file_level,
              const Slice& key) {
-  return FindFileInRange(icmp, file_level, key, 0, file_level.num_files);
+  return FindFileInRange(icmp, file_level, key, 0,
+                         static_cast<uint32_t>(file_level.num_files));
 }
 
 void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
@@ -411,7 +415,7 @@ class LevelFileNumIterator : public Iterator {
                        const LevelFilesBrief* flevel)
       : icmp_(icmp),
         flevel_(flevel),
-        index_(flevel->num_files),
+        index_(static_cast<uint32_t>(flevel->num_files)),
         current_value_(0, 0, 0) {  // Marks as invalid
   }
   virtual bool Valid() const {
@@ -422,7 +426,9 @@ class LevelFileNumIterator : public Iterator {
   }
   virtual void SeekToFirst() { index_ = 0; }
   virtual void SeekToLast() {
-    index_ = (flevel_->num_files == 0) ? 0 : flevel_->num_files - 1;
+    index_ = (flevel_->num_files == 0)
+                 ? 0
+                 : static_cast<uint32_t>(flevel_->num_files) - 1;
   }
   virtual void Next() {
     assert(Valid());
@@ -431,7 +437,7 @@ class LevelFileNumIterator : public Iterator {
   virtual void Prev() {
     assert(Valid());
     if (index_ == 0) {
-      index_ = flevel_->num_files;  // Marks as invalid
+      index_ = static_cast<uint32_t>(flevel_->num_files);  // Marks as invalid
     } else {
       index_--;
     }
@@ -599,6 +605,49 @@ size_t Version::GetMemoryUsageByTableReaders() {
   return total_usage;
 }
 
+void Version::GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta) {
+  assert(cf_meta);
+  assert(cfd_);
+
+  cf_meta->name = cfd_->GetName();
+  cf_meta->size = 0;
+  cf_meta->file_count = 0;
+  cf_meta->levels.clear();
+
+  auto* ioptions = cfd_->ioptions();
+  auto* vstorage = storage_info();
+
+  for (int level = 0; level < cfd_->NumberLevels(); level++) {
+    uint64_t level_size = 0;
+    cf_meta->file_count += vstorage->LevelFiles(level).size();
+    std::vector<SstFileMetaData> files;
+    for (const auto& file : vstorage->LevelFiles(level)) {
+      uint32_t path_id = file->fd.GetPathId();
+      std::string file_path;
+      if (path_id < ioptions->db_paths.size()) {
+        file_path = ioptions->db_paths[path_id].path;
+      } else {
+        assert(!ioptions->db_paths.empty());
+        file_path = ioptions->db_paths.back().path;
+      }
+      files.emplace_back(
+          MakeTableFileName("", file->fd.GetNumber()),
+          file_path,
+          file->fd.GetFileSize(),
+          file->smallest_seqno,
+          file->largest_seqno,
+          file->smallest.user_key().ToString(),
+          file->largest.user_key().ToString(),
+          file->being_compacted);
+      level_size += file->fd.GetFileSize();
+    }
+    cf_meta->levels.emplace_back(
+        level, level_size, std::move(files));
+    cf_meta->size += level_size;
+  }
+}
+
+
 uint64_t VersionStorageInfo::GetEstimatedActiveKeys() const {
   // Estimation will be not accurate when:
   // (1) there is merge keys
@@ -609,11 +658,16 @@ uint64_t VersionStorageInfo::GetEstimatedActiveKeys() const {
     return 0;
   }
 
-  if (num_samples_ < files_->size()) {
+  uint64_t file_count = 0;
+  for (int level = 0; level < num_levels_; ++level) {
+    file_count += files_[level].size();
+  }
+
+  if (num_samples_ < file_count) {
     // casting to avoid overflowing
     return static_cast<uint64_t>(static_cast<double>(
         accumulated_num_non_deletions_ - accumulated_num_deletions_) *
-        files_->size() / num_samples_);
+        static_cast<double>(file_count) / num_samples_);
   } else {
     return accumulated_num_non_deletions_ - accumulated_num_deletions_;
   }
@@ -624,6 +678,11 @@ void Version::AddIterators(const ReadOptions& read_options,
                            MergeIteratorBuilder* merge_iter_builder) {
   assert(storage_info_.finalized_);
 
+  if (storage_info_.num_non_empty_levels() == 0) {
+    // No file in the Version.
+    return;
+  }
+
   // Merge all level zero files together since they may overlap
   for (size_t i = 0; i < storage_info_.LevelFilesBrief(0).num_files; i++) {
     const auto& file = storage_info_.LevelFilesBrief(0).files[i];
@@ -635,8 +694,8 @@ void Version::AddIterators(const ReadOptions& read_options,
   // For levels > 0, we can use a concatenating iterator that sequentially
   // walks through the non-overlapping files in the level, opening them
   // lazily.
-  for (int level = 1; level < storage_info_.num_levels(); level++) {
-    if (storage_info_.level_files_brief_[level].num_files != 0) {
+  for (int level = 1; level < storage_info_.num_non_empty_levels(); level++) {
+    if (storage_info_.LevelFilesBrief(level).num_files != 0) {
       merge_iter_builder->AddIterator(NewTwoLevelIterator(
           new LevelFileIteratorState(
               cfd_->table_cache(), read_options, soptions,
@@ -657,7 +716,7 @@ VersionStorageInfo::VersionStorageInfo(
       user_comparator_(user_comparator),
       // cfd is nullptr if Version is dummy
       num_levels_(levels),
-      num_non_empty_levels_(num_levels_),
+      num_non_empty_levels_(0),
       file_indexer_(user_comparator),
       compaction_style_(compaction_style),
       files_(new std::vector<FileMetaData*>[num_levels_]),
@@ -992,7 +1051,7 @@ bool CompareCompensatedSizeDescending(const Fsize& first, const Fsize& second) {
 
 } // anonymous namespace
 
-void VersionStorageInfo::MaybeAddFile(int level, FileMetaData* f) {
+void VersionStorageInfo::AddFile(int level, FileMetaData* f) {
   assert(level < num_levels());
   auto* level_files = &files_[level];
   // Must not overlap
@@ -1066,22 +1125,6 @@ bool Version::Unref() {
   return false;
 }
 
-bool VersionStorageInfo::NeedsCompaction() const {
-  // In universal compaction case, this check doesn't really
-  // check the compaction condition, but checks num of files threshold
-  // only. We are not going to miss any compaction opportunity
-  // but it's likely that more compactions are scheduled but
-  // ending up with nothing to do. We can improve it later.
-  // TODO(sdong): improve this function to be accurate for universal
-  //              compactions.
-  for (int i = 0; i <= MaxInputLevel(); i++) {
-    if (compaction_score_[i] >= 1) {
-      return true;
-    }
-  }
-  return false;
-}
-
 bool VersionStorageInfo::OverlapInLevel(int level,
                                         const Slice* smallest_user_key,
                                         const Slice* largest_user_key) {
@@ -1169,7 +1212,7 @@ void VersionStorageInfo::GetOverlappingInputs(
           i = 0;
         }
       } else if (file_index) {
-        *file_index = i-1;
+        *file_index = static_cast<int>(i) - 1;
       }
     }
   }
@@ -1185,7 +1228,7 @@ void VersionStorageInfo::GetOverlappingInputsBinarySearch(
   assert(level > 0);
   int min = 0;
   int mid = 0;
-  int max = files_[level].size() -1;
+  int max = static_cast<int>(files_[level].size()) - 1;
   bool foundOverlap = false;
   const Comparator* user_cmp = user_comparator_;
 
@@ -1569,7 +1612,7 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
   if (!descriptor_log_ ||
       manifest_file_size_ > db_options_->max_manifest_file_size) {
     pending_manifest_file_number_ = NewFileNumber();
-    batch_edits.back()->SetNextFile(next_file_number_);
+    batch_edits.back()->SetNextFile(next_file_number_.load());
     new_descriptor_log = true;
   } else {
     pending_manifest_file_number_ = manifest_file_number_;
@@ -1770,7 +1813,7 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
 
 void VersionSet::LogAndApplyCFHelper(VersionEdit* edit) {
   assert(edit->IsColumnFamilyManipulation());
-  edit->SetNextFile(next_file_number_);
+  edit->SetNextFile(next_file_number_.load());
   edit->SetLastSequence(last_sequence_);
   if (edit->is_column_family_drop_) {
     // if we drop column family, we have to make sure to save max column family,
@@ -1787,13 +1830,13 @@ void VersionSet::LogAndApplyHelper(ColumnFamilyData* cfd,
 
   if (edit->has_log_number_) {
     assert(edit->log_number_ >= cfd->GetLogNumber());
-    assert(edit->log_number_ < next_file_number_);
+    assert(edit->log_number_ < next_file_number_.load());
   }
 
   if (!edit->has_prev_log_number_) {
     edit->SetPrevLogNumber(prev_log_number_);
   }
-  edit->SetNextFile(next_file_number_);
+  edit->SetNextFile(next_file_number_.load());
   edit->SetLastSequence(last_sequence_);
 
   builder->Apply(edit);
@@ -2020,8 +2063,8 @@ Status VersionSet::Recover(
 
     column_family_set_->UpdateMaxColumnFamily(max_column_family);
 
-    MarkFileNumberUsed(previous_log_number);
-    MarkFileNumberUsed(log_number);
+    MarkFileNumberUsedDuringRecovery(previous_log_number);
+    MarkFileNumberUsedDuringRecovery(log_number);
   }
 
   // there were some column families in the MANIFEST that weren't specified
@@ -2061,7 +2104,7 @@ Status VersionSet::Recover(
     }
 
     manifest_file_size_ = current_manifest_file_size;
-    next_file_number_ = next_file + 1;
+    next_file_number_.store(next_file + 1);
     last_sequence_ = last_sequence;
     prev_log_number_ = previous_log_number;
 
@@ -2072,7 +2115,7 @@ Status VersionSet::Recover(
         "prev_log_number is %lu,"
         "max_column_family is %u\n",
         manifest_filename.c_str(), (unsigned long)manifest_file_number_,
-        (unsigned long)next_file_number_, (unsigned long)last_sequence_,
+        (unsigned long)next_file_number_.load(), (unsigned long)last_sequence_,
         (unsigned long)log_number, (unsigned long)prev_log_number_,
         column_family_set_->GetMaxColumnFamily());
 
@@ -2408,14 +2451,14 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
       delete v;
     }
 
-    next_file_number_ = next_file + 1;
+    next_file_number_.store(next_file + 1);
     last_sequence_ = last_sequence;
     prev_log_number_ = previous_log_number;
 
     printf(
         "next_file_number %lu last_sequence "
         "%lu  prev_log_number %lu max_column_family %u\n",
-        (unsigned long)next_file_number_, (unsigned long)last_sequence,
+        (unsigned long)next_file_number_.load(), (unsigned long)last_sequence,
         (unsigned long)previous_log_number,
         column_family_set_->GetMaxColumnFamily());
   }
@@ -2424,9 +2467,11 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
 }
 #endif  // ROCKSDB_LITE
 
-void VersionSet::MarkFileNumberUsed(uint64_t number) {
-  if (next_file_number_ <= number) {
-    next_file_number_ = number + 1;
+void VersionSet::MarkFileNumberUsedDuringRecovery(uint64_t number) {
+  // only called during recovery which is single threaded, so this works because
+  // there can't be concurrent calls
+  if (next_file_number_.load(std::memory_order_relaxed) <= number) {
+    next_file_number_.store(number + 1, std::memory_order_relaxed);
   }
 }
 
@@ -2574,7 +2619,7 @@ void VersionSet::AddLiveFiles(std::vector<FileDescriptor>* live_list) {
   }
 
   // just one time extension to the right size
-  live_list->reserve(live_list->size() + total_files);
+  live_list->reserve(live_list->size() + static_cast<size_t>(total_files));
 
   for (auto cfd : *column_family_set_) {
     Version* dummy_versions = cfd->dummy_versions();
@@ -2600,12 +2645,12 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
   // Level-0 files have to be merged together.  For other levels,
   // we will make a concatenating iterator per level.
   // TODO(opt): use concatenating iterator for level-0 if there is no overlap
-  const int space = (c->level() == 0 ?
-      c->input_levels(0)->num_files + c->num_input_levels() - 1:
-      c->num_input_levels());
-  Iterator** list = new Iterator*[space];
-  int num = 0;
-  for (int which = 0; which < c->num_input_levels(); which++) {
+  const size_t space = (c->level() == 0 ? c->input_levels(0)->num_files +
+                                              c->num_input_levels() - 1
+                                        : c->num_input_levels());
+  Iterator** list = new Iterator* [space];
+  size_t num = 0;
+  for (size_t which = 0; which < c->num_input_levels(); which++) {
     if (c->input_levels(which)->num_files != 0) {
       if (c->level(which) == 0) {
         const LevelFilesBrief* flevel = c->input_levels(which);
@@ -2627,8 +2672,9 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
     }
   }
   assert(num <= space);
-  Iterator* result = NewMergingIterator(
-      &c->column_family_data()->internal_comparator(), list, num);
+  Iterator* result =
+      NewMergingIterator(&c->column_family_data()->internal_comparator(), list,
+                         static_cast<int>(num));
   delete[] list;
   return result;
 }
@@ -2645,41 +2691,22 @@ bool VersionSet::VerifyCompactionFileConsistency(Compaction* c) {
         c->column_family_data()->GetName().c_str());
   }
 
-  // verify files in level
-  int level = c->level();
-  for (int i = 0; i < c->num_input_files(0); i++) {
-    uint64_t number = c->input(0, i)->fd.GetNumber();
-
-    // look for this file in the current version
-    bool found = false;
-    for (unsigned int j = 0; j < vstorage->files_[level].size(); j++) {
-      FileMetaData* f = vstorage->files_[level][j];
-      if (f->fd.GetNumber() == number) {
-        found = true;
-        break;
+  for (size_t input = 0; input < c->num_input_levels(); ++input) {
+    int level = c->level(input);
+    for (size_t i = 0; i < c->num_input_files(input); ++i) {
+      uint64_t number = c->input(input, i)->fd.GetNumber();
+      bool found = false;
+      for (unsigned int j = 0; j < vstorage->files_[level].size(); j++) {
+        FileMetaData* f = vstorage->files_[level][j];
+        if (f->fd.GetNumber() == number) {
+          found = true;
+          break;
+        }
       }
-    }
-    if (!found) {
-      return false; // input files non existant in current version
-    }
-  }
-  // verify level+1 files
-  level++;
-  for (int i = 0; i < c->num_input_files(1); i++) {
-    uint64_t number = c->input(1, i)->fd.GetNumber();
-
-    // look for this file in the current version
-    bool found = false;
-    for (unsigned int j = 0; j < vstorage->files_[level].size(); j++) {
-      FileMetaData* f = vstorage->files_[level][j];
-      if (f->fd.GetNumber() == number) {
-        found = true;
-        break;
+      if (!found) {
+        return false;  // input files non existent in current version
       }
     }
-    if (!found) {
-      return false; // input files non existant in current version
-    }
   }
 #endif
   return true;     // everything good

db/version_set.h

@@ -94,7 +94,7 @@ class VersionStorageInfo {
 
   void Reserve(int level, size_t size) { files_[level].reserve(size); }
 
-  void MaybeAddFile(int level, FileMetaData* f);
+  void AddFile(int level, FileMetaData* f);
 
   void SetFinalized() { finalized_ = true; }
 
@@ -128,9 +128,6 @@ class VersionStorageInfo {
 
   int MaxInputLevel() const;
 
-  // Returns true iff some level needs a compaction.
-  bool NeedsCompaction() const;
-
   // Returns the maxmimum compaction score for levels 1 to max
   double max_compaction_score() const { return max_compaction_score_; }
 
@@ -194,7 +191,7 @@ class VersionStorageInfo {
   // REQUIRES: This version has been saved (see VersionSet::SaveTo)
   int NumLevelFiles(int level) const {
     assert(finalized_);
-    return files_[level].size();
+    return static_cast<int>(files_[level].size());
   }
 
   // Return the combined file size of all files at the specified level.
@@ -206,6 +203,7 @@ class VersionStorageInfo {
   }
 
   const rocksdb::LevelFilesBrief& LevelFilesBrief(int level) const {
+    assert(level < static_cast<int>(level_files_brief_.size()));
     return level_files_brief_[level];
   }
 
@@ -420,6 +418,8 @@ class Version {
 
   VersionSet* version_set() { return vset_; }
 
+  void GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta);
+
  private:
   friend class VersionSet;
 
@@ -530,17 +530,10 @@ class VersionSet {
     return pending_manifest_file_number_;
   }
 
-  // Allocate and return a new file number
-  uint64_t NewFileNumber() { return next_file_number_++; }
+  uint64_t current_next_file_number() const { return next_file_number_.load(); }
 
-  // Arrange to reuse "file_number" unless a newer file number has
-  // already been allocated.
-  // REQUIRES: "file_number" was returned by a call to NewFileNumber().
-  void ReuseLogFileNumber(uint64_t file_number) {
-    if (next_file_number_ == file_number + 1) {
-      next_file_number_ = file_number;
-    }
-  }
+  // Allocate and return a new file number
+  uint64_t NewFileNumber() { return next_file_number_.fetch_add(1); }
 
   // Return the last sequence number.
   uint64_t LastSequence() const {
@@ -554,7 +547,8 @@ class VersionSet {
   }
 
   // Mark the specified file number as used.
-  void MarkFileNumberUsed(uint64_t number);
+  // REQUIRED: this is only called during single-threaded recovery
+  void MarkFileNumberUsedDuringRecovery(uint64_t number);
 
   // Return the log file number for the log file that is currently
   // being compacted, or zero if there is no such log file.
@@ -595,7 +589,7 @@ class VersionSet {
   Status GetMetadataForFile(uint64_t number, int* filelevel,
                             FileMetaData** metadata, ColumnFamilyData** cfd);
 
-  void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
+  void GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata);
 
   void GetObsoleteFiles(std::vector<FileMetaData*>* files);
 
@@ -606,6 +600,7 @@ class VersionSet {
   struct ManifestWriter;
 
   friend class Version;
+  friend class DBImpl;
 
   struct LogReporter : public log::Reader::Reporter {
     Status* status;
@@ -630,7 +625,7 @@ class VersionSet {
   Env* const env_;
   const std::string dbname_;
   const DBOptions* const db_options_;
-  uint64_t next_file_number_;
+  std::atomic<uint64_t> next_file_number_;
   uint64_t manifest_file_number_;
   uint64_t pending_manifest_file_number_;
   std::atomic<uint64_t> last_sequence_;

db/wal_manager.h

@@ -38,9 +38,9 @@ class WalManager {
         env_(db_options.env),
         purge_wal_files_last_run_(0) {}
 
-  virtual Status GetSortedWalFiles(VectorLogPtr& files);
+  Status GetSortedWalFiles(VectorLogPtr& files);
 
-  virtual Status GetUpdatesSince(
+  Status GetUpdatesSince(
       SequenceNumber seq_number, std::unique_ptr<TransactionLogIterator>* iter,
       const TransactionLogIterator::ReadOptions& read_options,
       VersionSet* version_set);

db/write_batch.cc

@@ -374,7 +374,7 @@ class MemTableInserter : public WriteBatch::Handler {
         Status s = db_->Get(ropts, cf_handle, key, &prev_value);
 
         char* prev_buffer = const_cast<char*>(prev_value.c_str());
-        uint32_t prev_size = prev_value.size();
+        uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
         auto status = moptions->inplace_callback(s.ok() ? prev_buffer : nullptr,
                                                  s.ok() ? &prev_size : nullptr,
                                                  value, &merged_value);

examples/Makefile

@@ -1,8 +1,8 @@
 include ../build_config.mk
 
-.PHONY: main clean
+.PHONY: clean
 
-all: simple_example column_families_example
+all: simple_example column_families_example compact_files_example
 
 simple_example: simple_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
@@ -10,5 +10,8 @@ simple_example: simple_example.cc
 column_families_example: column_families_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
 
-clean: simple_example column_families_example
-	rm -rf ./simple_example ./column_families_example
+compact_files_example: compact_files_example.cc
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
+
+clean: simple_example column_families_example compact_files_example
+	rm -rf ./simple_example ./column_families_example ./compact_files_example

examples/compact_files_example.cc

@@ -0,0 +1,175 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+//
+// An example code demonstrating how to use CompactFiles, EventListener,
+// and GetColumnFamilyMetaData APIs to implement custom compaction algorithm.
+
+#include <mutex>
+#include <string>
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+
+using namespace rocksdb;
+std::string kDBPath = "/tmp/rocksdb_compact_files_example";
+class CompactionTask;
+
+// This is an example interface of external-compaction algorithm.
+// Compaction algorithm can be implemented outside the core-RocksDB
+// code by using the pluggable compaction APIs that RocksDb provides.
+class Compactor : public EventListener {
+ public:
+  // Picks and returns a compaction task given the specified DB
+  // and column family.  It is the caller's responsibility to
+  // destroy the returned CompactionTask.  Returns "nullptr"
+  // if it cannot find a proper compaction task.
+  virtual CompactionTask* PickCompaction(
+      DB* db, const std::string& cf_name) = 0;
+
+  // Schedule and run the specified compaction task in background.
+  virtual void ScheduleCompaction(CompactionTask *task) = 0;
+};
+
+// Example structure that describes a compaction task.
+struct CompactionTask {
+  CompactionTask(
+      DB* db, Compactor* compactor,
+      const std::string& column_family_name,
+      const std::vector<std::string>& input_file_names,
+      const int output_level,
+      const CompactionOptions& compact_options,
+      bool retry_on_fail)
+          : db(db),
+            compactor(compactor),
+            column_family_name(column_family_name),
+            input_file_names(input_file_names),
+            output_level(output_level),
+            compact_options(compact_options),
+            retry_on_fail(false) {}
+  DB* db;
+  Compactor* compactor;
+  const std::string& column_family_name;
+  std::vector<std::string> input_file_names;
+  int output_level;
+  CompactionOptions compact_options;
+  bool retry_on_fail;
+};
+
+// A simple compaction algorithm that always compacts everything
+// to the highest level whenever possible.
+class FullCompactor : public Compactor {
+ public:
+  explicit FullCompactor(const Options options) : options_(options) {
+    compact_options_.compression = options_.compression;
+    compact_options_.output_file_size_limit =
+        options_.target_file_size_base;
+  }
+
+  // When flush happens, it determins whether to trigger compaction.
+  // If triggered_writes_stop is true, it will also set the retry
+  // flag of compaction-task to true.
+  void OnFlushCompleted(
+      DB* db, const std::string& cf_name,
+      const std::string& file_path,
+      bool triggered_writes_slowdown,
+      bool triggered_writes_stop) override {
+    CompactionTask* task = PickCompaction(db, cf_name);
+    if (task != nullptr) {
+      if (triggered_writes_stop) {
+        task->retry_on_fail = true;
+      }
+      // Schedule compaction in a different thread.
+      ScheduleCompaction(task);
+    }
+  }
+
+  // Always pick a compaction which includes all files whenever possible.
+  CompactionTask* PickCompaction(
+      DB* db, const std::string& cf_name) override {
+    ColumnFamilyMetaData cf_meta;
+    db->GetColumnFamilyMetaData(&cf_meta);
+
+    std::vector<std::string> input_file_names;
+    for (auto level : cf_meta.levels) {
+      for (auto file : level.files) {
+        if (file.being_compacted) {
+          return nullptr;
+        }
+        input_file_names.push_back(file.name);
+      }
+    }
+    return new CompactionTask(
+        db, this, cf_name, input_file_names,
+        options_.num_levels - 1, compact_options_, false);
+  }
+
+  // Schedule the specified compaction task in background.
+  void ScheduleCompaction(CompactionTask* task) override {
+    options_.env->Schedule(&FullCompactor::CompactFiles, task);
+  }
+
+  static void CompactFiles(void* arg) {
+    CompactionTask* task = reinterpret_cast<CompactionTask*>(arg);
+    assert(task);
+    assert(task->db);
+    Status s = task->db->CompactFiles(
+        task->compact_options,
+        task->input_file_names,
+        task->output_level);
+    printf("CompactFiles() finished with status %s\n", s.ToString().c_str());
+    if (!s.ok() && !s.IsIOError() && task->retry_on_fail) {
+      // If a compaction task with its retry_on_fail=true failed,
+      // try to schedule another compaction in case the reason
+      // is not an IO error.
+      CompactionTask* new_task = task->compactor->PickCompaction(
+          task->db, task->column_family_name);
+      task->compactor->ScheduleCompaction(new_task);
+    }
+    // release the task
+    delete task;
+  }
+
+ private:
+  Options options_;
+  CompactionOptions compact_options_;
+};
+
+int main() {
+  Options options;
+  options.create_if_missing = true;
+  // Disable RocksDB background compaction.
+  options.compaction_style = kCompactionStyleNone;
+  // Small slowdown and stop trigger for experimental purpose.
+  options.level0_slowdown_writes_trigger = 3;
+  options.level0_stop_writes_trigger = 5;
+  options.IncreaseParallelism(5);
+  options.listeners.emplace_back(new FullCompactor(options));
+
+  DB* db = nullptr;
+  DestroyDB(kDBPath, options);
+  Status s = DB::Open(options, kDBPath, &db);
+  assert(s.ok());
+  assert(db);
+
+  // if background compaction is not working, write will stall
+  // because of options.level0_stop_writes_trigger
+  for (int i = 1000; i < 99999; ++i) {
+    db->Put(WriteOptions(), std::to_string(i),
+                            std::string(500, 'a' + (i % 26)));
+  }
+
+  // verify the values are still there
+  std::string value;
+  for (int i = 1000; i < 99999; ++i) {
+    db->Get(ReadOptions(), std::to_string(i),
+                           &value);
+    assert(value == std::string(500, 'a' + (i % 26)));
+  }
+
+  // close the db.
+  delete db;
+
+  return 0;
+}

include/rocksdb/comparator.h

@@ -63,7 +63,7 @@ class Comparator {
 extern const Comparator* BytewiseComparator();
 
 // Return a builtin comparator that uses reverse lexicographic byte-wise
-// ordering. 
+// ordering.
 extern const Comparator* ReverseBytewiseComparator();
 
 }  // namespace rocksdb

include/rocksdb/db.h

@@ -15,19 +15,34 @@
 #include <vector>
 #include <string>
 #include <unordered_map>
+#include "rocksdb/metadata.h"
 #include "rocksdb/version.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/options.h"
 #include "rocksdb/types.h"
 #include "rocksdb/transaction_log.h"
+#include "rocksdb/listener.h"
 
 namespace rocksdb {
 
+struct Options;
+struct DBOptions;
+struct ColumnFamilyOptions;
+struct ReadOptions;
+struct WriteOptions;
+struct FlushOptions;
+struct CompactionOptions;
+struct TableProperties;
+class WriteBatch;
+class Env;
+class EventListener;
+
 using std::unique_ptr;
 
 class ColumnFamilyHandle {
  public:
   virtual ~ColumnFamilyHandle() {}
+  virtual const std::string& GetName() const = 0;
 };
 extern const std::string kDefaultColumnFamilyName;
 
@@ -44,27 +59,6 @@ struct ColumnFamilyDescriptor {
 static const int kMajorVersion = __ROCKSDB_MAJOR__;
 static const int kMinorVersion = __ROCKSDB_MINOR__;
 
-struct Options;
-struct ReadOptions;
-struct WriteOptions;
-struct FlushOptions;
-struct TableProperties;
-class WriteBatch;
-class Env;
-
-// Metadata associated with each SST file.
-struct LiveFileMetaData {
-  std::string column_family_name;  // Name of the column family
-  std::string db_path;
-  std::string name;                // Name of the file
-  int level;               // Level at which this file resides.
-  size_t size;             // File size in bytes.
-  std::string smallestkey; // Smallest user defined key in the file.
-  std::string largestkey;  // Largest user defined key in the file.
-  SequenceNumber smallest_seqno; // smallest seqno in file
-  SequenceNumber largest_seqno;  // largest seqno in file
-};
-
 // Abstract handle to particular state of a DB.
 // A Snapshot is an immutable object and can therefore be safely
 // accessed from multiple threads without any external synchronization.
@@ -370,6 +364,26 @@ class DB {
     return SetOptions(DefaultColumnFamily(), new_options);
   }
 
+  // CompactFiles() inputs a list of files specified by file numbers
+  // and compacts them to the specified level.  Note that the behavior
+  // is different from CompactRange in that CompactFiles() will
+  // perform the compaction job using the CURRENT thread.
+  //
+  // @see GetDataBaseMetaData
+  // @see GetColumnFamilyMetaData
+  virtual Status CompactFiles(
+      const CompactionOptions& compact_options,
+      ColumnFamilyHandle* column_family,
+      const std::vector<std::string>& input_file_names,
+      const int output_level, const int output_path_id = -1) = 0;
+
+  virtual Status CompactFiles(
+      const CompactionOptions& compact_options,
+      const std::vector<std::string>& input_file_names,
+      const int output_level, const int output_path_id = -1) {
+    return CompactFiles(compact_options, DefaultColumnFamily(),
+                        input_file_names, output_level, output_path_id);
+  }
   // Number of levels used for this DB.
   virtual int NumberLevels(ColumnFamilyHandle* column_family) = 0;
   virtual int NumberLevels() { return NumberLevels(DefaultColumnFamily()); }
@@ -476,6 +490,21 @@ class DB {
   // and end key
   virtual void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {}
 
+  // Obtains the meta data of the specified column family of the DB.
+  // Status::NotFound() will be returned if the current DB does not have
+  // any column family match the specified name.
+  //
+  // If cf_name is not specified, then the metadata of the default
+  // column family will be returned.
+  virtual void GetColumnFamilyMetaData(
+      ColumnFamilyHandle* column_family,
+      ColumnFamilyMetaData* metadata) {}
+
+  // Get the metadata of the default column family.
+  void GetColumnFamilyMetaData(
+      ColumnFamilyMetaData* metadata) {
+    GetColumnFamilyMetaData(DefaultColumnFamily(), metadata);
+  }
 #endif  // ROCKSDB_LITE
 
   // Sets the globally unique ID created at database creation time by invoking

include/rocksdb/env.h

@@ -20,6 +20,7 @@
 #include <cstdarg>
 #include <string>
 #include <memory>
+#include <limits>
 #include <vector>
 #include <stdint.h>
 #include "rocksdb/status.h"
@@ -476,8 +477,8 @@ class WritableFile {
     if (new_last_preallocated_block > last_preallocated_block_) {
       size_t num_spanned_blocks =
         new_last_preallocated_block - last_preallocated_block_;
-      Allocate(block_size * last_preallocated_block_,
-               block_size * num_spanned_blocks);
+      Allocate(static_cast<off_t>(block_size * last_preallocated_block_),
+               static_cast<off_t>(block_size * num_spanned_blocks));
       last_preallocated_block_ = new_last_preallocated_block;
     }
   }
@@ -580,7 +581,8 @@ enum InfoLogLevel : unsigned char {
 // An interface for writing log messages.
 class Logger {
  public:
-  enum { DO_NOT_SUPPORT_GET_LOG_FILE_SIZE = -1 };
+  size_t kDoNotSupportGetLogFileSize = std::numeric_limits<size_t>::max();
+
   explicit Logger(const InfoLogLevel log_level = InfoLogLevel::INFO_LEVEL)
       : log_level_(log_level) {}
   virtual ~Logger();
@@ -613,9 +615,7 @@ class Logger {
       Logv(new_format, ap);
     }
   }
-  virtual size_t GetLogFileSize() const {
-    return DO_NOT_SUPPORT_GET_LOG_FILE_SIZE;
-  }
+  virtual size_t GetLogFileSize() const { return kDoNotSupportGetLogFileSize; }
   // Flush to the OS buffers
   virtual void Flush() {}
   virtual InfoLogLevel GetInfoLogLevel() const { return log_level_; }

include/rocksdb/immutable_options.h

@@ -90,6 +90,12 @@ struct ImmutableCFOptions {
   Options::AccessHint access_hint_on_compaction_start;
 
   int num_levels;
+
+#ifndef ROCKSDB_LITE
+  // A vector of EventListeners which call-back functions will be called
+  // when specific RocksDB event happens.
+  std::vector<std::shared_ptr<EventListener>> listeners;
+#endif  // ROCKSDB_LITE
 };
 
 }  // namespace rocksdb

include/rocksdb/listener.h

@@ -0,0 +1,66 @@
+// Copyright (c) 2014 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#ifndef ROCKSDB_LITE
+
+#include <string>
+#include "rocksdb/status.h"
+
+namespace rocksdb {
+
+class DB;
+class Status;
+
+// EventListener class contains a set of call-back functions that will
+// be called when specific RocksDB event happens such as flush.  It can
+// be used as a building block for developing custom features such as
+// stats-collector or external compaction algorithm.
+//
+// Note that call-back functions should not run for an extended period of
+// time before the function returns, otherwise RocksDB may be blocked.
+// For example, it is not suggested to do DB::CompactFiles() (as it may
+// run for a long while) or issue many of DB::Put() (as Put may be blocked
+// in certain cases) in the same thread in the EventListener callback.
+// However, doing DB::CompactFiles() and DB::Put() in another thread is
+// considered safe.
+//
+// [Threading] All EventListener callback will be called using the
+// actual thread that involves in that specific event.   For example, it
+// is the RocksDB background flush thread that does the actual flush to
+// call EventListener::OnFlushCompleted().
+class EventListener {
+ public:
+  // A call-back function to RocksDB which will be called whenever a
+  // registered RocksDB flushes a file.  The default implementation is
+  // no-op.
+  //
+  // Note that the this function must be implemented in a way such that
+  // it should not run for an extended period of time before the function
+  // returns.  Otherwise, RocksDB may be blocked.
+  //
+  // @param db a pointer to the rocksdb instance which just flushed
+  //     a memtable to disk.
+  // @param column_family_id the id of the flushed column family.
+  // @param file_path the path to the newly created file.
+  // @param triggered_writes_slowdown true when rocksdb is currently
+  //     slowing-down all writes to prevent creating too many Level 0
+  //     files as compaction seems not able to catch up the write request
+  //     speed.  This indicates that there're too many files in Level 0.
+  // @param triggered_writes_stop true when rocksdb is currently blocking
+  //     any writes to prevent creating more L0 files.  This indicates that
+  //     there're too many files in level 0.  Compactions should try to
+  //     compact L0 files down to lower levels as soon as possible.
+  virtual void OnFlushCompleted(
+      DB* db, const std::string& column_family_name,
+      const std::string& file_path,
+      bool triggered_writes_slowdown,
+      bool triggered_writes_stop) {}
+  virtual ~EventListener() {}
+};
+
+}  // namespace rocksdb
+
+#endif  // ROCKSDB_LITE

include/rocksdb/metadata.h

@@ -0,0 +1,90 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+#include <limits>
+#include <string>
+#include <vector>
+
+#include "rocksdb/types.h"
+
+#pragma once
+
+namespace rocksdb {
+struct ColumnFamilyMetaData;
+struct LevelMetaData;
+struct SstFileMetaData;
+
+// The metadata that describes a column family.
+struct ColumnFamilyMetaData {
+  ColumnFamilyMetaData() : size(0), name("") {}
+  ColumnFamilyMetaData(const std::string& _name, uint64_t _size,
+                       const std::vector<LevelMetaData>&& _levels) :
+      size(_size), name(_name), levels(_levels) {}
+
+  // The size of this column family in bytes, which is equal to the sum of
+  // the file size of its "levels".
+  uint64_t size;
+  // The number of files in this column family.
+  size_t file_count;
+  // The name of the column family.
+  std::string name;
+  // The metadata of all levels in this column family.
+  std::vector<LevelMetaData> levels;
+};
+
+// The metadata that describes a level.
+struct LevelMetaData {
+  LevelMetaData(int _level, uint64_t _size,
+                const std::vector<SstFileMetaData>&& _files) :
+      level(_level), size(_size),
+      files(_files) {}
+
+  // The level which this meta data describes.
+  const int level;
+  // The size of this level in bytes, which is equal to the sum of
+  // the file size of its "files".
+  const uint64_t size;
+  // The metadata of all sst files in this level.
+  const std::vector<SstFileMetaData> files;
+};
+
+// The metadata that describes a SST file.
+struct SstFileMetaData {
+  SstFileMetaData() {}
+  SstFileMetaData(const std::string& _file_name,
+                  const std::string& _path, uint64_t _size,
+                  SequenceNumber _smallest_seqno,
+                  SequenceNumber _largest_seqno,
+                  const std::string& _smallestkey,
+                  const std::string& _largestkey,
+                  bool _being_compacted) :
+    size(_size), name(_file_name),
+    db_path(_path), smallest_seqno(_smallest_seqno), largest_seqno(_largest_seqno),
+    smallestkey(_smallestkey), largestkey(_largestkey),
+    being_compacted(_being_compacted) {}
+
+  // File size in bytes.
+  uint64_t size;
+  // The name of the file.
+  std::string name;
+  // The full path where the file locates.
+  std::string db_path;
+
+  SequenceNumber smallest_seqno;  // Smallest sequence number in file.
+  SequenceNumber largest_seqno;   // Largest sequence number in file.
+  std::string smallestkey;     // Smallest user defined key in the file.
+  std::string largestkey;      // Largest user defined key in the file.
+  bool being_compacted;  // true if the file is currently being compacted.
+};
+
+// The full set of metadata associated with each SST file.
+struct LiveFileMetaData : SstFileMetaData {
+  std::string column_family_name;  // Name of the column family
+  int level;               // Level at which this file resides.
+};
+
+
+
+}  // namespace rocksdb

include/rocksdb/options.h

@@ -10,13 +10,16 @@
 #define STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_
 
 #include <stddef.h>
+#include <stdint.h>
 #include <string>
 #include <memory>
 #include <vector>
+#include <limits>
 #include <stdint.h>
 #include <unordered_map>
 
 #include "rocksdb/version.h"
+#include "rocksdb/listener.h"
 #include "rocksdb/universal_compaction.h"
 
 namespace rocksdb {
@@ -55,10 +58,11 @@ enum CompressionType : char {
 enum CompactionStyle : char {
   kCompactionStyleLevel = 0x0,      // level based compaction style
   kCompactionStyleUniversal = 0x1,  // Universal compaction style
-  kCompactionStyleFIFO = 0x2,       // FIFO compaction style
+  kCompactionStyleFIFO = 0x2,    // FIFO compaction style
+  kCompactionStyleNone = 0x3,  // Disable background compaction. Compaction
+                               // jobs are submitted via CompactFiles()
 };
 
-
 struct CompactionOptionsFIFO {
   // once the total sum of table files reaches this, we will delete the oldest
   // table file
@@ -97,6 +101,7 @@ struct Options;
 struct ColumnFamilyOptions {
   // Some functions that make it easier to optimize RocksDB
 
+#ifndef ROCKSDB_LITE
   // Use this if you don't need to keep the data sorted, i.e. you'll never use
   // an iterator, only Put() and Get() API calls
   ColumnFamilyOptions* OptimizeForPointLookup(
@@ -120,6 +125,7 @@ struct ColumnFamilyOptions {
       uint64_t memtable_memory_budget = 512 * 1024 * 1024);
   ColumnFamilyOptions* OptimizeUniversalStyleCompaction(
       uint64_t memtable_memory_budget = 512 * 1024 * 1024);
+#endif  // ROCKSDB_LITE
 
   // -------------------
   // Parameters that affect behavior
@@ -586,6 +592,12 @@ struct ColumnFamilyOptions {
   // Default: 2
   uint32_t min_partial_merge_operands;
 
+#ifndef ROCKSDB_LITE
+  // A vector of EventListeners which call-back functions will be called
+  // when specific RocksDB event happens.
+  std::vector<std::shared_ptr<EventListener>> listeners;
+#endif  // ROCKSDB_LITE
+
   // Create ColumnFamilyOptions with default values for all fields
   ColumnFamilyOptions();
   // Create ColumnFamilyOptions from Options
@@ -597,12 +609,14 @@ struct ColumnFamilyOptions {
 struct DBOptions {
   // Some functions that make it easier to optimize RocksDB
 
+#ifndef ROCKSDB_LITE
   // By default, RocksDB uses only one background thread for flush and
   // compaction. Calling this function will set it up such that total of
   // `total_threads` is used. Good value for `total_threads` is the number of
   // cores. You almost definitely want to call this function if your system is
   // bottlenecked by RocksDB.
   DBOptions* IncreaseParallelism(int total_threads = 16);
+#endif  // ROCKSDB_LITE
 
   // If true, the database will be created if it is missing.
   // Default: false
@@ -616,14 +630,11 @@ struct DBOptions {
   // Default: false
   bool error_if_exists;
 
-  // If true, the implementation will do aggressive checking of the
-  // data it is processing and will stop early if it detects any
-  // errors.  This may have unforeseen ramifications: for example, a
-  // corruption of one DB entry may cause a large number of entries to
-  // become unreadable or for the entire DB to become unopenable.
-  // If any of the  writes to the database fails (Put, Delete, Merge, Write),
-  // the database will switch to read-only mode and fail all other
+  // If true, RocksDB will aggressively check consistency of the data.
+  // Also, if any of the  writes to the database fails (Put, Delete, Merge,
+  // Write), the database will switch to read-only mode and fail all other
   // Write operations.
+  // In most cases you want this to be set to true.
   // Default: true
   bool paranoid_checks;
 
@@ -1067,6 +1078,19 @@ extern Options GetOptions(size_t total_write_buffer_limit,
                           int write_amplification_threshold = 32,
                           uint64_t target_db_size = 68719476736 /* 64GB */);
 
+// CompactionOptions are used in CompactFiles() call.
+struct CompactionOptions {
+  // Compaction output compression type
+  // Default: snappy
+  CompressionType compression;
+  // Compaction will create files of size `output_file_size_limit`.
+  // Default: MAX, which means that compaction will create a single file
+  uint64_t output_file_size_limit;
+
+  CompactionOptions()
+      : compression(kSnappyCompression),
+        output_file_size_limit(std::numeric_limits<uint64_t>::max()) {}
+};
 }  // namespace rocksdb
 
 #endif  // STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_

include/rocksdb/slice.h

@@ -123,7 +123,7 @@ inline bool operator!=(const Slice& x, const Slice& y) {
 }
 
 inline int Slice::compare(const Slice& b) const {
-  const int min_len = (size_ < b.size_) ? size_ : b.size_;
+  const size_t min_len = (size_ < b.size_) ? size_ : b.size_;
   int r = memcmp(data_, b.data_, min_len);
   if (r == 0) {
     if (size_ < b.size_) r = -1;

include/rocksdb/sst_dump_tool.h

@@ -0,0 +1,17 @@
+// Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+#ifndef ROCKSDB_LITE
+#pragma once
+
+namespace rocksdb {
+
+class SSTDumpTool {
+ public:
+  void Run(int argc, char** argv);
+};
+
+}  // namespace rocksdb
+
+#endif  // ROCKSDB_LITE

include/rocksdb/status.h

@@ -61,6 +61,9 @@ class Status {
   static Status Incomplete(const Slice& msg, const Slice& msg2 = Slice()) {
     return Status(kIncomplete, msg, msg2);
   }
+  static Status ShutdownInProgress() {
+    return Status(kShutdownInProgress);
+  }
   static Status ShutdownInProgress(const Slice& msg,
                                    const Slice& msg2 = Slice()) {
     return Status(kShutdownInProgress, msg, msg2);
@@ -71,6 +74,12 @@ class Status {
   static Status TimedOut(const Slice& msg, const Slice& msg2 = Slice()) {
     return Status(kTimedOut, msg, msg2);
   }
+  static Status Aborted() {
+    return Status(kAborted);
+  }
+  static Status Aborted(const Slice& msg, const Slice& msg2 = Slice()) {
+    return Status(kAborted, msg, msg2);
+  }
 
   // Returns true iff the status indicates success.
   bool ok() const { return code() == kOk; }
@@ -101,6 +110,8 @@ class Status {
 
   bool IsTimedOut() const { return code() == kTimedOut; }
 
+  bool IsAborted() const { return code() == kAborted; }
+
   // Return a string representation of this status suitable for printing.
   // Returns the string "OK" for success.
   std::string ToString() const;
@@ -115,7 +126,8 @@ class Status {
     kMergeInProgress = 6,
     kIncomplete = 7,
     kShutdownInProgress = 8,
-    kTimedOut = 9
+    kTimedOut = 9,
+    kAborted = 10
   };
 
   Code code() const {

include/rocksdb/utilities/backupable_db.h

@@ -177,6 +177,8 @@ class BackupEngineReadOnly {
   // You can GetBackupInfo safely, even with other BackupEngine performing
   // backups on the same directory
   virtual void GetBackupInfo(std::vector<BackupInfo>* backup_info) = 0;
+  virtual void GetCorruptedBackups(
+      std::vector<BackupID>* corrupt_backup_ids) = 0;
 
   // Restoring DB from backup is NOT safe when there is another BackupEngine
   // running that might call DeleteBackup() or PurgeOldBackups(). It is caller's
@@ -196,7 +198,12 @@ class BackupEngine {
   virtual ~BackupEngine() {}
 
   static BackupEngine* NewBackupEngine(Env* db_env,
-                                       const BackupableDBOptions& options);
+                                       const BackupableDBOptions& options)
+    __attribute__((deprecated("Please use Open() instead")));
+
+  static Status Open(Env* db_env,
+                     const BackupableDBOptions& options,
+                     BackupEngine** backup_engine_ptr);
 
   virtual Status CreateNewBackup(DB* db, bool flush_before_backup = false) = 0;
   virtual Status PurgeOldBackups(uint32_t num_backups_to_keep) = 0;
@@ -204,12 +211,16 @@ class BackupEngine {
   virtual void StopBackup() = 0;
 
   virtual void GetBackupInfo(std::vector<BackupInfo>* backup_info) = 0;
+  virtual void GetCorruptedBackups(
+      std::vector<BackupID>* corrupt_backup_ids) = 0;
   virtual Status RestoreDBFromBackup(
       BackupID backup_id, const std::string& db_dir, const std::string& wal_dir,
       const RestoreOptions& restore_options = RestoreOptions()) = 0;
   virtual Status RestoreDBFromLatestBackup(
       const std::string& db_dir, const std::string& wal_dir,
       const RestoreOptions& restore_options = RestoreOptions()) = 0;
+
+  virtual Status GarbageCollect() = 0;
 };
 
 // Stack your DB with BackupableDB to be able to backup the DB
@@ -228,6 +239,8 @@ class BackupableDB : public StackableDB {
   Status CreateNewBackup(bool flush_before_backup = false);
   // Returns info about backups in backup_info
   void GetBackupInfo(std::vector<BackupInfo>* backup_info);
+  // Returns info about corrupt backups in corrupt_backups
+  void GetCorruptedBackups(std::vector<BackupID>* corrupt_backup_ids);
   // deletes old backups, keeping latest num_backups_to_keep alive
   Status PurgeOldBackups(uint32_t num_backups_to_keep);
   // deletes a specific backup
@@ -241,6 +254,11 @@ class BackupableDB : public StackableDB {
   // next time you create BackupableDB or RestoreBackupableDB.
   void StopBackup();
 
+  // Will delete all the files we don't need anymore
+  // It will do the full scan of the files/ directory and delete all the
+  // files that are not referenced.
+  Status GarbageCollect();
+
  private:
   BackupEngine* backup_engine_;
 };
@@ -253,6 +271,8 @@ class RestoreBackupableDB {
 
   // Returns info about backups in backup_info
   void GetBackupInfo(std::vector<BackupInfo>* backup_info);
+  // Returns info about corrupt backups in corrupt_backups
+  void GetCorruptedBackups(std::vector<BackupID>* corrupt_backup_ids);
 
   // restore from backup with backup_id
   // IMPORTANT -- if options_.share_table_files == true and you restore DB
@@ -279,6 +299,11 @@ class RestoreBackupableDB {
   // deletes a specific backup
   Status DeleteBackup(BackupID backup_id);
 
+  // Will delete all the files we don't need anymore
+  // It will do the full scan of the files/ directory and delete all the
+  // files that are not referenced.
+  Status GarbageCollect();
+
  private:
   BackupEngine* backup_engine_;
 };

include/rocksdb/utilities/convenience.h

@@ -11,6 +11,7 @@
 
 namespace rocksdb {
 
+#ifndef ROCKSDB_LITE
 // Take a map of option name and option value, apply them into the
 // base_options, and return the new options as a result
 bool GetColumnFamilyOptionsFromMap(
@@ -36,5 +37,6 @@ bool GetDBOptionsFromString(
     const DBOptions& base_options,
     const std::string& opts_str,
     DBOptions* new_options);
+#endif  // ROCKSDB_LITE
 
 }  // namespace rocksdb

include/rocksdb/utilities/spatial_db.h

@@ -222,7 +222,9 @@ class SpatialDB : public StackableDB {
 
   // Calling Compact() after inserting a bunch of elements should speed up
   // reading. This is especially useful if you use SpatialDBOptions::bulk_load
-  virtual Status Compact() = 0;
+  // Num threads determines how many threads we'll use for compactions. Setting
+  // this to bigger number will use more IO and CPU, but finish faster
+  virtual Status Compact(int num_threads = 1) = 0;
 
   // Query the specified spatial_index. Query will return all elements that
   // intersect bbox, but it may also return some extra elements.

include/rocksdb/utilities/stackable_db.h

@@ -133,6 +133,17 @@ class StackableDB : public DB {
                              target_level, target_path_id);
   }
 
+  using DB::CompactFiles;
+  virtual Status CompactFiles(
+      const CompactionOptions& compact_options,
+      ColumnFamilyHandle* column_family,
+      const std::vector<std::string>& input_file_names,
+      const int output_level, const int output_path_id = -1) override {
+    return db_->CompactFiles(
+        compact_options, column_family, input_file_names,
+        output_level, output_path_id);
+  }
+
   using DB::NumberLevels;
   virtual int NumberLevels(ColumnFamilyHandle* column_family) override {
     return db_->NumberLevels(column_family);
@@ -170,6 +181,8 @@ class StackableDB : public DB {
     return db_->Flush(fopts, column_family);
   }
 
+#ifndef ROCKSDB_LITE
+
   virtual Status DisableFileDeletions() override {
     return db_->DisableFileDeletions();
   }
@@ -183,6 +196,14 @@ class StackableDB : public DB {
     db_->GetLiveFilesMetaData(metadata);
   }
 
+  virtual void GetColumnFamilyMetaData(
+      ColumnFamilyHandle *column_family,
+      ColumnFamilyMetaData* cf_meta) override {
+    db_->GetColumnFamilyMetaData(column_family, cf_meta);
+  }
+
+#endif  // ROCKSDB_LITE
+
   virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs,
                               bool flush_memtable = true) override {
       return db_->GetLiveFiles(vec, mfs, flush_memtable);

include/rocksdb/version.h

@@ -5,7 +5,7 @@
 #pragma once
 
 #define ROCKSDB_MAJOR 3
-#define ROCKSDB_MINOR 7
+#define ROCKSDB_MINOR 8
 #define ROCKSDB_PATCH 0
 
 // Do not use these. We made the mistake of declaring macros starting with

java/HISTORY-JAVA.md

@@ -1,5 +1,12 @@
 # RocksJava Change Log
 
+## By 11/14/2014
+### New Features
+* Full support for Column Family.
+* Slice and Comparator support.
+* Default merge operator support.
+* RateLimiter support.
+
 ## By 06/15/2014
 ### New Features
 * Added basic Java binding for rocksdb::Env such that multiple RocksDB can share the same thread pool and environment.

java/Makefile

@@ -1,4 +1,40 @@
-NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.DBOptions org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.RocksIterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.BlockBasedTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter org.rocksdb.ComparatorOptions org.rocksdb.AbstractComparator org.rocksdb.Comparator org.rocksdb.DirectComparator org.rocksdb.AbstractSlice org.rocksdb.Slice org.rocksdb.DirectSlice org.rocksdb.RestoreOptions org.rocksdb.RestoreBackupableDB org.rocksdb.RocksEnv org.rocksdb.GenericRateLimiterConfig org.rocksdb.ColumnFamilyHandle org.rocksdb.MergeOperator org.rocksdb.StringAppendOperator org.rocksdb.ComparatorOptions org.rocksdb.AbstractComparator org.rocksdb.Comparator org.rocksdb.DirectComparator org.rocksdb.AbstractSlice org.rocksdb.Slice org.rocksdb.DirectSlice
+NATIVE_JAVA_CLASSES = org.rocksdb.AbstractComparator\
+	org.rocksdb.AbstractSlice\
+	org.rocksdb.BackupableDB\
+	org.rocksdb.BackupableDBOptions\
+	org.rocksdb.BlockBasedTableConfig\
+	org.rocksdb.BloomFilter\
+	org.rocksdb.ColumnFamilyHandle\
+	org.rocksdb.ColumnFamilyOptions\
+	org.rocksdb.Comparator\
+	org.rocksdb.ComparatorOptions\
+	org.rocksdb.DBOptions\
+	org.rocksdb.DirectComparator\
+	org.rocksdb.DirectSlice\
+	org.rocksdb.FlushOptions\
+	org.rocksdb.Filter\
+	org.rocksdb.GenericRateLimiterConfig\
+	org.rocksdb.HashLinkedListMemTableConfig\
+	org.rocksdb.HashSkipListMemTableConfig\
+	org.rocksdb.MergeOperator\
+	org.rocksdb.Options\
+	org.rocksdb.PlainTableConfig\
+	org.rocksdb.ReadOptions\
+	org.rocksdb.RestoreBackupableDB\
+	org.rocksdb.RestoreOptions\
+	org.rocksdb.RocksDB\
+	org.rocksdb.RocksEnv\
+	org.rocksdb.RocksIterator\
+	org.rocksdb.SkipListMemTableConfig\
+	org.rocksdb.Slice\
+	org.rocksdb.Statistics\
+	org.rocksdb.VectorMemTableConfig\
+	org.rocksdb.StringAppendOperator\
+	org.rocksdb.WriteBatch\
+	org.rocksdb.WriteBatch.Handler\
+	org.rocksdb.WriteBatchInternal\
+	org.rocksdb.WriteBatchTest\
+	org.rocksdb.WriteOptions\
 
 ROCKSDB_MAJOR = $(shell egrep "ROCKSDB_MAJOR.[0-9]" ../include/rocksdb/version.h | cut -d ' ' -f 3)
 ROCKSDB_MINOR = $(shell egrep "ROCKSDB_MINOR.[0-9]" ../include/rocksdb/version.h | cut -d ' ' -f 3)
@@ -34,6 +70,12 @@ sample: java
 	@rm -rf /tmp/rocksdbjni
 	@rm -rf /tmp/rocksdbjni_not_found
 
+column_family_sample: java
+	javac -cp $(ROCKSDB_JAR) RocksDBColumnFamilySample.java
+	@rm -rf /tmp/rocksdbjni
+	java -ea -Djava.library.path=.:../ -cp ".:./*" -Xcheck:jni RocksDBColumnFamilySample /tmp/rocksdbjni
+	@rm -rf /tmp/rocksdbjni
+
 test: java
 	@rm -rf /tmp/rocksdbjni_*
 	javac org/rocksdb/test/*.java
@@ -42,13 +84,16 @@ test: java
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.BlockBasedTableConfigTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.DBOptionsTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ColumnFamilyTest
+	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ColumnFamilyOptionsTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.FilterTest
+	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.FlushTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.KeyMayExistTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MemTableTest
+	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MergeTest
+	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MixedOptionsTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.OptionsTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.PlainTableConfigTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOnlyTest
-	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MergeTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOptionsTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.RocksIteratorTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.SnapshotTest
@@ -56,6 +101,7 @@ test: java
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ComparatorOptionsTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ComparatorTest
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.DirectComparatorTest
+	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.WriteBatchHandlerTest
 	@rm -rf /tmp/rocksdbjni_*
 
 db_bench: java

java/RocksDBColumnFamilySample.java

@@ -0,0 +1,91 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+import org.rocksdb.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RocksDBColumnFamilySample {
+  static {
+    RocksDB.loadLibrary();
+  }
+
+  public static void main(String[] args) throws RocksDBException {
+    if (args.length < 1) {
+      System.out.println(
+          "usage: RocksDBColumnFamilySample db_path");
+      return;
+    }
+    String db_path = args[0];
+
+    System.out.println("RocksDBColumnFamilySample");
+    RocksDB db = null;
+    Options options = null;
+    ColumnFamilyHandle columnFamilyHandle = null;
+    WriteBatch wb = null;
+    try {
+      options = new Options().setCreateIfMissing(true);
+      db = RocksDB.open(options, db_path);
+      assert(db != null);
+
+      // create column family
+      columnFamilyHandle = db.createColumnFamily(
+          new ColumnFamilyDescriptor("new_cf", new ColumnFamilyOptions()));
+      assert(columnFamilyHandle != null);
+
+    } finally {
+      if (columnFamilyHandle != null) {
+        columnFamilyHandle.dispose();
+      }
+      if (db != null) {
+        db.close();
+        db = null;
+      }
+    }
+
+    // open DB with two column families
+    List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
+    // have to open default column family
+    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(
+        RocksDB.DEFAULT_COLUMN_FAMILY, new ColumnFamilyOptions()));
+    // open the new one, too
+    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(
+        "new_cf", new ColumnFamilyOptions()));
+    List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
+    try {
+      db = RocksDB.open(new DBOptions(), db_path,
+          columnFamilyDescriptors, columnFamilyHandles);
+      assert(db != null);
+
+      // put and get from non-default column family
+      db.put(columnFamilyHandles.get(0), new WriteOptions(),
+          "key".getBytes(), "value".getBytes());
+      String value = new String(db.get(columnFamilyHandles.get(0),
+          "key".getBytes()));
+
+      // atomic write
+      wb = new WriteBatch();
+      wb.put(columnFamilyHandles.get(0), "key2".getBytes(), "value2".getBytes());
+      wb.put(columnFamilyHandles.get(1), "key3".getBytes(), "value3".getBytes());
+      wb.remove(columnFamilyHandles.get(0), "key".getBytes());
+      db.write(new WriteOptions(), wb);
+
+      // drop column family
+      db.dropColumnFamily(columnFamilyHandles.get(1));
+
+    } finally {
+      for (ColumnFamilyHandle handle : columnFamilyHandles){
+        handle.dispose();
+      }
+      if (db != null) {
+        db.close();
+      }
+      if (wb != null) {
+        wb.dispose();
+      }
+    }
+  }
+}

java/org/rocksdb/AbstractComparator.java

@@ -46,15 +46,18 @@ public abstract class AbstractComparator<T extends AbstractSlice>
   public abstract int compare(final T a, final T b);
 
   /**
-   * Used to reduce the space requirements
-   * for internal data structures like index blocks.
+   * <p>Used to reduce the space requirements
+   * for internal data structures like index blocks.</p>
    *
-   * If start &lt; limit, you may return a new start which is a
-   * shorter string in [start, limit).
+   * <p>If start &lt; limit, you may return a new start which is a
+   * shorter string in [start, limit).</p>
    *
-   * Simple comparator implementations may return null if they
+   * <p>Simple comparator implementations may return null if they
    * wish to use start unchanged. i.e., an implementation of
-   * this method that does nothing is correct.
+   * this method that does nothing is correct.</p>
+   *
+   * @param start String
+   * @param limit of type T
    *
    * @return a shorter start, or null
    */
@@ -63,15 +66,17 @@ public abstract class AbstractComparator<T extends AbstractSlice>
   }
 
   /**
-   * Used to reduce the space requirements
-   * for internal data structures like index blocks.
+   * <p>Used to reduce the space requirements
+   * for internal data structures like index blocks.</p>
    *
-   * You may return a new short key (key1) where
-   * key1 &ge; key.
+   * <p>You may return a new short key (key1) where
+   * key1 &ge; key.</p>
    *
-   * Simple comparator implementations may return null if they
+   * <p>Simple comparator implementations may return null if they
    * wish to leave the key unchanged. i.e., an implementation of
-   * this method that does nothing is correct.
+   * this method that does nothing is correct.</p>
+   *
+   * @param key String
    *
    * @return a shorter key, or null
    */

java/org/rocksdb/BackupableDB.java

@@ -23,6 +23,9 @@ public class BackupableDB extends RocksDB {
    * @param db_path Path to store data to. The path for storing the backup should be
    *     specified in the {@link org.rocksdb.BackupableDBOptions}.
    * @return BackupableDB reference to the opened database.
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public static BackupableDB open(
       Options opt, BackupableDBOptions bopt, String db_path)
@@ -45,7 +48,9 @@ public class BackupableDB extends RocksDB {
    *
    * @param flushBeforeBackup if true, then all data will be flushed
    *     before creating backup.
-   * @throws org.rocksdb.RocksDBException
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void createNewBackup(boolean flushBeforeBackup)
       throws RocksDBException {
@@ -56,7 +61,9 @@ public class BackupableDB extends RocksDB {
    * Deletes old backups, keeping latest numBackupsToKeep alive.
    *
    * @param numBackupsToKeep Number of latest backups to keep.
-   * @throws org.rocksdb.RocksDBException
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void purgeOldBackups(int numBackupsToKeep)
       throws RocksDBException {
@@ -67,7 +74,9 @@ public class BackupableDB extends RocksDB {
    * Deletes a specific backup.
    *
    * @param backupId of backup to delete.
-   * @throws org.rocksdb.RocksDBException
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void deleteBackup(int backupId) throws RocksDBException {
     deleteBackup0(nativeHandle_, backupId);
@@ -104,7 +113,7 @@ public class BackupableDB extends RocksDB {
     super();
   }
 
-  @Override protected void finalize() {
+  @Override protected void finalize() throws Throwable {
     close();
     super.finalize();
   }

java/org/rocksdb/ColumnFamilyDescriptor.java

@@ -0,0 +1,58 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * <p>Describes a column family with a
+ * name and respective Options.</p>
+ */
+public class ColumnFamilyDescriptor {
+
+  /**
+   * <p>Creates a new Column Family using a name and default
+   * options,</p>
+   *
+   * @param columnFamilyName name of column family.
+   */
+  public ColumnFamilyDescriptor(final String columnFamilyName){
+    this(columnFamilyName, new ColumnFamilyOptions());
+  }
+
+  /**
+   * <p>Creates a new Column Family using a name and custom
+   * options.</p>
+   *
+   * @param columnFamilyName name of column family.
+   * @param columnFamilyOptions options to be used with
+   *     column family.
+   */
+  public ColumnFamilyDescriptor(final String columnFamilyName,
+      final ColumnFamilyOptions columnFamilyOptions) {
+    columnFamilyName_ = columnFamilyName;
+    columnFamilyOptions_ = columnFamilyOptions;
+  }
+
+  /**
+   * Retrieve name of column family.
+   *
+   * @return column family name.
+   */
+  public String columnFamilyName() {
+    return columnFamilyName_;
+  }
+
+  /**
+   * Retrieve assigned options instance.
+   *
+   * @return Options instance assigned to this instance.
+   */
+  public ColumnFamilyOptions columnFamilyOptions() {
+    return columnFamilyOptions_;
+  }
+
+  private final String columnFamilyName_;
+  private final ColumnFamilyOptions columnFamilyOptions_;
+}

java/org/rocksdb/ColumnFamilyOptions.java

@@ -0,0 +1,647 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * ColumnFamilyOptions to control the behavior of a database.  It will be used
+ * during the creation of a {@link org.rocksdb.RocksDB} (i.e., RocksDB.open()).
+ *
+ * If {@link #dispose()} function is not called, then it will be GC'd automatically
+ * and native resources will be released as part of the process.
+ */
+public class ColumnFamilyOptions extends RocksObject
+    implements ColumnFamilyOptionsInterface {
+  static {
+    RocksDB.loadLibrary();
+  }
+
+  /**
+   * Construct ColumnFamilyOptions.
+   *
+   * This constructor will create (by allocating a block of memory)
+   * an {@code rocksdb::DBOptions} in the c++ side.
+   */
+  public ColumnFamilyOptions() {
+    super();
+    newColumnFamilyOptions();
+  }
+
+  @Override
+  public ColumnFamilyOptions optimizeForPointLookup(
+      long blockCacheSizeMb) {
+    optimizeForPointLookup(nativeHandle_,
+        blockCacheSizeMb);
+    return this;
+  }
+
+  @Override
+  public ColumnFamilyOptions optimizeLevelStyleCompaction() {
+    optimizeLevelStyleCompaction(nativeHandle_,
+        DEFAULT_COMPACTION_MEMTABLE_MEMORY_BUDGET);
+    return this;
+  }
+
+  @Override
+  public ColumnFamilyOptions optimizeLevelStyleCompaction(
+      long memtableMemoryBudget) {
+    optimizeLevelStyleCompaction(nativeHandle_,
+        memtableMemoryBudget);
+    return this;
+  }
+
+  @Override
+  public ColumnFamilyOptions optimizeUniversalStyleCompaction() {
+    optimizeUniversalStyleCompaction(nativeHandle_,
+        DEFAULT_COMPACTION_MEMTABLE_MEMORY_BUDGET);
+    return this;
+  }
+
+  @Override
+  public ColumnFamilyOptions optimizeUniversalStyleCompaction(
+      long memtableMemoryBudget) {
+    optimizeUniversalStyleCompaction(nativeHandle_,
+        memtableMemoryBudget);
+    return this;
+  }
+
+  @Override
+  public ColumnFamilyOptions setComparator(BuiltinComparator builtinComparator) {
+    assert(isInitialized());
+    setComparatorHandle(nativeHandle_, builtinComparator.ordinal());
+    return this;
+  }
+
+  @Override
+  public ColumnFamilyOptions setComparator(AbstractComparator comparator) {
+    assert (isInitialized());
+    setComparatorHandle(nativeHandle_, comparator.nativeHandle_);
+    comparator_ = comparator;
+    return this;
+  }
+
+  @Override
+  public ColumnFamilyOptions setMergeOperatorName(String name) {
+    setMergeOperatorName(nativeHandle_, name);
+    return this;
+  }
+
+  @Override
+  public ColumnFamilyOptions setMergeOperator(MergeOperator mergeOperator) {
+    setMergeOperator(nativeHandle_, mergeOperator.newMergeOperatorHandle());
+    return this;
+  }
+
+  @Override
+  public ColumnFamilyOptions setWriteBufferSize(long writeBufferSize)
+      throws RocksDBException {
+    assert(isInitialized());
+    setWriteBufferSize(nativeHandle_, writeBufferSize);
+    return this;
+  }
+
+  @Override
+  public long writeBufferSize()  {
+    assert(isInitialized());
+    return writeBufferSize(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMaxWriteBufferNumber(
+      int maxWriteBufferNumber) {
+    assert(isInitialized());
+    setMaxWriteBufferNumber(nativeHandle_, maxWriteBufferNumber);
+    return this;
+  }
+
+  @Override
+  public int maxWriteBufferNumber() {
+    assert(isInitialized());
+    return maxWriteBufferNumber(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMinWriteBufferNumberToMerge(
+      int minWriteBufferNumberToMerge) {
+    setMinWriteBufferNumberToMerge(nativeHandle_, minWriteBufferNumberToMerge);
+    return this;
+  }
+
+  @Override
+  public int minWriteBufferNumberToMerge() {
+    return minWriteBufferNumberToMerge(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions useFixedLengthPrefixExtractor(int n) {
+    assert(isInitialized());
+    useFixedLengthPrefixExtractor(nativeHandle_, n);
+    return this;
+  }
+
+  @Override
+  public ColumnFamilyOptions setCompressionType(CompressionType compressionType) {
+    setCompressionType(nativeHandle_, compressionType.getValue());
+    return this;
+  }
+
+  @Override
+  public CompressionType compressionType() {
+    return CompressionType.values()[compressionType(nativeHandle_)];
+  }
+
+  @Override
+  public ColumnFamilyOptions setNumLevels(int numLevels) {
+    setNumLevels(nativeHandle_, numLevels);
+    return this;
+  }
+
+  @Override
+  public int numLevels() {
+    return numLevels(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setLevelZeroFileNumCompactionTrigger(
+      int numFiles) {
+    setLevelZeroFileNumCompactionTrigger(
+        nativeHandle_, numFiles);
+    return this;
+  }
+
+  @Override
+  public int levelZeroFileNumCompactionTrigger() {
+    return levelZeroFileNumCompactionTrigger(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setLevelZeroSlowdownWritesTrigger(
+      int numFiles) {
+    setLevelZeroSlowdownWritesTrigger(nativeHandle_, numFiles);
+    return this;
+  }
+
+  @Override
+  public int levelZeroSlowdownWritesTrigger() {
+    return levelZeroSlowdownWritesTrigger(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setLevelZeroStopWritesTrigger(int numFiles) {
+    setLevelZeroStopWritesTrigger(nativeHandle_, numFiles);
+    return this;
+  }
+
+  @Override
+  public int levelZeroStopWritesTrigger() {
+    return levelZeroStopWritesTrigger(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMaxMemCompactionLevel(
+      int maxMemCompactionLevel) {
+    setMaxMemCompactionLevel(nativeHandle_, maxMemCompactionLevel);
+    return this;
+  }
+
+  @Override
+  public int maxMemCompactionLevel() {
+    return maxMemCompactionLevel(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setTargetFileSizeBase(long targetFileSizeBase) {
+    setTargetFileSizeBase(nativeHandle_, targetFileSizeBase);
+    return this;
+  }
+
+  @Override
+  public long targetFileSizeBase() {
+    return targetFileSizeBase(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setTargetFileSizeMultiplier(int multiplier) {
+    setTargetFileSizeMultiplier(nativeHandle_, multiplier);
+    return this;
+  }
+
+  @Override
+  public int targetFileSizeMultiplier() {
+    return targetFileSizeMultiplier(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMaxBytesForLevelBase(
+      long maxBytesForLevelBase) {
+    setMaxBytesForLevelBase(nativeHandle_, maxBytesForLevelBase);
+    return this;
+  }
+
+  @Override
+  public long maxBytesForLevelBase() {
+    return maxBytesForLevelBase(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMaxBytesForLevelMultiplier(int multiplier) {
+    setMaxBytesForLevelMultiplier(nativeHandle_, multiplier);
+    return this;
+  }
+
+  @Override
+  public int maxBytesForLevelMultiplier() {
+    return maxBytesForLevelMultiplier(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setExpandedCompactionFactor(int expandedCompactionFactor) {
+    setExpandedCompactionFactor(nativeHandle_, expandedCompactionFactor);
+    return this;
+  }
+
+  @Override
+  public int expandedCompactionFactor() {
+    return expandedCompactionFactor(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setSourceCompactionFactor(int sourceCompactionFactor) {
+    setSourceCompactionFactor(nativeHandle_, sourceCompactionFactor);
+    return this;
+  }
+
+  @Override
+  public int sourceCompactionFactor() {
+    return sourceCompactionFactor(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMaxGrandparentOverlapFactor(
+      int maxGrandparentOverlapFactor) {
+    setMaxGrandparentOverlapFactor(nativeHandle_, maxGrandparentOverlapFactor);
+    return this;
+  }
+
+  @Override
+  public int maxGrandparentOverlapFactor() {
+    return maxGrandparentOverlapFactor(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setSoftRateLimit(double softRateLimit) {
+    setSoftRateLimit(nativeHandle_, softRateLimit);
+    return this;
+  }
+
+  @Override
+  public double softRateLimit() {
+    return softRateLimit(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setHardRateLimit(double hardRateLimit) {
+    setHardRateLimit(nativeHandle_, hardRateLimit);
+    return this;
+  }
+
+  @Override
+  public double hardRateLimit() {
+    return hardRateLimit(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setRateLimitDelayMaxMilliseconds(
+      int rateLimitDelayMaxMilliseconds) {
+    setRateLimitDelayMaxMilliseconds(
+        nativeHandle_, rateLimitDelayMaxMilliseconds);
+    return this;
+  }
+
+  @Override
+  public int rateLimitDelayMaxMilliseconds() {
+    return rateLimitDelayMaxMilliseconds(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setArenaBlockSize(long arenaBlockSize)
+      throws RocksDBException {
+    setArenaBlockSize(nativeHandle_, arenaBlockSize);
+    return this;
+  }
+
+  @Override
+  public long arenaBlockSize() {
+    return arenaBlockSize(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setDisableAutoCompactions(boolean disableAutoCompactions) {
+    setDisableAutoCompactions(nativeHandle_, disableAutoCompactions);
+    return this;
+  }
+
+  @Override
+  public boolean disableAutoCompactions() {
+    return disableAutoCompactions(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setPurgeRedundantKvsWhileFlush(
+      boolean purgeRedundantKvsWhileFlush) {
+    setPurgeRedundantKvsWhileFlush(
+        nativeHandle_, purgeRedundantKvsWhileFlush);
+    return this;
+  }
+
+  @Override
+  public boolean purgeRedundantKvsWhileFlush() {
+    return purgeRedundantKvsWhileFlush(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setCompactionStyle(CompactionStyle compactionStyle) {
+    setCompactionStyle(nativeHandle_, compactionStyle.getValue());
+    return this;
+  }
+
+  @Override
+  public CompactionStyle compactionStyle() {
+    return CompactionStyle.values()[compactionStyle(nativeHandle_)];
+  }
+
+  @Override
+  public ColumnFamilyOptions setVerifyChecksumsInCompaction(
+      boolean verifyChecksumsInCompaction) {
+    setVerifyChecksumsInCompaction(
+        nativeHandle_, verifyChecksumsInCompaction);
+    return this;
+  }
+
+  @Override
+  public boolean verifyChecksumsInCompaction() {
+    return verifyChecksumsInCompaction(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setFilterDeletes(boolean filterDeletes) {
+    setFilterDeletes(nativeHandle_, filterDeletes);
+    return this;
+  }
+
+  @Override
+  public boolean filterDeletes() {
+    return filterDeletes(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMaxSequentialSkipInIterations(long maxSequentialSkipInIterations) {
+    setMaxSequentialSkipInIterations(nativeHandle_, maxSequentialSkipInIterations);
+    return this;
+  }
+
+  @Override
+  public long maxSequentialSkipInIterations() {
+    return maxSequentialSkipInIterations(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMemTableConfig(MemTableConfig config)
+      throws RocksDBException {
+    memTableConfig_ = config;
+    setMemTableFactory(nativeHandle_, config.newMemTableFactoryHandle());
+    return this;
+  }
+
+  @Override
+  public String memTableFactoryName() {
+    assert(isInitialized());
+    return memTableFactoryName(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setTableFormatConfig(TableFormatConfig config) {
+    tableFormatConfig_ = config;
+    setTableFactory(nativeHandle_, config.newTableFactoryHandle());
+    return this;
+  }
+
+  @Override
+  public String tableFactoryName() {
+    assert(isInitialized());
+    return tableFactoryName(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setInplaceUpdateSupport(boolean inplaceUpdateSupport) {
+    setInplaceUpdateSupport(nativeHandle_, inplaceUpdateSupport);
+    return this;
+  }
+
+  @Override
+  public boolean inplaceUpdateSupport() {
+    return inplaceUpdateSupport(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setInplaceUpdateNumLocks(long inplaceUpdateNumLocks)
+      throws RocksDBException {
+    setInplaceUpdateNumLocks(nativeHandle_, inplaceUpdateNumLocks);
+    return this;
+  }
+
+  @Override
+  public long inplaceUpdateNumLocks() {
+    return inplaceUpdateNumLocks(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMemtablePrefixBloomBits(int memtablePrefixBloomBits) {
+    setMemtablePrefixBloomBits(nativeHandle_, memtablePrefixBloomBits);
+    return this;
+  }
+
+  @Override
+  public int memtablePrefixBloomBits() {
+    return memtablePrefixBloomBits(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMemtablePrefixBloomProbes(int memtablePrefixBloomProbes) {
+    setMemtablePrefixBloomProbes(nativeHandle_, memtablePrefixBloomProbes);
+    return this;
+  }
+
+  @Override
+  public int memtablePrefixBloomProbes() {
+    return memtablePrefixBloomProbes(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setBloomLocality(int bloomLocality) {
+    setBloomLocality(nativeHandle_, bloomLocality);
+    return this;
+  }
+
+  @Override
+  public int bloomLocality() {
+    return bloomLocality(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMaxSuccessiveMerges(long maxSuccessiveMerges)
+      throws RocksDBException {
+    setMaxSuccessiveMerges(nativeHandle_, maxSuccessiveMerges);
+    return this;
+  }
+
+  @Override
+  public long maxSuccessiveMerges() {
+    return maxSuccessiveMerges(nativeHandle_);
+  }
+
+  @Override
+  public ColumnFamilyOptions setMinPartialMergeOperands(int minPartialMergeOperands) {
+    setMinPartialMergeOperands(nativeHandle_, minPartialMergeOperands);
+    return this;
+  }
+
+  @Override
+  public int minPartialMergeOperands() {
+    return minPartialMergeOperands(nativeHandle_);
+  }
+
+  /**
+   * Release the memory allocated for the current instance
+   * in the c++ side.
+   */
+  @Override protected void disposeInternal() {
+    assert(isInitialized());
+    disposeInternal(nativeHandle_);
+  }
+
+  private native void newColumnFamilyOptions();
+  private native void disposeInternal(long handle);
+
+  private native void optimizeForPointLookup(long handle,
+      long blockCacheSizeMb);
+  private native void optimizeLevelStyleCompaction(long handle,
+      long memtableMemoryBudget);
+  private native void optimizeUniversalStyleCompaction(long handle,
+      long memtableMemoryBudget);
+  private native void setComparatorHandle(long handle, int builtinComparator);
+  private native void setComparatorHandle(long optHandle, long comparatorHandle);
+  private native void setMergeOperatorName(
+      long handle, String name);
+  private native void setMergeOperator(
+      long handle, long mergeOperatorHandle);
+  private native void setWriteBufferSize(long handle, long writeBufferSize)
+      throws RocksDBException;
+  private native long writeBufferSize(long handle);
+  private native void setMaxWriteBufferNumber(
+      long handle, int maxWriteBufferNumber);
+  private native int maxWriteBufferNumber(long handle);
+  private native void setMinWriteBufferNumberToMerge(
+      long handle, int minWriteBufferNumberToMerge);
+  private native int minWriteBufferNumberToMerge(long handle);
+  private native void setCompressionType(long handle, byte compressionType);
+  private native byte compressionType(long handle);
+  private native void useFixedLengthPrefixExtractor(
+      long handle, int prefixLength);
+  private native void setNumLevels(
+      long handle, int numLevels);
+  private native int numLevels(long handle);
+  private native void setLevelZeroFileNumCompactionTrigger(
+      long handle, int numFiles);
+  private native int levelZeroFileNumCompactionTrigger(long handle);
+  private native void setLevelZeroSlowdownWritesTrigger(
+      long handle, int numFiles);
+  private native int levelZeroSlowdownWritesTrigger(long handle);
+  private native void setLevelZeroStopWritesTrigger(
+      long handle, int numFiles);
+  private native int levelZeroStopWritesTrigger(long handle);
+  private native void setMaxMemCompactionLevel(
+      long handle, int maxMemCompactionLevel);
+  private native int maxMemCompactionLevel(long handle);
+  private native void setTargetFileSizeBase(
+      long handle, long targetFileSizeBase);
+  private native long targetFileSizeBase(long handle);
+  private native void setTargetFileSizeMultiplier(
+      long handle, int multiplier);
+  private native int targetFileSizeMultiplier(long handle);
+  private native void setMaxBytesForLevelBase(
+      long handle, long maxBytesForLevelBase);
+  private native long maxBytesForLevelBase(long handle);
+  private native void setMaxBytesForLevelMultiplier(
+      long handle, int multiplier);
+  private native int maxBytesForLevelMultiplier(long handle);
+  private native void setExpandedCompactionFactor(
+      long handle, int expandedCompactionFactor);
+  private native int expandedCompactionFactor(long handle);
+  private native void setSourceCompactionFactor(
+      long handle, int sourceCompactionFactor);
+  private native int sourceCompactionFactor(long handle);
+  private native void setMaxGrandparentOverlapFactor(
+      long handle, int maxGrandparentOverlapFactor);
+  private native int maxGrandparentOverlapFactor(long handle);
+  private native void setSoftRateLimit(
+      long handle, double softRateLimit);
+  private native double softRateLimit(long handle);
+  private native void setHardRateLimit(
+      long handle, double hardRateLimit);
+  private native double hardRateLimit(long handle);
+  private native void setRateLimitDelayMaxMilliseconds(
+      long handle, int rateLimitDelayMaxMilliseconds);
+  private native int rateLimitDelayMaxMilliseconds(long handle);
+  private native void setArenaBlockSize(
+      long handle, long arenaBlockSize) throws RocksDBException;
+  private native long arenaBlockSize(long handle);
+  private native void setDisableAutoCompactions(
+      long handle, boolean disableAutoCompactions);
+  private native boolean disableAutoCompactions(long handle);
+  private native void setCompactionStyle(long handle, byte compactionStyle);
+  private native byte compactionStyle(long handle);
+  private native void setPurgeRedundantKvsWhileFlush(
+      long handle, boolean purgeRedundantKvsWhileFlush);
+  private native boolean purgeRedundantKvsWhileFlush(long handle);
+  private native void setVerifyChecksumsInCompaction(
+      long handle, boolean verifyChecksumsInCompaction);
+  private native boolean verifyChecksumsInCompaction(long handle);
+  private native void setFilterDeletes(
+      long handle, boolean filterDeletes);
+  private native boolean filterDeletes(long handle);
+  private native void setMaxSequentialSkipInIterations(
+      long handle, long maxSequentialSkipInIterations);
+  private native long maxSequentialSkipInIterations(long handle);
+  private native void setMemTableFactory(long handle, long factoryHandle);
+  private native String memTableFactoryName(long handle);
+  private native void setTableFactory(long handle, long factoryHandle);
+  private native String tableFactoryName(long handle);
+  private native void setInplaceUpdateSupport(
+      long handle, boolean inplaceUpdateSupport);
+  private native boolean inplaceUpdateSupport(long handle);
+  private native void setInplaceUpdateNumLocks(
+      long handle, long inplaceUpdateNumLocks) throws RocksDBException;
+  private native long inplaceUpdateNumLocks(long handle);
+  private native void setMemtablePrefixBloomBits(
+      long handle, int memtablePrefixBloomBits);
+  private native int memtablePrefixBloomBits(long handle);
+  private native void setMemtablePrefixBloomProbes(
+      long handle, int memtablePrefixBloomProbes);
+  private native int memtablePrefixBloomProbes(long handle);
+  private native void setBloomLocality(
+      long handle, int bloomLocality);
+  private native int bloomLocality(long handle);
+  private native void setMaxSuccessiveMerges(
+      long handle, long maxSuccessiveMerges) throws RocksDBException;
+  private native long maxSuccessiveMerges(long handle);
+  private native void setMinPartialMergeOperands(
+      long handle, int minPartialMergeOperands);
+  private native int minPartialMergeOperands(long handle);
+
+  MemTableConfig memTableConfig_;
+  TableFormatConfig tableFormatConfig_;
+  AbstractComparator comparator_;
+}

java/org/rocksdb/ColumnFamilyOptionsInterface.java

@@ -152,7 +152,8 @@ public interface ColumnFamilyOptionsInterface {
    * Default: 4MB
    * @param writeBufferSize the size of write buffer.
    * @return the instance of the current Object.
-   * @throws org.rocksdb.RocksDBException
+   * @throws org.rocksdb.RocksDBException thrown on 32-Bit platforms while
+   *    overflowing the underlying platform specific value.
    */
   Object setWriteBufferSize(long writeBufferSize)
       throws RocksDBException;
@@ -223,6 +224,7 @@ public interface ColumnFamilyOptionsInterface {
    * extract the prefix given a key.
    *
    * @param n use the first n bytes of a key as its prefix.
+   * @return the reference to the current option.
    */
   Object useFixedLengthPrefixExtractor(int n);
 
@@ -415,6 +417,8 @@ public interface ColumnFamilyOptionsInterface {
    * and total file size for level-3 will be 2GB.
    * by default 'maxBytesForLevelBase' is 10MB.
    *
+   * @param maxBytesForLevelBase maximum bytes for level base.
+   *
    * @return the reference to the current option.
    * @see #setMaxBytesForLevelMultiplier(int)
    */
@@ -614,7 +618,8 @@ public interface ColumnFamilyOptionsInterface {
    *
    * @param arenaBlockSize the size of an arena block
    * @return the reference to the current option.
-   * @throws org.rocksdb.RocksDBException
+   * @throws org.rocksdb.RocksDBException thrown on 32-Bit platforms while
+   *    overflowing the underlying platform specific value.
    */
   Object setArenaBlockSize(long arenaBlockSize)
       throws RocksDBException;
@@ -762,7 +767,8 @@ public interface ColumnFamilyOptionsInterface {
    *
    * @param config the mem-table config.
    * @return the instance of the current Object.
-   * @throws org.rocksdb.RocksDBException
+   * @throws org.rocksdb.RocksDBException thrown on 32-Bit platforms while
+   *    overflowing the underlying platform specific value.
    */
   Object setMemTableConfig(MemTableConfig config)
       throws RocksDBException;
@@ -826,7 +832,8 @@ public interface ColumnFamilyOptionsInterface {
    * @param inplaceUpdateNumLocks the number of locks used for
    *     inplace updates.
    * @return the reference to the current option.
-   * @throws org.rocksdb.RocksDBException
+   * @throws org.rocksdb.RocksDBException thrown on 32-Bit platforms while
+   *    overflowing the underlying platform specific value.
    */
   Object setInplaceUpdateNumLocks(long inplaceUpdateNumLocks)
       throws RocksDBException;
@@ -920,7 +927,8 @@ public interface ColumnFamilyOptionsInterface {
    *
    * @param maxSuccessiveMerges the maximum number of successive merges.
    * @return the reference to the current option.
-   * @throws org.rocksdb.RocksDBException
+   * @throws org.rocksdb.RocksDBException thrown on 32-Bit platforms while
+   *    overflowing the underlying platform specific value.
    */
   Object setMaxSuccessiveMerges(long maxSuccessiveMerges)
       throws RocksDBException;

java/org/rocksdb/DBOptionsInterface.java

@@ -123,7 +123,7 @@ public interface DBOptionsInterface {
    * Default: 5000
    *
    * @param maxOpenFiles the maximum number of open files.
-   * @return the reference to the current DBOptions.
+   * @return the instance of the current Object.
    */
   Object setMaxOpenFiles(int maxOpenFiles);
 
@@ -147,6 +147,9 @@ public interface DBOptionsInterface {
    * <p>If set to 0 (default), we will dynamically choose the WAL size limit to
    * be [sum of all write_buffer_size * max_write_buffer_number] * 2</p>
    * <p>Default: 0</p>
+   *
+   * @param maxTotalWalSize max total wal size.
+   * @return the instance of the current Object.
    */
   Object setMaxTotalWalSize(long maxTotalWalSize);
 
@@ -197,7 +200,7 @@ public interface DBOptionsInterface {
    *
    * @param disableDataSync a boolean flag to specify whether to
    *     disable data sync.
-   * @return the reference to the current DBOptions.
+   * @return the instance of the current Object.
    */
   Object setDisableDataSync(boolean disableDataSync);
 
@@ -370,7 +373,8 @@ public interface DBOptionsInterface {
    *
    * @param maxLogFileSize the maximum size of a info log file.
    * @return the instance of the current Object.
-   * @throws org.rocksdb.RocksDBException
+   * @throws org.rocksdb.RocksDBException thrown on 32-Bit platforms while
+   *    overflowing the underlying platform specific value.
    */
   Object setMaxLogFileSize(long maxLogFileSize)
       throws RocksDBException;
@@ -392,7 +396,8 @@ public interface DBOptionsInterface {
    *
    * @param logFileTimeToRoll the time interval in seconds.
    * @return the instance of the current Object.
-   * @throws org.rocksdb.RocksDBException
+   * @throws org.rocksdb.RocksDBException thrown on 32-Bit platforms while
+   *    overflowing the underlying platform specific value.
    */
   Object setLogFileTimeToRoll(long logFileTimeToRoll)
       throws RocksDBException;
@@ -413,7 +418,8 @@ public interface DBOptionsInterface {
    *
    * @param keepLogFileNum the maximum number of info log files to be kept.
    * @return the instance of the current Object.
-   * @throws org.rocksdb.RocksDBException
+   * @throws org.rocksdb.RocksDBException thrown on 32-Bit platforms while
+   *    overflowing the underlying platform specific value.
    */
   Object setKeepLogFileNum(long keepLogFileNum)
       throws RocksDBException;
@@ -584,7 +590,8 @@ public interface DBOptionsInterface {
    *
    * @param size the size in byte
    * @return the instance of the current Object.
-   * @throws org.rocksdb.RocksDBException
+   * @throws org.rocksdb.RocksDBException thrown on 32-Bit platforms while
+   *    overflowing the underlying platform specific value.
    */
   Object setManifestPreallocationSize(long size)
       throws RocksDBException;

java/org/rocksdb/FlushOptions.java

@@ -0,0 +1,51 @@
+package org.rocksdb;
+
+/**
+ * FlushOptions to be passed to flush operations of
+ * {@link org.rocksdb.RocksDB}.
+ */
+public class FlushOptions extends RocksObject {
+
+  /**
+   * Construct a new instance of FlushOptions.
+   */
+  public FlushOptions(){
+    super();
+    newFlushOptions();
+  }
+
+  /**
+   * Set if the flush operation shall block until it terminates.
+   *
+   * @param waitForFlush boolean value indicating if the flush
+   *     operations waits for termination of the flush process.
+   *
+   * @return instance of current FlushOptions.
+   */
+  public FlushOptions setWaitForFlush(boolean waitForFlush) {
+    assert(isInitialized());
+    waitForFlush(nativeHandle_);
+    return this;
+  }
+
+  /**
+   * Wait for flush to finished.
+   *
+   * @return boolean value indicating if the flush operation
+   *     waits for termination of the flush process.
+   */
+  public boolean waitForFlush() {
+    assert(isInitialized());
+    return waitForFlush(nativeHandle_);
+  }
+
+  @Override protected void disposeInternal() {
+    disposeInternal(nativeHandle_);
+  }
+
+  private native void newFlushOptions();
+  private native void disposeInternal(long handle);
+  private native void setWaitForFlush(long handle,
+      boolean wait);
+  private native boolean waitForFlush(long handle);
+}

java/org/rocksdb/HashSkipListMemTableConfig.java

@@ -50,6 +50,8 @@ public class HashSkipListMemTableConfig extends MemTableConfig {
   /**
    * Set the height of the skip list.  Default = 4.
    *
+   * @param height height to set.
+   *
    * @return the reference to the current HashSkipListMemTableConfig.
    */
   public HashSkipListMemTableConfig setHeight(int height) {

java/org/rocksdb/MemTableConfig.java

@@ -22,6 +22,11 @@ public abstract class MemTableConfig {
    * that associated with the Java MemTableConfig.
    *
    * @see Options#setMemTableConfig(MemTableConfig)
+   *
+   * @return native handle address to native memory table instance.
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   abstract protected long newMemTableFactoryHandle()
       throws RocksDBException;

java/org/rocksdb/MergeOperator.java

@@ -5,8 +5,6 @@
 
 package org.rocksdb;
 
-import java.util.*;
-
 /**
  * MergeOperator holds an operator to be applied when compacting
  * two merge operands held under the same key in order to obtain a single

java/org/rocksdb/NativeLibraryLoader.java

@@ -9,12 +9,12 @@ import org.rocksdb.util.Environment;
  */
 public class NativeLibraryLoader {
   private static String sharedLibraryName = Environment.getJniLibraryName("rocksdb");
-  private static String tempFilePrefix = "librocksdbjni";
   private static String tempFileSuffix = "." + Environment.getJniLibraryExtension();
 
   public static void loadLibraryFromJar(String tmpDir)
       throws IOException {
     File temp;
+    String tempFilePrefix = "librocksdbjni";
     if(tmpDir == null || tmpDir.equals(""))
       temp = File.createTempFile(tempFilePrefix, tempFileSuffix);
     else
@@ -43,9 +43,7 @@ public class NativeLibraryLoader {
     } finally {
       if(os != null)
         os.close();
-
-      if(is != null)
-        is.close();
+      is.close();
     }
 
     System.load(temp.getAbsolutePath());

java/org/rocksdb/Options.java

@@ -29,6 +29,20 @@ public class Options extends RocksObject
     env_ = RocksEnv.getDefault();
   }
 
+  /**
+   * Construct options for opening a RocksDB. Reusing database options
+   * and column family options.
+   *
+   * @param dbOptions {@link org.rocksdb.DBOptions} instance
+   * @param columnFamilyOptions {@link org.rocksdb.ColumnFamilyOptions}
+   *     instance
+   */
+  public Options(DBOptions dbOptions, ColumnFamilyOptions columnFamilyOptions) {
+    super();
+    newOptions(dbOptions.nativeHandle_, columnFamilyOptions.nativeHandle_);
+    env_ = RocksEnv.getDefault();
+  }
+
   @Override
   public Options setCreateIfMissing(boolean flag) {
     assert(isInitialized());
@@ -47,6 +61,9 @@ public class Options extends RocksObject
    * Use the specified object to interact with the environment,
    * e.g. to read/write files, schedule background work, etc.
    * Default: {@link RocksEnv#getDefault()}
+   *
+   * @param env {@link RocksEnv} instance.
+   * @return the instance of the current Options.
    */
   public Options setEnv(RocksEnv env) {
     assert(isInitialized());
@@ -55,6 +72,11 @@ public class Options extends RocksObject
     return this;
   }
 
+  /**
+   * Returns the set RocksEnv instance.
+   *
+   * @return {@link RocksEnv} instance set in the Options.
+   */
   public RocksEnv getEnv() {
     return env_;
   }

java/org/rocksdb/PlainTableConfig.java

@@ -131,7 +131,7 @@ public class PlainTableConfig extends TableFormatConfig {
    *
    * <p>See linux doc Documentation/vm/hugetlbpage.txt</p>
    *
-   * @param hugePageTlbSize
+   * @param hugePageTlbSize huge page tlb size
    * @return the reference to the current config.
    */
   public PlainTableConfig setHugePageTlbSize(int hugePageTlbSize) {

java/org/rocksdb/RateLimiterConfig.java

@@ -16,6 +16,8 @@ public abstract class RateLimiterConfig {
    * with a Java RateLimiterConfig.
    *
    * @see org.rocksdb.DBOptions#setRateLimiter(long, long)
+   *
+   * @return native handle address to rate limiter instance.
    */
   abstract protected long newRateLimiterHandle();
 }

java/org/rocksdb/RestoreBackupableDB.java

@@ -40,8 +40,10 @@ public class RestoreBackupableDB extends RocksObject {
    * @param backupId id pointing to backup
    * @param dbDir database directory to restore to
    * @param walDir directory where wal files are located
-   * @param restoreOptions {@link org.rocksdb.RestoreOptions} instance
-   * @throws RocksDBException
+   * @param restoreOptions {@link org.rocksdb.RestoreOptions} instance.
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void restoreDBFromBackup(long backupId, String dbDir, String walDir,
       RestoreOptions restoreOptions) throws RocksDBException {
@@ -55,7 +57,9 @@ public class RestoreBackupableDB extends RocksObject {
    * @param dbDir database directory to restore to
    * @param walDir directory where wal files are located
    * @param restoreOptions {@link org.rocksdb.RestoreOptions} instance
-   * @throws RocksDBException
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void restoreDBFromLatestBackup(String dbDir, String walDir,
       RestoreOptions restoreOptions) throws RocksDBException {
@@ -67,7 +71,9 @@ public class RestoreBackupableDB extends RocksObject {
    * Deletes old backups, keeping latest numBackupsToKeep alive.
    *
    * @param numBackupsToKeep of latest backups to keep
-   * @throws org.rocksdb.RocksDBException
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void purgeOldBackups(int numBackupsToKeep) throws RocksDBException {
     purgeOldBackups0(nativeHandle_, numBackupsToKeep);
@@ -77,7 +83,9 @@ public class RestoreBackupableDB extends RocksObject {
    * Deletes a specific backup.
    *
    * @param backupId of backup to delete.
-   * @throws org.rocksdb.RocksDBException
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void deleteBackup(int backupId) throws RocksDBException {
     deleteBackup0(nativeHandle_, backupId);

java/org/rocksdb/RocksDB.java

@@ -16,6 +16,7 @@ import org.rocksdb.util.Environment;
  * indicates sth wrong at the RocksDB library side and the call failed.
  */
 public class RocksDB extends RocksObject {
+  public static final String DEFAULT_COLUMN_FAMILY = "default";
   public static final int NOT_FOUND = -1;
   private static final String[] compressionLibs_ = {
       "snappy", "z", "bzip2", "lz4", "lz4hc"};
@@ -94,7 +95,8 @@ public class RocksDB extends RocksObject {
    * @return a {@link RocksDB} instance on success, null if the specified
    *     {@link RocksDB} can not be opened.
    *
-   * @throws org.rocksdb.RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    * @see Options#setCreateIfMissing(boolean)
    */
   public static RocksDB open(String path) throws RocksDBException {
@@ -123,21 +125,23 @@ public class RocksDB extends RocksObject {
    * </p>
    *
    * @param path the path to the rocksdb.
-   * @param columnFamilyNames list of column family names
+   * @param columnFamilyDescriptors list of column family descriptors
    * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances
    *     on open.
    * @return a {@link RocksDB} instance on success, null if the specified
    *     {@link RocksDB} can not be opened.
    *
-   * @throws org.rocksdb.RocksDBException
-   * @see Options#setCreateIfMissing(boolean)
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
+   * @see DBOptions#setCreateIfMissing(boolean)
    */
-  public static RocksDB open(String path, List<String> columnFamilyNames,
+  public static RocksDB open(String path,
+      List<ColumnFamilyDescriptor> columnFamilyDescriptors,
       List<ColumnFamilyHandle> columnFamilyHandles) throws RocksDBException {
     // This allows to use the rocksjni default Options instead of
     // the c++ one.
-    Options options = new Options();
-    return open(options, path, columnFamilyNames, columnFamilyHandles);
+    DBOptions options = new DBOptions();
+    return open(options, path, columnFamilyDescriptors, columnFamilyHandles);
   }
 
   /**
@@ -159,7 +163,9 @@ public class RocksDB extends RocksObject {
    * @return a {@link RocksDB} instance on success, null if the specified
    *     {@link RocksDB} can not be opened.
    *
-   * @throws org.rocksdb.RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
+   *
    * @see Options#setCreateIfMissing(boolean)
    */
   public static RocksDB open(Options options, String path)
@@ -196,24 +202,27 @@ public class RocksDB extends RocksObject {
    * <p>
    * ColumnFamily handles are disposed when the RocksDB instance is disposed.</p>
    *
-   * @param options {@link org.rocksdb.Options} instance.
+   * @param options {@link org.rocksdb.DBOptions} instance.
    * @param path the path to the rocksdb.
-   * @param columnFamilyNames list of column family names
+   * @param columnFamilyDescriptors list of column family descriptors
    * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances
    *     on open.
    * @return a {@link RocksDB} instance on success, null if the specified
    *     {@link RocksDB} can not be opened.
    *
-   * @throws org.rocksdb.RocksDBException
-   * @see Options#setCreateIfMissing(boolean)
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
+   *
+   * @see DBOptions#setCreateIfMissing(boolean)
    */
-  public static RocksDB open(Options options, String path, List<String> columnFamilyNames,
+  public static RocksDB open(DBOptions options, String path,
+      List<ColumnFamilyDescriptor> columnFamilyDescriptors,
       List<ColumnFamilyHandle> columnFamilyHandles)
       throws RocksDBException {
     RocksDB db = new RocksDB();
     List<Long> cfReferences = db.open(options.nativeHandle_, path,
-        columnFamilyNames, columnFamilyNames.size());
-    for (int i=0; i<columnFamilyNames.size(); i++) {
+        columnFamilyDescriptors, columnFamilyDescriptors.size());
+    for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
       columnFamilyHandles.add(new ColumnFamilyHandle(db, cfReferences.get(i)));
     }
     db.storeOptionsInstance(options);
@@ -228,7 +237,9 @@ public class RocksDB extends RocksObject {
    * @param path the path to the RocksDB.
    * @return a {@link RocksDB} instance on success, null if the specified
    *     {@link RocksDB} can not be opened.
-   * @throws RocksDBException
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public static RocksDB openReadOnly(String path)
       throws RocksDBException {
@@ -244,19 +255,23 @@ public class RocksDB extends RocksObject {
    * options.
    *
    * @param path the path to the RocksDB.
-   * @param columnFamilyNames list of column family names
+   * @param columnFamilyDescriptors list of column family descriptors
    * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances
    *     on open.
    * @return a {@link RocksDB} instance on success, null if the specified
    *     {@link RocksDB} can not be opened.
-   * @throws RocksDBException
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
-  public static RocksDB openReadOnly(String path, List<String> columnFamilyNames,
+  public static RocksDB openReadOnly(String path,
+      List<ColumnFamilyDescriptor> columnFamilyDescriptors,
       List<ColumnFamilyHandle> columnFamilyHandles) throws RocksDBException {
     // This allows to use the rocksjni default Options instead of
     // the c++ one.
-    Options options = new Options();
-    return openReadOnly(options, path, columnFamilyNames, columnFamilyHandles);
+    DBOptions options = new DBOptions();
+    return openReadOnly(options, path, columnFamilyDescriptors,
+        columnFamilyHandles);
   }
 
   /**
@@ -272,7 +287,9 @@ public class RocksDB extends RocksObject {
    * @param path the path to the RocksDB.
    * @return a {@link RocksDB} instance on success, null if the specified
    *     {@link RocksDB} can not be opened.
-   * @throws RocksDBException
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public static RocksDB openReadOnly(Options options, String path)
       throws RocksDBException {
@@ -297,25 +314,28 @@ public class RocksDB extends RocksObject {
    * options instance have been closed. If user doesn't call options dispose
    * explicitly,then this options instance will be GC'd automatically.</p>
    *
-   * @param options {@link Options} instance.
+   * @param options {@link DBOptions} instance.
    * @param path the path to the RocksDB.
-   * @param columnFamilyNames list of column family names
+   * @param columnFamilyDescriptors list of column family descriptors
    * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances
    *     on open.
    * @return a {@link RocksDB} instance on success, null if the specified
    *     {@link RocksDB} can not be opened.
-   * @throws RocksDBException
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
-  public static RocksDB openReadOnly(Options options, String path,
-      List<String> columnFamilyNames, List<ColumnFamilyHandle> columnFamilyHandles)
+  public static RocksDB openReadOnly(DBOptions options, String path,
+      List<ColumnFamilyDescriptor> columnFamilyDescriptors,
+      List<ColumnFamilyHandle> columnFamilyHandles)
       throws RocksDBException {
     // when non-default Options is used, keeping an Options reference
     // in RocksDB can prevent Java to GC during the life-time of
     // the currently-created RocksDB.
     RocksDB db = new RocksDB();
     List<Long> cfReferences = db.openROnly(options.nativeHandle_, path,
-        columnFamilyNames, columnFamilyNames.size());
-    for (int i=0; i<columnFamilyNames.size(); i++) {
+        columnFamilyDescriptors, columnFamilyDescriptors.size());
+    for (int i=0; i<columnFamilyDescriptors.size(); i++) {
       columnFamilyHandles.add(new ColumnFamilyHandle(db, cfReferences.get(i)));
     }
 
@@ -330,14 +350,15 @@ public class RocksDB extends RocksObject {
    * @param path Absolute path to rocksdb database
    * @return List&lt;byte[]&gt; List containing the column family names
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public static List<byte[]> listColumnFamilies(Options options, String path)
       throws RocksDBException {
     return RocksDB.listColumnFamilies(options.nativeHandle_, path);
   }
 
-  private void storeOptionsInstance(Options options) {
+  private void storeOptionsInstance(DBOptionsInterface options) {
     options_ = options;
   }
 
@@ -360,7 +381,8 @@ public class RocksDB extends RocksObject {
    * @param key the specified key to be inserted.
    * @param value the value associated with the specified key.
    *
-   * @see RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void put(byte[] key, byte[] value) throws RocksDBException {
     put(nativeHandle_, key, key.length, value, value.length);
@@ -377,7 +399,8 @@ public class RocksDB extends RocksObject {
    *
    * throws IllegalArgumentException if column family is not present
    *
-   * @see RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void put(ColumnFamilyHandle columnFamilyHandle, byte[] key,
       byte[] value) throws RocksDBException {
@@ -388,10 +411,12 @@ public class RocksDB extends RocksObject {
   /**
    * Set the database entry for "key" to "value".
    *
+   * @param writeOpts {@link org.rocksdb.WriteOptions} instance.
    * @param key the specified key to be inserted.
    * @param value the value associated with the specified key.
    *
-   * @see RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void put(WriteOptions writeOpts, byte[] key, byte[] value)
       throws RocksDBException {
@@ -405,12 +430,14 @@ public class RocksDB extends RocksObject {
    *
    * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle}
    *     instance
+   * @param writeOpts {@link org.rocksdb.WriteOptions} instance.
    * @param key the specified key to be inserted.
    * @param value the value associated with the specified key.
    *
    * throws IllegalArgumentException if column family is not present
    *
-   * @see RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    * @see IllegalArgumentException
    */
   public void put(ColumnFamilyHandle columnFamilyHandle, WriteOptions writeOpts,
@@ -500,7 +527,8 @@ public class RocksDB extends RocksObject {
    * @param writeOpts WriteOptions instance
    * @param updates WriteBatch instance
    *
-   * @see RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void write(WriteOptions writeOpts, WriteBatch updates)
       throws RocksDBException {
@@ -511,8 +539,11 @@ public class RocksDB extends RocksObject {
    * Add merge operand for key/value pair.
    *
    * @param key the specified key to be merged.
-   * @param value the value to be nerged with the current value for
+   * @param value the value to be merged with the current value for
    * the specified key.
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void merge(byte[] key, byte[] value) throws RocksDBException {
     merge(nativeHandle_, key, key.length, value, value.length);
@@ -523,8 +554,11 @@ public class RocksDB extends RocksObject {
    *
    * @param columnFamilyHandle {@link ColumnFamilyHandle} instance
    * @param key the specified key to be merged.
-   * @param value the value to be nerged with the current value for
+   * @param value the value to be merged with the current value for
    * the specified key.
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void merge(ColumnFamilyHandle columnFamilyHandle, byte[] key,
       byte[] value) throws RocksDBException {
@@ -539,6 +573,9 @@ public class RocksDB extends RocksObject {
    * @param key the specified key to be merged.
    * @param value the value to be merged with the current value for
    * the specified key.
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void merge(WriteOptions writeOpts, byte[] key, byte[] value)
       throws RocksDBException {
@@ -554,6 +591,9 @@ public class RocksDB extends RocksObject {
    * @param key the specified key to be merged.
    * @param value the value to be merged with the current value for
    * the specified key.
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void merge(ColumnFamilyHandle columnFamilyHandle,
       WriteOptions writeOpts, byte[] key, byte[] value)
@@ -574,7 +614,8 @@ public class RocksDB extends RocksObject {
    *     be returned.  RocksDB.NOT_FOUND will be returned if the value not
    *     found.
    *
-   * @see RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public int get(byte[] key, byte[] value) throws RocksDBException {
     return get(nativeHandle_, key, key.length, value, value.length);
@@ -594,7 +635,8 @@ public class RocksDB extends RocksObject {
    *     be returned.  RocksDB.NOT_FOUND will be returned if the value not
    *     found.
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public int get(ColumnFamilyHandle columnFamilyHandle, byte[] key, byte[] value)
       throws RocksDBException, IllegalArgumentException {
@@ -605,6 +647,7 @@ public class RocksDB extends RocksObject {
   /**
    * Get the value associated with the specified key.
    *
+   * @param opt {@link org.rocksdb.ReadOptions} instance.
    * @param key the key to retrieve the value.
    * @param value the out-value to receive the retrieved value.
    * @return The size of the actual value that matches the specified
@@ -614,7 +657,8 @@ public class RocksDB extends RocksObject {
    *     be returned.  RocksDB.NOT_FOUND will be returned if the value not
    *     found.
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public int get(ReadOptions opt, byte[] key, byte[] value)
       throws RocksDBException {
@@ -626,6 +670,7 @@ public class RocksDB extends RocksObject {
    *
    * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle}
    *     instance
+   * @param opt {@link org.rocksdb.ReadOptions} instance.
    * @param key the key to retrieve the value.
    * @param value the out-value to receive the retrieved value.
    * @return The size of the actual value that matches the specified
@@ -635,7 +680,8 @@ public class RocksDB extends RocksObject {
    *     be returned.  RocksDB.NOT_FOUND will be returned if the value not
    *     found.
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public int get(ColumnFamilyHandle columnFamilyHandle, ReadOptions opt, byte[] key,
       byte[] value) throws RocksDBException {
@@ -652,7 +698,8 @@ public class RocksDB extends RocksObject {
    * @return a byte array storing the value associated with the input key if
    *     any.  null if it does not find the specified key.
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public byte[] get(byte[] key) throws RocksDBException {
     return get(nativeHandle_, key, key.length);
@@ -669,7 +716,8 @@ public class RocksDB extends RocksObject {
    * @return a byte array storing the value associated with the input key if
    *     any.  null if it does not find the specified key.
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public byte[] get(ColumnFamilyHandle columnFamilyHandle, byte[] key)
       throws RocksDBException {
@@ -686,7 +734,8 @@ public class RocksDB extends RocksObject {
    * @return a byte array storing the value associated with the input key if
    *     any.  null if it does not find the specified key.
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public byte[] get(ReadOptions opt, byte[] key) throws RocksDBException {
     return get(nativeHandle_, opt.nativeHandle_, key, key.length);
@@ -704,7 +753,8 @@ public class RocksDB extends RocksObject {
    * @return a byte array storing the value associated with the input key if
    *     any.  null if it does not find the specified key.
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public byte[] get(ColumnFamilyHandle columnFamilyHandle, ReadOptions opt,
       byte[] key) throws RocksDBException {
@@ -719,7 +769,8 @@ public class RocksDB extends RocksObject {
    * @return Map where key of map is the key passed by user and value for map
    * entry is the corresponding value in DB.
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public Map<byte[], byte[]> multiGet(List<byte[]> keys)
       throws RocksDBException {
@@ -728,7 +779,7 @@ public class RocksDB extends RocksObject {
     List<byte[]> values = multiGet(
         nativeHandle_, keys, keys.size());
 
-    Map<byte[], byte[]> keyValueMap = new HashMap<byte[], byte[]>();
+    Map<byte[], byte[]> keyValueMap = new HashMap<>();
     for(int i = 0; i < values.size(); i++) {
       if(values.get(i) == null) {
         continue;
@@ -753,8 +804,10 @@ public class RocksDB extends RocksObject {
    * @return Map where key of map is the key passed by user and value for map
    * entry is the corresponding value in DB.
    *
-   * @throws RocksDBException
-   * @throws IllegalArgumentException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
+   * @throws IllegalArgumentException thrown if the size of passed keys is not
+   *    equal to the amount of passed column family handles.
    */
   public Map<byte[], byte[]> multiGet(List<ColumnFamilyHandle> columnFamilyHandleList,
       List<byte[]> keys) throws RocksDBException, IllegalArgumentException {
@@ -768,7 +821,7 @@ public class RocksDB extends RocksObject {
     List<byte[]> values = multiGet(nativeHandle_, keys, keys.size(),
         columnFamilyHandleList);
 
-    Map<byte[], byte[]> keyValueMap = new HashMap<byte[], byte[]>();
+    Map<byte[], byte[]> keyValueMap = new HashMap<>();
     for(int i = 0; i < values.size(); i++) {
       if (values.get(i) == null) {
         continue;
@@ -786,7 +839,8 @@ public class RocksDB extends RocksObject {
    * @return Map where key of map is the key passed by user and value for map
    * entry is the corresponding value in DB.
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public Map<byte[], byte[]> multiGet(ReadOptions opt, List<byte[]> keys)
       throws RocksDBException {
@@ -795,7 +849,7 @@ public class RocksDB extends RocksObject {
     List<byte[]> values = multiGet(
         nativeHandle_, opt.nativeHandle_, keys, keys.size());
 
-    Map<byte[], byte[]> keyValueMap = new HashMap<byte[], byte[]>();
+    Map<byte[], byte[]> keyValueMap = new HashMap<>();
     for(int i = 0; i < values.size(); i++) {
       if(values.get(i) == null) {
         continue;
@@ -821,8 +875,10 @@ public class RocksDB extends RocksObject {
    * @return Map where key of map is the key passed by user and value for map
    * entry is the corresponding value in DB.
    *
-   * @throws RocksDBException
-   * @throws java.lang.IllegalArgumentException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
+   * @throws IllegalArgumentException thrown if the size of passed keys is not
+   *    equal to the amount of passed column family handles.
    */
   public Map<byte[], byte[]> multiGet(ReadOptions opt,
       List<ColumnFamilyHandle> columnFamilyHandleList, List<byte[]> keys)
@@ -838,7 +894,7 @@ public class RocksDB extends RocksObject {
     List<byte[]> values = multiGet(nativeHandle_, opt.nativeHandle_,
         keys, keys.size(), columnFamilyHandleList);
 
-    Map<byte[], byte[]> keyValueMap = new HashMap<byte[], byte[]>();
+    Map<byte[], byte[]> keyValueMap = new HashMap<>();
     for(int i = 0; i < values.size(); i++) {
       if(values.get(i) == null) {
         continue;
@@ -856,7 +912,8 @@ public class RocksDB extends RocksObject {
    *
    * @param key Key to delete within database
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void remove(byte[] key) throws RocksDBException {
     remove(nativeHandle_, key, key.length);
@@ -871,7 +928,8 @@ public class RocksDB extends RocksObject {
    *     instance
    * @param key Key to delete within database
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void remove(ColumnFamilyHandle columnFamilyHandle, byte[] key)
       throws RocksDBException {
@@ -886,7 +944,8 @@ public class RocksDB extends RocksObject {
    * @param writeOpt WriteOptions to be used with delete operation
    * @param key Key to delete within database
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void remove(WriteOptions writeOpt, byte[] key)
       throws RocksDBException {
@@ -903,7 +962,8 @@ public class RocksDB extends RocksObject {
    * @param writeOpt WriteOptions to be used with delete operation
    * @param key Key to delete within database
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void remove(ColumnFamilyHandle columnFamilyHandle, WriteOptions writeOpt,
       byte[] key) throws RocksDBException {
@@ -934,7 +994,8 @@ public class RocksDB extends RocksObject {
    * @param property to be fetched. See above for examples
    * @return property value
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public String getProperty(ColumnFamilyHandle columnFamilyHandle, String property)
       throws RocksDBException {
@@ -961,7 +1022,8 @@ public class RocksDB extends RocksObject {
    * @param property to be fetched. See above for examples
    * @return property value
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public String getProperty(String property) throws RocksDBException {
     return getProperty0(nativeHandle_, property, property.length());
@@ -991,7 +1053,7 @@ public class RocksDB extends RocksObject {
    * <p>nullptr will be returned if the DB fails to take a snapshot or does
    * not support snapshot.</p>
    *
-   * @return Snapshot
+   * @return Snapshot {@link Snapshot} instance
    */
   public Snapshot getSnapshot() {
     long snapshotHandle = getSnapshot(nativeHandle_);
@@ -1040,12 +1102,13 @@ public class RocksDB extends RocksObject {
    * @return {@link java.util.List} containing {@link org.rocksdb.RocksIterator}
    *     instances
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public List<RocksIterator> newIterators(
       List<ColumnFamilyHandle> columnFamilyHandleList) throws RocksDBException {
     List<RocksIterator> iterators =
-        new ArrayList<RocksIterator>(columnFamilyHandleList.size());
+        new ArrayList<>(columnFamilyHandleList.size());
 
     long[] iteratorRefs = iterators(nativeHandle_, columnFamilyHandleList);
     for (int i=0; i<columnFamilyHandleList.size(); i++){
@@ -1059,14 +1122,17 @@ public class RocksDB extends RocksObject {
    * allocates a ColumnFamilyHandle within an internal structure.
    * The ColumnFamilyHandle is automatically disposed with DB disposal.
    *
-   * @param columnFamilyName Name of column family to be created.
-   * @return {@link org.rocksdb.ColumnFamilyHandle} instance
-   * @see RocksDBException
+   * @param columnFamilyDescriptor column family to be created.
+   * @return {@link org.rocksdb.ColumnFamilyHandle} instance.
+   *
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
-  public ColumnFamilyHandle createColumnFamily(String columnFamilyName)
+  public ColumnFamilyHandle createColumnFamily(
+      ColumnFamilyDescriptor columnFamilyDescriptor)
       throws RocksDBException {
     return new ColumnFamilyHandle(this, createColumnFamily(nativeHandle_,
-        options_.nativeHandle_, columnFamilyName));
+        columnFamilyDescriptor));
   }
 
   /**
@@ -1077,7 +1143,8 @@ public class RocksDB extends RocksObject {
    * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle}
    *     instance
    *
-   * @throws RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void dropColumnFamily(ColumnFamilyHandle columnFamilyHandle)
       throws RocksDBException, IllegalArgumentException {
@@ -1085,6 +1152,40 @@ public class RocksDB extends RocksObject {
     dropColumnFamily(nativeHandle_, columnFamilyHandle.nativeHandle_);
   }
 
+  /**
+   * <p>Flush all memory table data.</p>
+   *
+   * <p>Note: it must be ensured that the FlushOptions instance
+   * is not GC'ed before this method finishes. If the wait parameter is
+   * set to false, flush processing is asynchronous.</p>
+   *
+   * @param flushOptions {@link org.rocksdb.FlushOptions} instance.
+   * @throws RocksDBException thrown if an error occurs within the native
+   *     part of the library.
+   */
+  public void flush(FlushOptions flushOptions)
+      throws RocksDBException {
+    flush(nativeHandle_, flushOptions.nativeHandle_);
+  }
+
+  /**
+   * <p>Flush all memory table data.</p>
+   *
+   * <p>Note: it must be ensured that the FlushOptions instance
+   * is not GC'ed before this method finishes. If the wait parameter is
+   * set to false, flush processing is asynchronous.</p>
+   *
+   * @param flushOptions {@link org.rocksdb.FlushOptions} instance.
+   * @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle} instance.
+   * @throws RocksDBException thrown if an error occurs within the native
+   *     part of the library.
+   */
+  public void flush(FlushOptions flushOptions,
+      ColumnFamilyHandle columnFamilyHandle) throws RocksDBException {
+    flush(nativeHandle_, flushOptions.nativeHandle_,
+        columnFamilyHandle.nativeHandle_);
+  }
+
   /**
    * Private constructor.
    */
@@ -1096,15 +1197,17 @@ public class RocksDB extends RocksObject {
   protected native void open(
       long optionsHandle, String path) throws RocksDBException;
   protected native List<Long> open(long optionsHandle, String path,
-      List<String> columnFamilyNames, int columnFamilyNamesLength)
+      List<ColumnFamilyDescriptor> columnFamilyDescriptors,
+      int columnFamilyDescriptorsLength)
       throws RocksDBException;
   protected native static List<byte[]> listColumnFamilies(
       long optionsHandle, String path) throws RocksDBException;
   protected native void openROnly(
       long optionsHandle, String path) throws RocksDBException;
   protected native List<Long> openROnly(
-      long optionsHandle, String path, List<String> columnFamilyNames,
-      int columnFamilyNamesLength) throws RocksDBException;
+      long optionsHandle, String path,
+      List<ColumnFamilyDescriptor> columnFamilyDescriptors,
+      int columnFamilyDescriptorsLength) throws RocksDBException;
   protected native void put(
       long handle, byte[] key, int keyLen,
       byte[] value, int valueLen) throws RocksDBException;
@@ -1197,10 +1300,13 @@ public class RocksDB extends RocksObject {
   protected native void releaseSnapshot(
       long nativeHandle, long snapshotHandle);
   private native void disposeInternal(long handle);
-
-  private native long createColumnFamily(long handle, long opt_handle,
-      String name) throws RocksDBException;
+  private native long createColumnFamily(long handle,
+      ColumnFamilyDescriptor columnFamilyDescriptor) throws RocksDBException;
   private native void dropColumnFamily(long handle, long cfHandle) throws RocksDBException;
+  private native void flush(long handle, long flushOptHandle)
+      throws RocksDBException;
+  private native void flush(long handle, long flushOptHandle,
+      long cfHandle) throws RocksDBException;
 
-  protected Options options_;
+  protected DBOptionsInterface options_;
 }

java/org/rocksdb/RocksDBException.java

@@ -5,8 +5,6 @@
 
 package org.rocksdb;
 
-import java.util.*;
-
 /**
  * A RocksDBException encapsulates the error of an operation.  This exception
  * type is used to describe an internal error from the c++ rocksdb library.

java/org/rocksdb/RocksEnv.java

@@ -29,6 +29,8 @@ public class RocksEnv extends RocksObject {
    * belongs to rocksdb c++.  As a result, the returned RocksEnv will not
    * have the ownership of its c++ resource, and calling its dispose()
    * will be no-op.</p>
+   *
+   * @return the default {@link org.rocksdb.RocksEnv} instance.
    */
   public static RocksEnv getDefault() {
     return default_env_;
@@ -38,6 +40,10 @@ public class RocksEnv extends RocksObject {
    * <p>Sets the number of background worker threads of the flush pool
    * for this environment.</p>
    * <p>Default number: 1</p>
+   *
+   * @param num the number of threads
+   *
+   * @return current {@link org.rocksdb.RocksEnv} instance.
    */
   public RocksEnv setBackgroundThreads(int num) {
     return setBackgroundThreads(num, FLUSH_POOL);
@@ -52,6 +58,7 @@ public class RocksEnv extends RocksObject {
    *     FLUSH_POOL or COMPACTION_POOL.
    *
    * <p>Default number: 1</p>
+   * @return current {@link org.rocksdb.RocksEnv} instance.
    */
   public RocksEnv setBackgroundThreads(int num, int poolID) {
     setBackgroundThreads(nativeHandle_, num, poolID);
@@ -66,6 +73,8 @@ public class RocksEnv extends RocksObject {
    *
    * @param poolID the id to specified a thread pool.  Should be either
    *     FLUSH_POOL or COMPACTION_POOL.
+   *
+   * @return the thread pool queue length.
    */
   public int getThreadPoolQueueLen(int poolID) {
     return getThreadPoolQueueLen(nativeHandle_, poolID);

java/org/rocksdb/RocksIterator.java

@@ -112,6 +112,9 @@ public class RocksIterator extends RocksObject {
    * <p>Position at the first key in the source that at or past target
    * The iterator is valid after this call iff the source contains
    * an entry that comes at or past target.</p>
+   *
+   * @param target byte array describing a key or a
+   *     key prefix to seek for.
    */
   public void seek(byte[] target) {
     assert(isInitialized());
@@ -123,7 +126,8 @@ public class RocksIterator extends RocksObject {
    * If non-blocking IO is requested and this operation cannot be
    * satisfied without doing some IO, then this returns Status::Incomplete().
    *
-   * @throws org.rocksdb.RocksDBException
+   * @throws RocksDBException thrown if error happens in underlying
+   *    native library.
    */
   public void status() throws RocksDBException {
     assert(isInitialized());