Fix the valgrind error

Summary: I this bug from valgrind report and found a place that may potentially leak memory. Test Plan: re-ran the valgrind and no error any more Reviewers: emayanke Reviewed By: emayanke CC: leveldb Differential Revision: https://reviews.facebook.net/D13959
Flush the log outside of lock
2026-11-06 02:52:35 -05:00 · 2013-11-07 15:46:48 -08:00 · 2013-11-07 11:31:56 -08:00 · 2013-11-06 20:45:49 -08:00 · 2013-11-06 18:46:28 -08:00 · 2013-11-06 15:40:26 -08:00
536 changed files with 9417 additions and 108943 deletions
@@ -1,4 +1,5 @@
 build_config.mk
+
 *.a
 *.arc
 *.d
@@ -9,12 +10,10 @@ build_config.mk
 *.so
 *.so.*
 *_test
-db_bench
-db_repl_stress
-db_stress
+*_bench
+*_stress
+
 ldb
-leveldb_server
-leveldb_shell
 manifest_dump
 sst_dump
 util/build_version.cc
@@ -1,3 +1,11 @@
+BSD License
+
+For rocksdb software
+
+Copyright (c) 2013, Facebook, Inc.
+All rights reserved.
+---------------------------------------------------------------------
+
 Copyright (c) 2011 The LevelDB Authors. All rights reserved.

 Redistribution and use in source and binary forms, with or without
@@ -36,6 +36,7 @@ VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)
 VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full

 TESTS = \
+	table_stats_collector_test \
 	arena_test \
 	auto_roll_logger_test \
 	block_test \
@@ -48,6 +49,7 @@ TESTS = \
 	db_test \
 	dbformat_test \
 	env_test \
+	blob_store_test \
 	filelock_test \
 	filename_test \
 	filter_block_test \
@@ -58,6 +60,7 @@ TESTS = \
 	merge_test \
 	redis_test \
 	reduce_levels_test \
+	simple_table_db_test \
 	skiplist_test \
 	stringappend_test \
 	table_test \
@@ -71,10 +74,11 @@ TOOLS = \
        sst_dump \
        db_stress \
        ldb \
-	db_repl_stress
+	db_repl_stress \
+	blob_store_bench

 PROGRAMS = db_bench signal_test $(TESTS) $(TOOLS)
-BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
+BENCHMARKS = db_bench_sqlite3 db_bench_tree_db table_reader_bench

 # The library name is configurable since we are maintaining libraries of both
 # debug/release mode.
@@ -84,7 +88,9 @@ MEMENVLIBRARY = libmemenv.a

 default: all

-# Should we build shared libraries?
+#-----------------------------------------------
+# Create platform independent shared libraries.
+#-----------------------------------------------
 ifneq ($(PLATFORM_SHARED_EXT),)

 ifneq ($(PLATFORM_SHARED_VERSIONED),true)
@@ -111,7 +117,7 @@ $(SHARED3):

 endif  # PLATFORM_SHARED_EXT

-all: $(SHARED) $(LIBRARY) $(PROGRAMS)
+all: $(LIBRARY) $(PROGRAMS)

 .PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests \
 	release tags valgrind_check whitebox_crash_test
@@ -156,7 +162,7 @@ valgrind_check: all $(PROGRAMS) $(TESTS)
 	done

 clean:
-	-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) $(THRIFTSERVER) build_config.mk
+	-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) build_config.mk
 	-rm -rf ios-x86/* ios-arm/*
 	-find . -name "*.[od]" | xargs --no-run-if-empty rm
 	-find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
@@ -180,6 +186,9 @@ db_stress: tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL)
 db_repl_stress: tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL)
 	$(CXX) tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)

+blob_store_bench: tools/blob_store_bench.o $(LIBOBJECTS) $(TESTUTIL)
+	$(CXX) tools/blob_store_bench.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
+
 db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL)
 	$(CXX) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lsqlite3 $(COVERAGEFLAGS)

@@ -192,6 +201,9 @@ signal_test: util/signal_test.o $(LIBOBJECTS)
 arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

+table_stats_collector_test: db/table_stats_collector_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) db/table_stats_collector_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
 bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

@@ -204,6 +216,9 @@ cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
 coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

+blob_store_test: util/blob_store_test.o $(LIBOBJECTS) $(TESTHARNESS) $(TESTUTIL)
+	$(CXX) util/blob_store_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o$@ $(LDFLAGS) $(COVERAGEFLAGS)
+
 stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

@@ -222,9 +237,18 @@ crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS)
 db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

+simple_table_db_test: db/simple_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) db/simple_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
+table_reader_bench: table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
 perf_context_test: db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)

+prefix_test: db/prefix_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) db/prefix_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+
 ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)

@@ -0,0 +1,23 @@
+Additional Grant of Patent Rights
+
+“Software” means the rocksdb software distributed by Facebook, Inc.
+
+Facebook hereby grants you a perpetual, worldwide, royalty-free,
+non-exclusive, irrevocable (subject to the termination provision below)
+license under any rights in any patent claims owned by Facebook, to make,
+have made, use, sell, offer to sell, import, and otherwise transfer the
+Software. For avoidance of doubt, no license is granted under Facebook’s
+rights in any patent claims that are infringed by (i) modifications to the
+Software made by you or a third party, or (ii) the Software in combination
+with any software or other technology provided by you or a third party.
+
+The license granted hereunder will terminate, automatically and without
+notice, for anyone that makes any claim (including by filing any lawsuit,
+assertion or other action) alleging (a) direct, indirect, or contributory
+infringement or inducement to infringe any patent: (i) by Facebook or any
+of its subsidiaries or affiliates, whether or not such claim is related
+to the Software, (ii) by any party if such claim arises in whole or in
+part from any software, product or service of Facebook or any of its
+subsidiaries or affiliates, whether or not such claim is related to the
+Software, or (iii) by any party relating to the Software; or (b) that
+any right in any patent claim of Facebook is invalid or unenforceable.
@@ -1,3 +1,3 @@
 * Detailed instructions on how to compile using fbcode and jemalloc

-* Latest release is 2.4.fb
+* Latest release is 2.5.fb
@@ -134,14 +134,6 @@ $PWD/build_tools/build_detect_version
 # of all files matching either rule, so we need to append -print to make the
 # prune take effect.
 DIRS="util db table utilities"
-if test "$USE_THRIFT"; then
-  DIRS="$DIRS thrift/server_utils.cpp thrift/gen-cpp "
-  THRIFTSERVER=leveldb_server
-fi
-
-if test "$USE_SCRIBE"; then
-  DIRS="$DIRS scribe "
-fi

 set -f # temporarily disable globbing so that our patterns arent expanded
 PRUNE_TEST="-name *test*.cc -prune"
@@ -183,6 +175,17 @@ EOF
        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS ${SNAPPY_LDFLAGS:-./snappy/libs/libsnappy.a}"
    fi

+
+    # Test whether gflags library is installed
+    # http://code.google.com/p/gflags/
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <gflags/gflags.h>
+      int main() {}
+EOF
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS"
+    fi
+
    # Test whether zlib library is installed
    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
      #include <zlib.h>
@@ -227,14 +230,6 @@ if test "$USE_HDFS"; then
  PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $HDFS_LDFLAGS"
 fi

-# shall we build thrift server or scribe logger
-if test "$USE_THRIFT" ||  test "$USE_SCRIBE" ; then
-    THRIFT_CCFLAGS=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp -I/usr/include -std=gnu++0x"
-    THRIFT_LDFLAGS=" -lexample -lserver -lthrift_base -ltransport -lthrift_exception -lutil -L./thrift/libs "
-    COMMON_FLAGS="$COMMON_FLAGS $THRIFT_CCFLAGS"
-    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $THRIFT_LDFLAGS"
-fi
-
 # if Intel SSE instruction set is supported, set USE_SSE=" -msse -msse4.2 "
 COMMON_FLAGS="$COMMON_FLAGS $USE_SSE"

@@ -254,5 +249,4 @@ echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
 echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT
 echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT
 echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT
-echo "THRIFTSERVER=$THRIFTSERVER" >> $OUTPUT
 echo "EXEC_LDFLAGS=$EXEC_LDFLAGS" >> $OUTPUT
@@ -20,7 +20,7 @@ if [ "$?" = 0 ]; then
  BEGIN {
    print "#include \"build_version.h\"\n"
  }
-  { print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";" }
+  { print "const char* rocksdb_build_git_sha = \"rocksdb_build_git_sha:" $0"\";" }
  ' >  ${VFILE}
 else
  echo "git not found" |
@@ -28,13 +28,13 @@ else
    BEGIN {
      print "#include \"build_version.h\""
    }
-    { print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";" }
+    { print "const char* rocksdb_build_git_sha = \"rocksdb_build_git_sha:git not found\";" }
    ' > ${VFILE}
 fi

-echo "const char* leveldb_build_git_datetime = \"leveldb_build_git_datetime:$(date)\";" >> ${VFILE}
-echo "const char* leveldb_build_compile_date = __DATE__;" >> ${VFILE}
-echo "const char* leveldb_build_compile_time = __TIME__;" >> ${VFILE}
+echo "const char* rocksdb_build_git_datetime = \"rocksdb_build_git_datetime:$(date)\";" >> ${VFILE}
+echo "const char* rocksdb_build_compile_date = __DATE__;" >> ${VFILE}
+echo "const char* rocksdb_build_compile_time = __TIME__;" >> ${VFILE}

 OUTFILE=$ROCKSDB_ROOT/util/build_version.cc
 if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
@@ -1,86 +0,0 @@
-#!/bin/bash -e
-function print_usage {
-  echo "Usage: $0 [build | release | bump_version version ]"
-}
-
-function check_env {
-  if [ -z $LEVELDB_HOME ] ; then
-    echo "Expect LEVELDB_HOME to be SET"
-    exit 1
-  fi
-  if [ -z $JAVA_HOME ] ; then
-    export JAVA_HOME=/usr/local/jdk-6u14-64/
-    echo "JAVA_HOME not set. Assuming JAVA_HOME=$JAVA_HOME"
-  fi
-  if [ -z $LEVELDBJNI_HOME ] ; then
-    export LEVELDBJNI_HOME=$LEVELDB_HOME/java/leveldbjni/
-    echo "LEVELDBJNI_HOME not set. Assuming LEVELDBJNI_HOME=$LEVELDBJNI_HOME"
-  fi
-  if [ -z $SNAPPY_HOME ] ; then
-    export SNAPPY_HOME="/home/dhruba/snappy-1.0.5"
-    echo "SNAPPY_HOME not set. Assuming SNAPPY_HOME=$SNAPPY_HOME"
-  fi
-  if [ -z $LEVELDB_PATCH ] ; then 
-    LEVELDB_PATCH=$LEVELDB_HOME/java/leveldbjni/db.h.patch
-    echo "LEVELDB_PATCH not set. Assuming LEVELDB_PATCH=$LEVELDB_PATCH"
-  fi
-}
-
-function build {
-  cd $LEVELDB_HOME
-  git apply $LEVELDB_PATCH
-  make clean libleveldb.a
-  cd $LEVELDB_HOME/java/leveldb/leveldb-api
-  mvn clean package
-  cd $LEVELDBJNI_HOME
-  mvn clean install -P linux64
-  cd $LEVELDB_HOME
-  git checkout $LEVELDB_HOME/include/leveldb/db.h
-}
-
-function release {
-  cd $LEVELDB_HOME
-  git apply $LEVELDB_PATCH
-  make clean libleveldb.a
-  cd $LEVELDB_HOME/java/leveldb/leveldb-api
-  mvn clean package
-  cd $LEVELDBJNI_HOME
-  mvn deploy -P linux64 -DskipTests
-  cd $LEVELDB_HOME
-  git checkout $LEVELDB_HOME/include/leveldb/db.h
-}
-
-CMD=$1
-if [ -z $CMD ]; then
-  print_usage
-  exit 1
-fi
-
-case "$CMD" in 
-  build) 
-    check_env
-    build
-    ;;
-  bump_version)
-    if [ -z $2 ] ; then
-      echo "bump_version requires a version.no parameter at the end"
-      exit 1
-    fi 
-    check_env
-    VERSION=$2
-    pushd $LEVELDBJNI_HOME
-    mvn versions:set -DnewVersion="$VERSION"
-    popd
-    echo "VERSION SET TO $VERSION"
-    ;;
-  release)
-    check_env
-    pushd $LEVELDBJNI_HOME
-    release
-    popd
-    ;;
-  *)
-    print_usage
-    exit 1
-    ;;
-esac
@@ -4,30 +4,29 @@
 # fbcode settings.  It uses the latest g++ compiler and also
 # uses jemalloc

-TOOLCHAIN_REV=f365dbeae46a30414a2874a6f45e73e10f1caf7d
+TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
 TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
 TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
-TOOL_JEMALLOC=jemalloc-3.0.0/2f45f3a
+TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3
 GLIBC_RUNTIME_PATH=/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1

 # location of snappy headers and libraries
 SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include"
 SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"

-# location of boost headers and libraries
-THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/include -std=gnu++0x"
-THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp"
-THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/lib"
-
 # location of libevent
 LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include"
 LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"

+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include"
+GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a"
+
 # use Intel SSE support for checksum calculations
 export USE_SSE=" -msse -msse4.2 "

 CC="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang $CLANG_INCLUDES"
-CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
+CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $LIBEVENT_INCLUDE $GFLAGS_INCLUDE"
 AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
 RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib

@@ -37,8 +36,8 @@ CXXFLAGS="$CFLAGS -nostdinc++ -std=gnu++0x"
 CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"

 EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
-EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a"
-EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
+EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
+EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $LIBEVENT_LIBS $GFLAGS_LIBS"
 EXEC_LDFLAGS+=" -Wl,--dynamic-linker,$GLIBC_RUNTIME_PATH/lib/ld-linux-x86-64.so.2"
 EXEC_LDFLAGS+=" -B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin"

@@ -4,10 +4,10 @@
 # fbcode settings.  It uses the latest g++ compiler and also
 # uses jemalloc

-TOOLCHAIN_REV=83eb773e262fa705eaebbf3de40db71d53fabf2e
+TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
 TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
 TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
-TOOL_JEMALLOC=jemalloc-3.3.1/2f45f3a
+TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3

 # location of libhdfs libraries
 if test "$USE_HDFS"; then
@@ -15,7 +15,7 @@ if test "$USE_HDFS"; then
  JINCLUDE="-I$JAVA_HOME/include -I$JAVA_HOME/include/linux"
  GLIBC_RUNTIME_PATH="/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1"
  HDFSLIB=" -Wl,--no-whole-archive hdfs/libhdfs.a -L$JAVA_HOME/jre/lib/amd64 "
-  HDFSLIB+=" -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib " 
+  HDFSLIB+=" -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib "
  HDFSLIB+=" -ldl -lverify -ljava -ljvm "
 fi

@@ -27,20 +27,19 @@ SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
 ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include"
 ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a"

-# location of boost headers and libraries
-THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/include"
-THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp"
-THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/lib"
-
 # location of libevent
 LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include"
 LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"

+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include"
+GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a"
+
 # use Intel SSE support for checksum calculations
 export USE_SSE=" -msse -msse4.2 "

 CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/gcc"
-CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
+CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $LIBEVENT_INCLUDE $GFLAGS_INCLUDE"
 AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
 RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib

@@ -48,8 +47,8 @@ CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/bin/gold -m64 -mtune=g
 CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"

 EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
-EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a"
-EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
+EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
+EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $LIBEVENT_LIBS $GFLAGS_LIBS"

 PLATFORM_LDFLAGS="-L$TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/lib -L$TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"

@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -10,12 +15,23 @@
 #include "db/table_cache.h"
 #include "db/version_edit.h"
 #include "rocksdb/db.h"
+#include "rocksdb/table.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
+#include "rocksdb/options.h"
+#include "table/block_based_table_builder.h"
 #include "util/stop_watch.h"

 namespace rocksdb {

+class TableFactory;
+
+TableBuilder* GetTableBuilder(const Options& options, WritableFile* file,
+                              CompressionType compression_type) {
+  return options.table_factory->GetTableBuilder(options, file,
+                                                compression_type);
+}
+
 Status BuildTable(const std::string& dbname,
                  Env* env,
                  const Options& options,
@@ -25,7 +41,8 @@ Status BuildTable(const std::string& dbname,
                  FileMetaData* meta,
                  const Comparator* user_comparator,
                  const SequenceNumber newest_snapshot,
-                  const SequenceNumber earliest_seqno_in_memtable) {
+                  const SequenceNumber earliest_seqno_in_memtable,
+                  const bool enable_compression) {
  Status s;
  meta->file_size = 0;
  meta->smallest_seqno = meta->largest_seqno = 0;
@@ -46,7 +63,9 @@ Status BuildTable(const std::string& dbname,
    if (!s.ok()) {
      return s;
    }
-    TableBuilder* builder = new TableBuilder(options, file.get(), 0);
+
+    TableBuilder* builder = GetTableBuilder(options, file.get(),
+                                            options.compression);

    // the first key is the smallest key
    Slice key = iter->key();
@@ -1,13 +1,15 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_BUILDER_H_
-#define STORAGE_LEVELDB_DB_BUILDER_H_
-
+#pragma once
 #include "rocksdb/comparator.h"
 #include "rocksdb/status.h"
 #include "rocksdb/types.h"
+#include "rocksdb/options.h"

 namespace rocksdb {

@@ -19,6 +21,12 @@ class EnvOptions;
 class Iterator;
 class TableCache;
 class VersionEdit;
+class TableBuilder;
+class WritableFile;
+
+
+extern TableBuilder* GetTableBuilder(const Options& options, WritableFile* file,
+                                     CompressionType compression_type);

 // Build a Table file from the contents of *iter.  The generated file
 // will be named according to meta->number.  On success, the rest of
@@ -34,8 +42,7 @@ extern Status BuildTable(const std::string& dbname,
                         FileMetaData* meta,
                         const Comparator* user_comparator,
                         const SequenceNumber newest_snapshot,
-                         const SequenceNumber earliest_seqno_in_memtable);
+                         const SequenceNumber earliest_seqno_in_memtable,
+                         const bool enable_compression);

 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_BUILDER_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -549,6 +554,11 @@ void leveldb_options_set_WAL_ttl_seconds(leveldb_options_t* opt, uint64_t ttl) {
  opt->rep.WAL_ttl_seconds = ttl;
 }

+void leveldb_options_set_WAL_size_limit_MB(
+    leveldb_options_t* opt, uint64_t limit) {
+  opt->rep.WAL_size_limit_MB = limit;
+}
+
 leveldb_comparator_t* leveldb_comparator_create(
    void* state,
    void (*destructor)(void*),
@@ -296,7 +296,7 @@ int main(int argc, char** argv) {
  {
    char* prop = leveldb_property_value(db, "nosuchprop");
    CheckCondition(prop == NULL);
-    prop = leveldb_property_value(db, "leveldb.stats");
+    prop = leveldb_property_value(db, "rocksdb.stats");
    CheckCondition(prop != NULL);
    Free(&prop);
  }
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -10,12 +15,12 @@
 #include <sys/types.h>
 #include "rocksdb/cache.h"
 #include "rocksdb/env.h"
+#include "rocksdb/table.h"
 #include "rocksdb/write_batch.h"
 #include "db/db_impl.h"
 #include "db/filename.h"
 #include "db/log_format.h"
 #include "db/version_set.h"
-#include "table/table.h"
 #include "util/logging.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
@@ -216,10 +221,15 @@ TEST(CorruptionTest, NewFileErrorDuringWrite) {
  const int num = 3 + (Options().write_buffer_size / kValueSize);
  std::string value_storage;
  Status s;
-  for (int i = 0; s.ok() && i < num; i++) {
+  bool failed = false;
+  for (int i = 0; i < num; i++) {
    WriteBatch batch;
    batch.Put("a", Value(100, &value_storage));
    s = db_->Write(WriteOptions(), &batch);
+    if (!s.ok()) {
+      failed = true;
+    }
+    ASSERT_TRUE(!failed || !s.ok());
  }
  ASSERT_TRUE(!s.ok());
  ASSERT_GE(env_.num_writable_file_errors_, 1);
@@ -230,7 +240,7 @@ TEST(CorruptionTest, NewFileErrorDuringWrite) {
 TEST(CorruptionTest, TableFile) {
  Build(100);
  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  dbi->TEST_CompactMemTable();
+  dbi->TEST_FlushMemTable();
  dbi->TEST_CompactRange(0, nullptr, nullptr);
  dbi->TEST_CompactRange(1, nullptr, nullptr);

@@ -241,7 +251,7 @@ TEST(CorruptionTest, TableFile) {
 TEST(CorruptionTest, TableFileIndexData) {
  Build(10000);  // Enough to build multiple Tables
  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  dbi->TEST_CompactMemTable();
+  dbi->TEST_FlushMemTable();

  Corrupt(kTableFile, -2000, 500);
  Reopen();
@@ -279,7 +289,7 @@ TEST(CorruptionTest, SequenceNumberRecovery) {
 TEST(CorruptionTest, CorruptedDescriptor) {
  ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  dbi->TEST_CompactMemTable();
+  dbi->TEST_FlushMemTable();
  dbi->TEST_CompactRange(0, nullptr, nullptr);

  Corrupt(kDescriptorFile, 0, 1000);
@@ -296,9 +306,9 @@ TEST(CorruptionTest, CorruptedDescriptor) {
 TEST(CorruptionTest, CompactionInputError) {
  Build(10);
  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  dbi->TEST_CompactMemTable();
+  dbi->TEST_FlushMemTable();
  const int last = dbi->MaxMemCompactionLevel();
-  ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
+  ASSERT_EQ(1, Property("rocksdb.num-files-at-level" + NumberToString(last)));

  Corrupt(kTableFile, 100, 1);
  Check(9, 9);
@@ -319,12 +329,13 @@ TEST(CorruptionTest, CompactionInputErrorParanoid) {
  for (int level = 1; level < dbi->NumberLevels(); level++) {
    dbi->Put(WriteOptions(), "", "begin");
    dbi->Put(WriteOptions(), "~", "end");
-    dbi->TEST_CompactMemTable();
+    dbi->TEST_FlushMemTable();
  }

  Build(10);
-  dbi->TEST_CompactMemTable();
-  ASSERT_EQ(1, Property("leveldb.num-files-at-level0"));
+  dbi->TEST_FlushMemTable();
+  dbi->TEST_WaitForCompact();
+  ASSERT_EQ(1, Property("rocksdb.num-files-at-level0"));

  Corrupt(kTableFile, 100, 1);
  Check(9, 9);
@@ -332,8 +343,14 @@ TEST(CorruptionTest, CompactionInputErrorParanoid) {
  // Write must eventually fail because of corrupted table
  Status s;
  std::string tmp1, tmp2;
+  bool failed = false;
  for (int i = 0; i < 10000 && s.ok(); i++) {
    s = db_->Put(WriteOptions(), Key(i, &tmp1), Value(i, &tmp2));
+    if (!s.ok()) {
+      failed = true;
+    }
+    // if one write failed, every subsequent write must fail, too
+    ASSERT_TRUE(!failed || !s.ok()) << "write did not fail in a corrupted db";
  }
  ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
 }
@@ -341,7 +358,7 @@ TEST(CorruptionTest, CompactionInputErrorParanoid) {
 TEST(CorruptionTest, UnrelatedKeys) {
  Build(10);
  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-  dbi->TEST_CompactMemTable();
+  dbi->TEST_FlushMemTable();
  Corrupt(kTableFile, 100, 1);

  std::string tmp1, tmp2;
@@ -349,7 +366,7 @@ TEST(CorruptionTest, UnrelatedKeys) {
  std::string v;
  ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
  ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
-  dbi->TEST_CompactMemTable();
+  dbi->TEST_FlushMemTable();
  ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
  ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
 }
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2012 Facebook.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
@@ -75,7 +80,7 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {

  // list wal files in archive dir.
  Status s;
-  std::string archivedir = ArchivalDirectory(dbname_);
+  std::string archivedir = ArchivalDirectory(options_.wal_dir);
  if (env_->FileExists(archivedir)) {
    s = AppendSortedWalsOfType(archivedir, files, kArchivedLogFile);
    if (!s.ok()) {
@@ -83,32 +88,7 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
    }
  }
  // list wal files in main db dir.
-  return AppendSortedWalsOfType(dbname_, files, kAliveLogFile);
-}
-
-Status DBImpl::DeleteWalFiles(const VectorLogPtr& files) {
-  Status s;
-  std::string archivedir = ArchivalDirectory(dbname_);
-  std::string files_not_deleted;
-  for (const auto& wal : files) {
-    /* Try deleting in the dir that pathname points to for the logfile.
-       This may fail if we try to delete a log file which was live when captured
-       but is archived now. Try deleting it from archive also
-     */
-    Status st = env_->DeleteFile(dbname_ + "/" + wal->PathName());
-    if (!st.ok()) {
-      if (wal->Type() == kAliveLogFile &&
-          env_->DeleteFile(LogFileName(archivedir, wal->LogNumber())).ok()) {
-        continue;
-      }
-      files_not_deleted.append(wal->PathName());
-    }
-  }
-  if (!files_not_deleted.empty()) {
-    return Status::IOError("Deleted all requested files except: " +
-                           files_not_deleted);
-  }
-  return Status::OK();
+  return AppendSortedWalsOfType(options_.wal_dir, files, kAliveLogFile);
 }

 }
@@ -1,10 +1,12 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_
-#define STORAGE_LEVELDB_DB_DB_IMPL_H_
-
+#pragma once
 #include <atomic>
 #include <deque>
 #include <set>
@@ -20,10 +22,6 @@
 #include "util/stats_logger.h"
 #include "memtablelist.h"

-#ifdef USE_SCRIBE
-#include "scribe/scribe_logger.h"
-#endif
-
 namespace rocksdb {

 class MemTable;
@@ -75,8 +73,7 @@ class DBImpl : public DB {
                              uint64_t* manifest_file_size,
                              bool flush_memtable = true);
  virtual Status GetSortedWalFiles(VectorLogPtr& files);
-  virtual Status DeleteWalFiles(const VectorLogPtr& files);
-  virtual SequenceNumber GetLatestSequenceNumber();
+  virtual SequenceNumber GetLatestSequenceNumber() const;
  virtual Status GetUpdatesSince(SequenceNumber seq_number,
                                 unique_ptr<TransactionLogIterator>* iter);
  virtual Status DeleteFile(std::string name);
@@ -89,11 +86,11 @@ class DBImpl : public DB {
  // Compact any files in the named level that overlap [*begin, *end]
  void TEST_CompactRange(int level, const Slice* begin, const Slice* end);

-  // Force current memtable contents to be compacted.
-  Status TEST_CompactMemTable();
+  // Force current memtable contents to be flushed.
+  Status TEST_FlushMemTable();

  // Wait for memtable compaction
-  Status TEST_WaitForCompactMemTable();
+  Status TEST_WaitForFlushMemTable();

  // Wait for any compaction
  Status TEST_WaitForCompact();
@@ -116,6 +113,14 @@ class DBImpl : public DB {
  // Trigger's a background call for testing.
  void TEST_PurgeObsoleteteWAL();

+  // get total level0 file size. Only for testing.
+  uint64_t TEST_GetLevel0TotalSize() { return versions_->NumLevelBytes(0);}
+
+  void TEST_SetDefaultTimeToCheck(uint64_t default_interval_to_delete_obsolete_WAL)
+  {
+    default_interval_to_delete_obsolete_WAL_ = default_interval_to_delete_obsolete_WAL;
+  }
+
 protected:
  Env* const env_;
  const std::string dbname_;
@@ -155,9 +160,9 @@ class DBImpl : public DB {
  // Delete any unneeded files and stale in-memory entries.
  void DeleteObsoleteFiles();

-  // Compact the in-memory write buffer to disk.  Switches to a new
+  // Flush the in-memory write buffer to storage.  Switches to a new
  // log-file/memtable and writes a new descriptor iff successful.
-  Status CompactMemTable(bool* madeProgress = nullptr);
+  Status FlushMemTableToOutputFile(bool* madeProgress = nullptr);

  Status RecoverLogFile(uint64_t log_number,
                        VersionEdit* edit,
@@ -180,21 +185,21 @@ class DBImpl : public DB {
  // Force current memtable contents to be flushed.
  Status FlushMemTable(const FlushOptions& options);

-  // Wait for memtable compaction
-  Status WaitForCompactMemTable();
+  // Wait for memtable flushed
+  Status WaitForFlushMemTable();

  void MaybeScheduleLogDBDeployStats();
  static void BGLogDBDeployStats(void* db);
  void LogDBDeployStats();

-  void MaybeScheduleCompaction();
+  void MaybeScheduleFlushOrCompaction();
  static void BGWorkCompaction(void* db);
  static void BGWorkFlush(void* db);
  void BackgroundCallCompaction();
  void BackgroundCallFlush();
  Status BackgroundCompaction(bool* madeProgress,DeletionState& deletion_state);
-  Status BackgroundFlush();
-  void CleanupCompaction(CompactionState* compact);
+  Status BackgroundFlush(bool* madeProgress);
+  void CleanupCompaction(CompactionState* compact, Status status);
  Status DoCompactionWork(CompactionState* compact);

  Status OpenCompactionOutputFile(CompactionState* compact);
@@ -240,7 +245,7 @@ class DBImpl : public DB {

  void PrintStatistics();

-  // dump leveldb.stats to LOG
+  // dump rocksdb.stats to LOG
  void MaybeDumpStats();

  // Return the minimum empty level that could hold the total data in the
@@ -323,6 +328,10 @@ class DBImpl : public DB {
  // last time stats were dumped to LOG
  std::atomic<uint64_t> last_stats_dump_time_microsec_;

+  // obsolete files will be deleted every this seconds if ttl deletion is
+  // enabled and archive size_limit is disabled.
+  uint64_t default_interval_to_delete_obsolete_WAL_;
+
  // These count the number of microseconds for which MakeRoomForWrite stalls.
  uint64_t stall_level0_slowdown_;
  uint64_t stall_memtable_compaction_;
@@ -402,10 +411,6 @@ class DBImpl : public DB {
  // count of the number of contiguous delaying writes
  int delayed_writes_;

-  // store the last flushed sequence.
-  // Used by transaction log iterator.
-  SequenceNumber last_flushed_sequence_;
-
  // The options to access storage files
  const EnvOptions storage_options_;

@@ -444,6 +449,13 @@ extern Options SanitizeOptions(const std::string& db,
                               const InternalFilterPolicy* ipolicy,
                               const Options& src);

-}  // namespace rocksdb

-#endif  // STORAGE_LEVELDB_DB_DB_IMPL_H_
+// Determine compression type, based on user options, level of the output
+// file and whether compression is disabled.
+// If enable_compression is false, then compression is always disabled no
+// matter what the values of the other two parameters are.
+// Otherwise, the compression type is determined based on options and level.
+CompressionType GetCompressionType(const Options& options, int level,
+                                   const bool enable_compression);
+
+}  // namespace rocksdb
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2012 Facebook. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
@@ -24,11 +29,10 @@
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/status.h"
-#include "rocksdb/table_builder.h"
+#include "rocksdb/table.h"
 #include "port/port.h"
 #include "table/block.h"
 #include "table/merger.h"
-#include "table/table.h"
 #include "table/two_level_iterator.h"
 #include "util/coding.h"
 #include "util/logging.h"
@@ -1,10 +1,13 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2012 Facebook. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

-#ifndef STORAGE_LEVELDB_DB_DB_IMPL_READONLY_H_
-#define STORAGE_LEVELDB_DB_DB_IMPL_READONLY_H_
-
+#pragma once
 #include "db/db_impl.h"

 #include <deque>
@@ -17,10 +20,6 @@
 #include "port/port.h"
 #include "util/stats_logger.h"

-#ifdef USE_SCRIBE
-#include "scribe/scribe_logger.h"
-#endif
-
 namespace rocksdb {

 class DBImplReadOnly : public DBImpl {
@@ -77,5 +76,3 @@ private:
 };

 }
-
-#endif
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -368,6 +373,7 @@ void DBIter::FindPrevUserEntry() {
  if (iter_->Valid()) {
    do {
      ParsedInternalKey ikey;
+      bool saved_key_cleared = false;
      if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
        if ((value_type != kTypeDeletion) &&
            user_comparator_->Compare(ikey.user_key, saved_key_) < 0) {
@@ -378,6 +384,7 @@ void DBIter::FindPrevUserEntry() {
        if (value_type == kTypeDeletion) {
          saved_key_.clear();
          ClearSavedValue();
+          saved_key_cleared = true;
        } else {
          Slice raw_value = iter_->value();
          if (saved_value_.capacity() > raw_value.size() + 1048576) {
@@ -393,7 +400,7 @@ void DBIter::FindPrevUserEntry() {
      // found the prev user-key, then it is better to seek so that we can
      // avoid too many key comparisons. We seek to the first occurence of
      // our current key by looking for max sequence number.
-      if (num_skipped > max_skip_) {
+      if (!saved_key_cleared && num_skipped > max_skip_) {
        num_skipped = 0;
        std::string last_key;
        AppendInternalKey(&last_key,
@@ -1,10 +1,13 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#ifndef STORAGE_LEVELDB_DB_DB_ITER_H_
-#define STORAGE_LEVELDB_DB_DB_ITER_H_
-
+#pragma once
 #include <stdint.h>
 #include "rocksdb/db.h"
 #include "db/dbformat.h"
@@ -23,5 +26,3 @@ extern Iterator* NewDBIterator(
    const SequenceNumber& sequence);

 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_DB_ITER_H_
@@ -1,10 +1,13 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#ifndef LEVELDB_STORAGE_DB_DB_STATISTICS_H_
-#define LEVELDB_STORAGE_DB_DB_STATISTICS_H_
-
+#pragma once
 #include <cassert>
 #include <stdlib.h>
 #include <vector>
@@ -60,5 +63,3 @@ std::shared_ptr<Statistics> CreateDBStatistics() {
 }

 } // namespace rocksdb
-
-#endif // LEVELDB_STORAGE_DB_DB_STATISTICS_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -1,16 +1,19 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#ifndef STORAGE_LEVELDB_DB_FORMAT_H_
-#define STORAGE_LEVELDB_DB_FORMAT_H_
-
+#pragma once
 #include <stdio.h>
 #include "rocksdb/comparator.h"
 #include "rocksdb/db.h"
 #include "rocksdb/filter_policy.h"
 #include "rocksdb/slice.h"
-#include "rocksdb/table_builder.h"
+#include "rocksdb/table.h"
 #include "rocksdb/types.h"
 #include "util/coding.h"
 #include "util/logging.h"
@@ -90,7 +93,7 @@ class InternalKeyComparator : public Comparator {
  std::string name_;
 public:
  explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c),
-    name_("leveldb.InternalKeyComparator:" +
+    name_("rocksdb.InternalKeyComparator:" +
          std::string(user_comparator_->Name())) {
  }

@@ -224,5 +227,3 @@ inline LookupKey::~LookupKey() {
 }

 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_FORMAT_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -9,12 +14,12 @@
 #include "db/write_batch_internal.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
-#include "boost/lexical_cast.hpp"
 #include "rocksdb/env.h"
+#include "rocksdb/transaction_log.h"
 #include <vector>
-#include <boost/algorithm/string.hpp>
 #include <stdlib.h>
 #include <map>
+#include <string>

 namespace rocksdb {

@@ -32,6 +37,8 @@ class DeleteFileTest {
    options_.write_buffer_size = 1024*1024*1000;
    options_.target_file_size_base = 1024*1024*1000;
    options_.max_bytes_for_level_base = 1024*1024*1000;
+    options_.WAL_ttl_seconds = 300; // Used to test log files
+    options_.WAL_size_limit_MB = 1024; // Used to test log files
    dbname_ = test::TmpDir() + "/deletefile_test";
    DestroyDB(dbname_, options_);
    numlevels_ = 7;
@@ -57,7 +64,7 @@ class DeleteFileTest {
    options.sync = false;
    ReadOptions roptions;
    for (int i = startkey; i < (numkeys + startkey) ; i++) {
-      std::string temp = boost::lexical_cast<std::string>(i);
+      std::string temp = std::to_string(i);
      Slice key(temp);
      Slice value(temp);
      ASSERT_OK(db_->Put(options, key, value));
@@ -92,12 +99,12 @@ class DeleteFileTest {
  void CreateTwoLevels() {
    AddKeys(50000, 10000);
    DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-    ASSERT_OK(dbi->TEST_CompactMemTable());
-    ASSERT_OK(dbi->TEST_WaitForCompactMemTable());
+    ASSERT_OK(dbi->TEST_FlushMemTable());
+    ASSERT_OK(dbi->TEST_WaitForFlushMemTable());

    AddKeys(50000, 10000);
-    ASSERT_OK(dbi->TEST_CompactMemTable());
-    ASSERT_OK(dbi->TEST_WaitForCompactMemTable());
+    ASSERT_OK(dbi->TEST_FlushMemTable());
+    ASSERT_OK(dbi->TEST_WaitForFlushMemTable());
  }

 };
@@ -149,7 +156,6 @@ TEST(DeleteFileTest, AddKeysAndQueryLevels) {
  CloseDB();
 }

-
 TEST(DeleteFileTest, DeleteFileWithIterator) {
  CreateTwoLevels();
  ReadOptions options;
@@ -180,6 +186,42 @@ TEST(DeleteFileTest, DeleteFileWithIterator) {
  delete it;
  CloseDB();
 }
+
+TEST(DeleteFileTest, DeleteLogFiles) {
+  AddKeys(10, 0);
+  VectorLogPtr logfiles;
+  db_->GetSortedWalFiles(logfiles);
+  ASSERT_GT(logfiles.size(), 0UL);
+  // Take the last log file which is expected to be alive and try to delete it
+  // Should not succeed because live logs are not allowed to be deleted
+  std::unique_ptr<LogFile> alive_log = std::move(logfiles.back());
+  ASSERT_EQ(alive_log->Type(), kAliveLogFile);
+  ASSERT_TRUE(env_->FileExists(dbname_ + "/" + alive_log->PathName()));
+  fprintf(stdout, "Deleting alive log file %s\n",
+          alive_log->PathName().c_str());
+  ASSERT_TRUE(!db_->DeleteFile(alive_log->PathName()).ok());
+  ASSERT_TRUE(env_->FileExists(dbname_ + "/" + alive_log->PathName()));
+  logfiles.clear();
+
+  // Call Flush to bring about a new working log file and add more keys
+  // Call Flush again to flush out memtable and move alive log to archived log
+  // and try to delete the archived log file
+  FlushOptions fopts;
+  db_->Flush(fopts);
+  AddKeys(10, 0);
+  db_->Flush(fopts);
+  db_->GetSortedWalFiles(logfiles);
+  ASSERT_GT(logfiles.size(), 0UL);
+  std::unique_ptr<LogFile> archived_log = std::move(logfiles.front());
+  ASSERT_EQ(archived_log->Type(), kArchivedLogFile);
+  ASSERT_TRUE(env_->FileExists(dbname_ + "/" + archived_log->PathName()));
+  fprintf(stdout, "Deleting archived log file %s\n",
+          archived_log->PathName().c_str());
+  ASSERT_OK(db_->DeleteFile(archived_log->PathName()));
+  ASSERT_TRUE(!env_->FileExists(dbname_ + "/" + archived_log->PathName()));
+  CloseDB();
+}
+
 } //namespace rocksdb

 int main(int argc, char** argv) {
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -57,8 +62,8 @@ std::string LogFileName(const std::string& name, uint64_t number) {
  return MakeFileName(name, number, "log");
 }

-std::string ArchivalDirectory(const std::string& dbname) {
-  return dbname + "/" + ARCHIVAL_DIR;
+std::string ArchivalDirectory(const std::string& dir) {
+  return dir + "/" + ARCHIVAL_DIR;
 }
 std::string ArchivedLogFileName(const std::string& name, uint64_t number) {
  assert(number > 0);
@@ -87,7 +92,7 @@ std::string LockFileName(const std::string& dbname) {
 }

 std::string TempFileName(const std::string& dbname, uint64_t number) {
-  assert(number > 0);
+  assert(number >= 0);
  return MakeFileName(dbname, number, "dbtmp");
 }

@@ -122,7 +127,12 @@ std::string MetaDatabaseName(const std::string& dbname, uint64_t number) {
  return dbname + buf;
 }

+std::string IdentityFileName(const std::string& dbname) {
+  return dbname + "/IDENTITY";
+}
+
 // Owned filenames have the form:
+//    dbname/IDENTITY
 //    dbname/CURRENT
 //    dbname/LOCK
 //    dbname/LOG
@@ -133,12 +143,16 @@ std::string MetaDatabaseName(const std::string& dbname, uint64_t number) {
 //    Disregards / at the beginning
 bool ParseFileName(const std::string& fname,
                   uint64_t* number,
-                   FileType* type) {
+                   FileType* type,
+                   WalFileType* log_type) {
  Slice rest(fname);
  if (fname.length() > 1 && fname[0] == '/') {
    rest.remove_prefix(1);
  }
-  if (rest == "CURRENT") {
+  if (rest == "IDENTITY") {
+    *number = 0;
+    *type = kIdentityFile;
+  } else if (rest == "CURRENT") {
    *number = 0;
    *type = kCurrentFile;
  } else if (rest == "LOCK") {
@@ -181,6 +195,17 @@ bool ParseFileName(const std::string& fname,
  } else {
    // Avoid strtoull() to keep filename format independent of the
    // current locale
+    bool archive_dir_found = false;
+    if (rest.starts_with(ARCHIVAL_DIR)) {
+      if (rest.size() <= ARCHIVAL_DIR.size()) {
+        return false;
+      }
+      rest.remove_prefix(ARCHIVAL_DIR.size() + 1); // Add 1 to remove / also
+      if (log_type) {
+        *log_type = kArchivedLogFile;
+      }
+      archive_dir_found = true;
+    }
    uint64_t num;
    if (!ConsumeDecimalNumber(&rest, &num)) {
      return false;
@@ -188,6 +213,11 @@ bool ParseFileName(const std::string& fname,
    Slice suffix = rest;
    if (suffix == Slice(".log")) {
      *type = kLogFile;
+      if (log_type && !archive_dir_found) {
+        *log_type = kAliveLogFile;
+      }
+    } else if (archive_dir_found) {
+      return false; // Archive dir can contain only log files
    } else if (suffix == Slice(".sst")) {
      *type = kTableFile;
    } else if (suffix == Slice(".dbtmp")) {
@@ -218,4 +248,19 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
  return s;
 }

+Status SetIdentityFile(Env* env, const std::string& dbname) {
+  std::string id = env->GenerateUniqueId();
+  assert(!id.empty());
+  // Reserve the filename dbname/000000.dbtmp for the temporary identity file
+  std::string tmp = TempFileName(dbname, 0);
+  Status s = WriteStringToFileSync(env, id, tmp);
+  if (s.ok()) {
+    s = env->RenameFile(tmp, IdentityFileName(dbname));
+  }
+  if (!s.ok()) {
+    env->DeleteFile(tmp);
+  }
+  return s;
+}
+
 }  // namespace rocksdb
@@ -1,16 +1,20 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 //
 // File names used by DB code

-#ifndef STORAGE_LEVELDB_DB_FILENAME_H_
-#define STORAGE_LEVELDB_DB_FILENAME_H_
-
+#pragma once
 #include <stdint.h>
 #include <string>
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"
+#include "rocksdb/transaction_log.h"
 #include "port/port.h"

 namespace rocksdb {
@@ -25,7 +29,8 @@ enum FileType {
  kCurrentFile,
  kTempFile,
  kInfoLogFile,  // Either the current one, or an old one
-  kMetaDatabase
+  kMetaDatabase,
+  kIdentityFile
 };

 // Return the name of the log file with the specified number
@@ -79,19 +84,25 @@ extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
 extern std::string MetaDatabaseName(const std::string& dbname,
                                    uint64_t number);

-// If filename is a leveldb file, store the type of the file in *type.
+// Return the name of the Identity file which stores a unique number for the db
+// that will get regenerated if the db loses all its data and is recreated fresh
+// either from a backup-image or empty
+extern std::string IdentityFileName(const std::string& dbname);
+
+// If filename is a rocksdb file, store the type of the file in *type.
 // The number encoded in the filename is stored in *number.  If the
 // filename was successfully parsed, returns true.  Else return false.
 extern bool ParseFileName(const std::string& filename,
                          uint64_t* number,
-                          FileType* type);
+                          FileType* type,
+                          WalFileType* log_type = nullptr);

 // Make the CURRENT file point to the descriptor file with the
 // specified number.
 extern Status SetCurrentFile(Env* env, const std::string& dbname,
                             uint64_t descriptor_number);

+// Make the IDENTITY file for the db
+extern Status SetIdentityFile(Env* env, const std::string& dbname);

 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_FILENAME_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -5,9 +10,7 @@
 // Log format information shared by reader and writer.
 // See ../doc/log_format.txt for more detail.

-#ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_
-#define STORAGE_LEVELDB_DB_LOG_FORMAT_H_
-
+#pragma once
 namespace rocksdb {
 namespace log {

@@ -31,5 +34,3 @@ static const int kHeaderSize = 4 + 1 + 2;

 }  // namespace log
 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_LOG_FORMAT_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -1,10 +1,13 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#ifndef STORAGE_LEVELDB_DB_LOG_READER_H_
-#define STORAGE_LEVELDB_DB_LOG_READER_H_
-
+#pragma once
 #include <memory>
 #include <stdint.h>

@@ -119,5 +122,3 @@ class Reader {

 }  // namespace log
 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_LOG_READER_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -1,10 +1,13 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_
-#define STORAGE_LEVELDB_DB_LOG_WRITER_H_
-
+#pragma once
 #include <memory>
 #include <stdint.h>
 #include "db/log_format.h"
@@ -50,5 +53,3 @@ class Writer {

 }  // namespace log
 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_LOG_WRITER_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -12,8 +17,18 @@
 #include "rocksdb/iterator.h"
 #include "rocksdb/merge_operator.h"
 #include "util/coding.h"
+#include "util/mutexlock.h"
 #include "util/murmurhash.h"

+namespace std {
+template <>
+struct hash<rocksdb::Slice> {
+  size_t operator()(const rocksdb::Slice& slice) const {
+    return MurmurHash(slice.data(), slice.size(), 0);
+  }
+};
+}
+
 namespace rocksdb {

 MemTable::MemTable(const InternalKeyComparator& cmp,
@@ -30,7 +45,10 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
      edit_(numlevel),
      first_seqno_(0),
      mem_next_logfile_number_(0),
-      mem_logfile_number_(0) { }
+      mem_logfile_number_(0),
+      locks_(options.inplace_update_support
+             ? options.inplace_update_num_locks
+             : 0) { }

 MemTable::~MemTable() {
  assert(refs_ == 0);
@@ -66,11 +84,16 @@ static const char* EncodeKey(std::string* scratch, const Slice& target) {

 class MemTableIterator: public Iterator {
 public:
-  explicit MemTableIterator(MemTableRep* table)
-    : iter_(table->GetIterator()) { }
-
-  MemTableIterator(MemTableRep* table, const Slice* prefix)
-    : iter_(table->GetPrefixIterator(*prefix)) { }
+  MemTableIterator(MemTableRep* table, const ReadOptions& options)
+    : iter_() {
+    if (options.prefix) {
+      iter_ = table->GetPrefixIterator(*options.prefix);
+    } else if (options.prefix_seek) {
+      iter_ = table->GetDynamicPrefixIterator();
+    } else {
+      iter_ = table->GetIterator();
+    }
+  }

  virtual bool Valid() const { return iter_->Valid(); }
  virtual void Seek(const Slice& k) { iter_->Seek(EncodeKey(&tmp_, k)); }
@@ -97,12 +120,12 @@ class MemTableIterator: public Iterator {
  void operator=(const MemTableIterator&);
 };

-Iterator* MemTable::NewIterator(const Slice* prefix) {
-  if (prefix) {
-    return new MemTableIterator(table_.get(), prefix);
-  } else {
-    return new MemTableIterator(table_.get());
-  }
+Iterator* MemTable::NewIterator(const ReadOptions& options) {
+  return new MemTableIterator(table_.get(), options);
+}
+
+port::RWMutex* MemTable::GetLock(const Slice& key) {
+  return &locks_[std::hash<Slice>()(key) % locks_.size()];
 }

 void MemTable::Add(SequenceNumber s, ValueType type,
@@ -164,14 +187,16 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
    // all entries with overly large sequence numbers.
    const char* entry = iter->key();
    uint32_t key_length;
-    const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length);
+    const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
    if (comparator_.comparator.user_comparator()->Compare(
-            Slice(key_ptr, key_length - 8),
-            key.user_key()) == 0) {
+        Slice(key_ptr, key_length - 8), key.user_key()) == 0) {
      // Correct user key
      const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
      switch (static_cast<ValueType>(tag & 0xff)) {
        case kTypeValue: {
+          if (options.inplace_update_support) {
+            GetLock(key.user_key())->ReadLock();
+          }
          Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
          *s = Status::OK();
          if (merge_in_progress) {
@@ -184,6 +209,9 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
          } else {
            value->assign(v.data(), v.size());
          }
+          if (options.inplace_update_support) {
+            GetLock(key.user_key())->Unlock();
+          }
          return true;
        }
        case kTypeDeletion: {
@@ -238,4 +266,65 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
  return false;
 }

+bool MemTable::Update(SequenceNumber seq, ValueType type,
+                      const Slice& key,
+                      const Slice& value) {
+  LookupKey lkey(key, seq);
+  Slice memkey = lkey.memtable_key();
+
+  std::shared_ptr<MemTableRep::Iterator> iter(
+    table_.get()->GetIterator(lkey.user_key()));
+  iter->Seek(memkey.data());
+
+  if (iter->Valid()) {
+    // entry format is:
+    //    klength  varint32
+    //    userkey  char[klength-8]
+    //    tag      uint64
+    //    vlength  varint32
+    //    value    char[vlength]
+    // Check that it belongs to same user key.  We do not check the
+    // sequence number since the Seek() call above should have skipped
+    // all entries with overly large sequence numbers.
+    const char* entry = iter->key();
+    uint32_t key_length;
+    const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
+    if (comparator_.comparator.user_comparator()->Compare(
+        Slice(key_ptr, key_length - 8), lkey.user_key()) == 0) {
+      // Correct user key
+      const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
+      switch (static_cast<ValueType>(tag & 0xff)) {
+        case kTypeValue: {
+          uint32_t vlength;
+          GetVarint32Ptr(key_ptr + key_length,
+                         key_ptr + key_length+5, &vlength);
+          // Update value, if newValue size  <= curValue size
+          if (value.size() <= vlength) {
+            char* p = EncodeVarint32(const_cast<char*>(key_ptr) + key_length,
+                                     value.size());
+            WriteLock wl(GetLock(lkey.user_key()));
+            memcpy(p, value.data(), value.size());
+            assert(
+              (p + value.size()) - entry ==
+              (unsigned) (VarintLength(key_length) +
+                          key_length +
+                          VarintLength(value.size()) +
+                          value.size())
+            );
+            return true;
+          }
+        }
+        default:
+          // If the latest value is kTypeDeletion, kTypeMerge or kTypeLogData
+          // then we probably don't have enough space to update in-place
+          // Maybe do something later
+          // Return false, and do normal Add()
+          return false;
+      }
+    }
+  }
+
+  // Key doesn't exist
+  return false;
+}
 }  // namespace rocksdb
@@ -1,10 +1,13 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#ifndef STORAGE_LEVELDB_DB_MEMTABLE_H_
-#define STORAGE_LEVELDB_DB_MEMTABLE_H_
-
+#pragma once
 #include <string>
 #include <memory>
 #include <deque>
@@ -62,10 +65,14 @@ class MemTable {
  // iterator are internal keys encoded by AppendInternalKey in the
  // db/dbformat.{h,cc} module.
  //
-  // If a prefix is supplied, it is passed to the underlying MemTableRep as a
-  // hint that the iterator only need to support access to keys with that
-  // prefix.
-  Iterator* NewIterator(const Slice* prefix = nullptr);
+  // If options.prefix is supplied, it is passed to the underlying MemTableRep
+  // as a hint that the iterator only need to support access to keys with that
+  // specific prefix.
+  // If options.prefix is not supplied and options.prefix_seek is set, the
+  // iterator is not bound to a specific prefix. However, the semantics of
+  // Seek is changed - the result might only include keys with the same prefix
+  // as the seek-key.
+  Iterator* NewIterator(const ReadOptions& options = ReadOptions());

  // Add an entry into memtable that maps key to value at the
  // specified sequence number and with the specified type.
@@ -85,6 +92,16 @@ class MemTable {
  bool Get(const LookupKey& key, std::string* value, Status* s,
           std::deque<std::string>* operands, const Options& options);

+  // Update the value and return status ok,
+  //   if key exists in current memtable
+  //     if new sizeof(new_value) <= sizeof(old_value) &&
+  //       old_value for that key is a put i.e. kTypeValue
+  //     else return false, and status - NotUpdatable()
+  //   else return false, and status - NotFound()
+  bool Update(SequenceNumber seq, ValueType type,
+              const Slice& key,
+              const Slice& value);
+
  // Returns the edits area that is needed for flushing the memtable
  VersionEdit* GetEdits() { return &edit_; }

@@ -141,11 +158,15 @@ class MemTable {
  // memtable flush is done)
  uint64_t mem_logfile_number_;

+  // rw locks for inplace updates
+  std::vector<port::RWMutex> locks_;
+
  // No copying allowed
  MemTable(const MemTable&);
  void operator=(const MemTable&);
+
+  // Get the lock associated for the key
+  port::RWMutex* GetLock(const Slice& key);
 };

 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_MEMTABLE_H_
@@ -1,6 +1,8 @@
-// Copyright (c) 2012 Facebook.
-
-
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 #include "db/memtablelist.h"

 #include <string>
@@ -94,15 +96,12 @@ Status MemTableList::InstallMemtableFlushResults(
  }

  // flush was sucessful
-  bool first = true;
-  for (MemTable* m : mems) {
-
+  for (size_t i = 0; i < mems.size(); ++i) {
    // All the edits are associated with the first memtable of this batch.
-    assert(first || m->GetEdits()->NumEntries() == 0);
-    first = false;
+    assert(i == 0 || mems[i]->GetEdits()->NumEntries() == 0);

-    m->flush_completed_ = true;
-    m->file_number_ = file_number;
+    mems[i]->flush_completed_ = true;
+    mems[i]->file_number_ = file_number;
  }

  // if some other thread is already commiting, then return
@@ -122,22 +121,21 @@ Status MemTableList::InstallMemtableFlushResults(
    if (!m->flush_completed_) {
      break;
    }
-    first = true;

-    Log(info_log,
-        "Level-0 commit table #%llu: started",
-        (unsigned long long)m->file_number_);
+    Log(info_log, "Level-0 commit table #%lu started", m->file_number_);

    // this can release and reacquire the mutex.
    s = vset->LogAndApply(&m->edit_, mu);

    // All the later memtables that have the same filenum
    // are part of the same batch. They can be committed now.
+    uint64_t mem_id = 1;  // how many memtables has been flushed.
    do {
      if (s.ok()) { // commit new state
-        Log(info_log, "Level-0 commit table #%llu: done %s",
-                       (unsigned long long)m->file_number_,
-                       first ? "": "bulk");
+        Log(info_log,
+            "Level-0 commit table #%lu: memtable #%lu done",
+            m->file_number_,
+            mem_id);
        memlist_.remove(m);
        assert(m->file_number_ > 0);

@@ -150,8 +148,10 @@ Status MemTableList::InstallMemtableFlushResults(
        size_--;
      } else {
        //commit failed. setup state so that we can flush again.
-        Log(info_log, "Level-0 commit table #%llu: failed",
-                       (unsigned long long)m->file_number_);
+        Log(info_log,
+            "Level-0 commit table #%lu: memtable #%lu failed",
+            m->file_number_,
+            mem_id);
        m->flush_completed_ = false;
        m->flush_in_progress_ = false;
        m->edit_.Clear();
@@ -161,7 +161,7 @@ Status MemTableList::InstallMemtableFlushResults(
        imm_flush_needed.Release_Store((void *)1);
        s = Status::IOError("Unable to commit flushed memtable");
      }
-      first = false;
+      ++mem_id;
    } while (!memlist_.empty() && (m = memlist_.back()) &&
             m->file_number_ == file_number);
  }
@@ -1,8 +1,10 @@
-// Copyright (c) 2012 Facebook.
-
-#ifndef STORAGE_LEVELDB_DB_MEMTABLELIST_H_
-#define STORAGE_LEVELDB_DB_MEMTABLELIST_H_
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//

+#pragma once
 #include <string>
 #include <list>
 #include <deque>
@@ -52,7 +54,8 @@ class MemTableList {
  // not yet started.
  bool IsFlushPending(int min_write_buffer_number_to_merge);

-  // Returns the earliest memtables that needs to be flushed.
+  // Returns the earliest memtables that needs to be flushed. The returned
+  // memtables are guaranteed to be in the ascending order of created time.
  void PickMemtablesToFlush(std::vector<MemTable*>* mems);

  // Commit a successful flush in the manifest file
@@ -100,5 +103,3 @@ class MemTableList {
 };

 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_MEMTABLELIST_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 #include "merge_helper.h"
 #include "db/dbformat.h"
 #include "rocksdb/comparator.h"
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 #ifndef MERGE_HELPER_H
 #define MERGE_HELPER_H

@@ -1,8 +1,10 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 /**
 * Back-end implementation details specific to the Merge Operator.
- *
- * @author Deon Nicholas (dnicholas@fb.com)
- * Copyright 2013 Facebook
 */

 #include "rocksdb/merge_operator.h"
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 #include <assert.h>
 #include <memory>
 #include <iostream>
@@ -85,7 +90,7 @@ class Counters {
    }
  }

-  // mapped to a leveldb Delete
+  // mapped to a rocksdb Delete
  bool remove(const string& key) {
    auto s = db_->Delete(delete_option_, key);

@@ -97,7 +102,7 @@ class Counters {
    }
  }

-  // mapped to a leveldb Get
+  // mapped to a rocksdb Get
  bool get(const string& key, uint64_t *value) {
    string str;
    auto s = db_->Get(get_option_, key, &str);
@@ -159,7 +164,7 @@ class MergeBasedCounters : public Counters {
        merge_option_() {
  }

-  // mapped to a leveldb Merge operation
+  // mapped to a rocksdb Merge operation
  virtual bool add(const string& key, uint64_t value) override {
    char encoded[sizeof(uint64_t)];
    EncodeFixed64(encoded, value);
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 #include <algorithm>
 #include <iostream>
 #include <vector>
@@ -20,16 +25,6 @@ int FLAGS_min_write_buffer_number_to_merge = 7;
 // Path to the database on file system
 const std::string kDbName = rocksdb::test::TmpDir() + "/perf_context_test";

-void SeekToFirst(rocksdb::Iterator* iter) {
-//  std::cout << "Press a key to continue:";
-//  std::string s;
-//  std::cin >> s;
-  iter->SeekToFirst();
-//  std::cout << "Press a key to continue:";
-//  std::string s2;
-//  std::cin >> s2;
-}
-
 namespace rocksdb {

 std::shared_ptr<DB> OpenDb() {
@@ -95,10 +90,7 @@ TEST(PerfContextTest, SeekIntoDeletion) {

  perf_context.Reset();
  StopWatchNano timer(Env::Default(), true);
-  //CALLGRIND_ZERO_STATS;
-  SeekToFirst(iter.get());
-  //iter->SeekToFirst();
-  //CALLGRIND_DUMP_STATS;
+  iter->SeekToFirst();
  hist_seek_to_first.Add(perf_context.user_key_comparison_count);
  auto elapsed_nanos = timer.ElapsedNanos();

@@ -224,7 +216,7 @@ TEST(PerfContextTest, KeyComparisonCount) {
 }

 // make perf_context_test
-// export LEVELDB_TESTS=PerfContextTest.SeekKeyComparison
+// export ROCKSDB_TESTS=PerfContextTest.SeekKeyComparison
 // For one memtable:
 // ./perf_context_test --write_buffer_size=500000 --total_keys=10000
 // For two memtables:
@@ -252,13 +244,29 @@ TEST(PerfContextTest, SeekKeyComparison) {
    std::random_shuffle(keys.begin(), keys.end());
  }

+  HistogramImpl hist_put_time;
+  HistogramImpl hist_wal_time;
+  HistogramImpl hist_time_diff;
+
+  SetPerfLevel(kEnableTime);
+  StopWatchNano timer(Env::Default());
  for (const int i : keys) {
    std::string key = "k" + std::to_string(i);
    std::string value = "v" + std::to_string(i);

+    perf_context.Reset();
+    timer.Start();
    db->Put(write_options, key, value);
+    auto put_time = timer.ElapsedNanos();
+    hist_put_time.Add(put_time);
+    hist_wal_time.Add(perf_context.wal_write_time);
+    hist_time_diff.Add(put_time - perf_context.wal_write_time);
  }

+  std::cout << "Put time:\n" << hist_put_time.ToString()
+            << "WAL time:\n" << hist_wal_time.ToString()
+            << "time diff:\n" << hist_time_diff.ToString();
+
  HistogramImpl hist_seek;
  HistogramImpl hist_next;

@@ -1,6 +1,7 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Wrap an underlying iterator, but exclude any results not starting
 // with a given prefix.  Seeking to keys not beginning with the prefix
@@ -9,9 +10,7 @@
 // underlying iterator would if there happened to be no non-matching
 // keys in the dataset.

-#ifndef STORAGE_LEVELDB_DB_PREFIX_FILTER_ITERATOR_H_
-#define STORAGE_LEVELDB_DB_PREFIX_FILTER_ITERATOR_H_
-
+#pragma once
 #include "rocksdb/iterator.h"

 namespace rocksdb {
@@ -72,5 +71,3 @@ class PrefixFilterIterator : public Iterator {
 };

 }  // namespace rocksdb
-
-#endif
@@ -0,0 +1,320 @@
+#include <algorithm>
+#include <iostream>
+#include <vector>
+
+#include <gflags/gflags.h>
+#include "rocksdb/comparator.h"
+#include "rocksdb/db.h"
+#include "rocksdb/perf_context.h"
+#include "util/histogram.h"
+#include "util/stop_watch.h"
+#include "util/testharness.h"
+
+DEFINE_bool(use_prefix_hash_memtable, true, "");
+DEFINE_bool(use_nolock_version, true, "");
+DEFINE_bool(trigger_deadlock, false,
+            "issue delete in range scan to trigger PrefixHashMap deadlock");
+DEFINE_uint64(bucket_count, 100000, "number of buckets");
+DEFINE_uint64(num_locks, 10001, "number of locks");
+DEFINE_bool(random_prefix, false, "randomize prefix");
+DEFINE_uint64(total_prefixes, 1000, "total number of prefixes");
+DEFINE_uint64(items_per_prefix, 10, "total number of values per prefix");
+DEFINE_int64(write_buffer_size, 1000000000, "");
+DEFINE_int64(max_write_buffer_number, 8, "");
+DEFINE_int64(min_write_buffer_number_to_merge, 7, "");
+
+// Path to the database on file system
+const std::string kDbName = rocksdb::test::TmpDir() + "/prefix_test";
+
+namespace rocksdb {
+
+struct TestKey {
+  uint64_t prefix;
+  uint64_t sorted;
+
+  TestKey(uint64_t prefix, uint64_t sorted) : prefix(prefix), sorted(sorted) {}
+};
+
+// return a slice backed by test_key
+inline Slice TestKeyToSlice(const TestKey& test_key) {
+  return Slice((const char*)&test_key, sizeof(test_key));
+}
+
+inline const TestKey* SliceToTestKey(const Slice& slice) {
+  assert(slice.size() == sizeof(TestKey));
+  return (const TestKey*)slice.data();
+}
+
+class TestKeyComparator : public Comparator {
+ public:
+  virtual int Compare(const Slice& a, const Slice& b) const {
+    const TestKey* key_a = SliceToTestKey(a);
+    const TestKey* key_b = SliceToTestKey(b);
+    if (key_a->prefix != key_b->prefix) {
+      if (key_a->prefix < key_b->prefix) return -1;
+      if (key_a->prefix > key_b->prefix) return 1;
+      assert(false);
+    } else {
+      if (key_a->sorted < key_b->sorted) return -1;
+      if (key_a->sorted > key_b->sorted) return 1;
+      if (key_a->sorted == key_b->sorted) return 0;
+      assert(false);
+    }
+    assert(false);
+    return 0;
+  }
+
+  virtual const char* Name() const override {
+    return "TestKeyComparator";
+  }
+
+  virtual void FindShortestSeparator(
+      std::string* start,
+      const Slice& limit) const {
+  }
+
+  virtual void FindShortSuccessor(std::string* key) const {}
+
+};
+
+class PrefixTest {
+ public:
+  std::shared_ptr<DB> OpenDb() {
+    DB* db;
+
+    options.create_if_missing = true;
+    options.write_buffer_size = FLAGS_write_buffer_size;
+    options.max_write_buffer_number = FLAGS_max_write_buffer_number;
+    options.min_write_buffer_number_to_merge =
+      FLAGS_min_write_buffer_number_to_merge;
+
+    options.comparator = new TestKeyComparator();
+    if (FLAGS_use_prefix_hash_memtable) {
+      auto prefix_extractor = NewFixedPrefixTransform(8);
+      options.prefix_extractor = prefix_extractor;
+      if (FLAGS_use_nolock_version) {
+        options.memtable_factory =
+          std::make_shared<rocksdb::PrefixHashRepNoLockFactory>(
+            prefix_extractor, FLAGS_bucket_count);
+      } else {
+        options.memtable_factory =
+          std::make_shared<rocksdb::PrefixHashRepFactory>(
+            prefix_extractor, FLAGS_bucket_count, FLAGS_num_locks);
+      }
+    }
+
+    Status s = DB::Open(options, kDbName,  &db);
+    ASSERT_OK(s);
+    return std::shared_ptr<DB>(db);
+  }
+  ~PrefixTest() {
+    delete options.comparator;
+  }
+ protected:
+  Options options;
+};
+
+TEST(PrefixTest, DynamicPrefixIterator) {
+
+  DestroyDB(kDbName, Options());
+  auto db = OpenDb();
+  WriteOptions write_options;
+  ReadOptions read_options;
+
+  std::vector<uint64_t> prefixes;
+  for (uint64_t i = 0; i < FLAGS_total_prefixes; ++i) {
+    prefixes.push_back(i);
+  }
+
+  if (FLAGS_random_prefix) {
+    std::random_shuffle(prefixes.begin(), prefixes.end());
+  }
+
+  // insert x random prefix, each with y continuous element.
+  for (auto prefix : prefixes) {
+     for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) {
+      TestKey test_key(prefix, sorted);
+
+      Slice key = TestKeyToSlice(test_key);
+      std::string value = "v" + std::to_string(sorted);
+
+      ASSERT_OK(db->Put(write_options, key, value));
+    }
+  }
+
+  // test seek existing keys
+  HistogramImpl hist_seek_time;
+  HistogramImpl hist_seek_comparison;
+
+  if (FLAGS_use_prefix_hash_memtable) {
+    read_options.prefix_seek = true;
+  }
+  std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
+
+  for (auto prefix : prefixes) {
+    TestKey test_key(prefix, FLAGS_items_per_prefix / 2);
+    Slice key = TestKeyToSlice(test_key);
+    std::string value = "v" + std::to_string(0);
+
+    perf_context.Reset();
+    StopWatchNano timer(Env::Default(), true);
+    uint64_t total_keys = 0;
+    for (iter->Seek(key); iter->Valid(); iter->Next()) {
+      if (FLAGS_trigger_deadlock) {
+        std::cout << "Behold the deadlock!\n";
+        db->Delete(write_options, iter->key());
+      }
+      auto test_key = SliceToTestKey(iter->key());
+      if (test_key->prefix != prefix) break;
+      total_keys++;
+    }
+    hist_seek_time.Add(timer.ElapsedNanos());
+    hist_seek_comparison.Add(perf_context.user_key_comparison_count);
+    ASSERT_EQ(total_keys, FLAGS_items_per_prefix - FLAGS_items_per_prefix/2);
+  }
+
+  std::cout << "Seek key comparison: \n"
+            << hist_seek_comparison.ToString()
+            << "Seek time: \n"
+            << hist_seek_time.ToString();
+
+  // test non-existing keys
+  HistogramImpl hist_no_seek_time;
+  HistogramImpl hist_no_seek_comparison;
+
+  for (auto prefix = FLAGS_total_prefixes;
+       prefix < FLAGS_total_prefixes + 100;
+       prefix++) {
+    TestKey test_key(prefix, 0);
+    Slice key = TestKeyToSlice(test_key);
+
+    perf_context.Reset();
+    StopWatchNano timer(Env::Default(), true);
+    iter->Seek(key);
+    hist_no_seek_time.Add(timer.ElapsedNanos());
+    hist_no_seek_comparison.Add(perf_context.user_key_comparison_count);
+    ASSERT_TRUE(!iter->Valid());
+  }
+
+  std::cout << "non-existing Seek key comparison: \n"
+            << hist_no_seek_comparison.ToString()
+            << "non-existing Seek time: \n"
+            << hist_no_seek_time.ToString();
+}
+
+TEST(PrefixTest, PrefixHash) {
+
+  DestroyDB(kDbName, Options());
+  auto db = OpenDb();
+  WriteOptions write_options;
+  ReadOptions read_options;
+
+  std::vector<uint64_t> prefixes;
+  for (uint64_t i = 0; i < FLAGS_total_prefixes; ++i) {
+    prefixes.push_back(i);
+  }
+
+  if (FLAGS_random_prefix) {
+    std::random_shuffle(prefixes.begin(), prefixes.end());
+  }
+
+  // insert x random prefix, each with y continuous element.
+  HistogramImpl hist_put_time;
+  HistogramImpl hist_put_comparison;
+
+  for (auto prefix : prefixes) {
+     for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) {
+      TestKey test_key(prefix, sorted);
+
+      Slice key = TestKeyToSlice(test_key);
+      std::string value = "v" + std::to_string(sorted);
+
+      perf_context.Reset();
+      StopWatchNano timer(Env::Default(), true);
+      ASSERT_OK(db->Put(write_options, key, value));
+      hist_put_time.Add(timer.ElapsedNanos());
+      hist_put_comparison.Add(perf_context.user_key_comparison_count);
+    }
+  }
+
+  std::cout << "Put key comparison: \n" << hist_put_comparison.ToString()
+            << "Put time: \n" << hist_put_time.ToString();
+
+
+  // test seek existing keys
+  HistogramImpl hist_seek_time;
+  HistogramImpl hist_seek_comparison;
+
+  for (auto prefix : prefixes) {
+    TestKey test_key(prefix, 0);
+    Slice key = TestKeyToSlice(test_key);
+    std::string value = "v" + std::to_string(0);
+
+    Slice key_prefix;
+    if (FLAGS_use_prefix_hash_memtable) {
+      key_prefix = options.prefix_extractor->Transform(key);
+      read_options.prefix = &key_prefix;
+    }
+    std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
+
+    perf_context.Reset();
+    StopWatchNano timer(Env::Default(), true);
+    uint64_t total_keys = 0;
+    for (iter->Seek(key); iter->Valid(); iter->Next()) {
+      if (FLAGS_trigger_deadlock) {
+        std::cout << "Behold the deadlock!\n";
+        db->Delete(write_options, iter->key());
+      }
+      auto test_key = SliceToTestKey(iter->key());
+      if (test_key->prefix != prefix) break;
+      total_keys++;
+    }
+    hist_seek_time.Add(timer.ElapsedNanos());
+    hist_seek_comparison.Add(perf_context.user_key_comparison_count);
+    ASSERT_EQ(total_keys, FLAGS_items_per_prefix);
+  }
+
+  std::cout << "Seek key comparison: \n"
+            << hist_seek_comparison.ToString()
+            << "Seek time: \n"
+            << hist_seek_time.ToString();
+
+  // test non-existing keys
+  HistogramImpl hist_no_seek_time;
+  HistogramImpl hist_no_seek_comparison;
+
+  for (auto prefix = FLAGS_total_prefixes;
+       prefix < FLAGS_total_prefixes + 100;
+       prefix++) {
+    TestKey test_key(prefix, 0);
+    Slice key = TestKeyToSlice(test_key);
+
+    if (FLAGS_use_prefix_hash_memtable) {
+      Slice key_prefix = options.prefix_extractor->Transform(key);
+      read_options.prefix = &key_prefix;
+    }
+    std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
+
+    perf_context.Reset();
+    StopWatchNano timer(Env::Default(), true);
+    iter->Seek(key);
+    hist_no_seek_time.Add(timer.ElapsedNanos());
+    hist_no_seek_comparison.Add(perf_context.user_key_comparison_count);
+    ASSERT_TRUE(!iter->Valid());
+  }
+
+  std::cout << "non-existing Seek key comparison: \n"
+            << hist_no_seek_comparison.ToString()
+            << "non-existing Seek time: \n"
+            << hist_no_seek_time.ToString();
+}
+
+}
+
+int main(int argc, char** argv) {
+  google::ParseCommandLineFlags(&argc, &argv, true);
+  std::cout << kDbName << "\n";
+
+  rocksdb::test::RunAllTests();
+  return 0;
+}
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -74,7 +79,7 @@ class Repairer {
        bytes += tables_[i].meta.file_size;
      }
      Log(options_.info_log,
-          "**** Repaired leveldb %s; "
+          "**** Repaired rocksdb %s; "
          "recovered %d files; %llu bytes. "
          "Some data may have been lost. "
          "****",
@@ -203,7 +208,7 @@ class Repairer {
        continue;
      }
      WriteBatchInternal::SetContents(&batch, record);
-      status = WriteBatchInternal::InsertInto(&batch, mem);
+      status = WriteBatchInternal::InsertInto(&batch, mem, &options_);
      if (status.ok()) {
        counter += WriteBatchInternal::Count(&batch);
      } else {
@@ -221,7 +226,7 @@ class Repairer {
    Iterator* iter = mem->NewIterator();
    status = BuildTable(dbname_, env_, options_, storage_options_,
                        table_cache_, iter, &meta,
-                        icmp_.user_comparator(), 0, 0);
+                        icmp_.user_comparator(), 0, 0, true);
    delete iter;
    mem->Unref();
    mem = nullptr;
@@ -0,0 +1,793 @@
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include <algorithm>
+#include <set>
+
+#include "rocksdb/db.h"
+#include "rocksdb/filter_policy.h"
+#include "db/db_impl.h"
+#include "db/filename.h"
+#include "db/version_set.h"
+#include "db/write_batch_internal.h"
+#include "db/db_statistics.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/env.h"
+#include "rocksdb/table.h"
+#include "util/hash.h"
+#include "util/logging.h"
+#include "util/mutexlock.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+#include "utilities/merge_operators.h"
+
+using std::unique_ptr;
+
+namespace rocksdb {
+
+// SimpleTable is a simple table format for UNIT TEST ONLY. It is not built
+// as production quality.
+// SimpleTable requires the input key size to be fixed 16 bytes, value cannot
+// be longer than 150000 bytes and stored data on disk in this format:
+// +--------------------------------------------+  <= key1 offset
+// | key1            | value_size (4 bytes) |   |
+// +----------------------------------------+   |
+// | value1                                     |
+// |                                            |
+// +----------------------------------------+---+  <= key2 offset
+// | key2            | value_size (4 bytes) |   |
+// +----------------------------------------+   |
+// | value2                                     |
+// |                                            |
+// |        ......                              |
+// +-----------------+--------------------------+   <= index_block_offset
+// | key1            | key1 offset (8 bytes)    |
+// +-----------------+--------------------------+
+// | key2            | key2 offset (8 bytes)    |
+// +-----------------+--------------------------+
+// | key3            | key3 offset (8 bytes)    |
+// +-----------------+--------------------------+
+// |        ......                              |
+// +-----------------+------------+-------------+
+// | index_block_offset (8 bytes) |
+// +------------------------------+
+
+// SimpleTable is a simple table format for UNIT TEST ONLY. It is not built
+// as production quality.
+class SimpleTableReader: public TableReader {
+public:
+  // Attempt to open the table that is stored in bytes [0..file_size)
+  // of "file", and read the metadata entries necessary to allow
+  // retrieving data from the table.
+  //
+  // If successful, returns ok and sets "*table" to the newly opened
+  // table.  The client should delete "*table" when no longer needed.
+  // If there was an error while initializing the table, sets "*table"
+  // to nullptr and returns a non-ok status.  Does not take ownership of
+  // "*source", but the client must ensure that "source" remains live
+  // for the duration of the returned table's lifetime.
+  //
+  // *file must remain live while this Table is in use.
+  static Status Open(const Options& options, const EnvOptions& soptions,
+                     unique_ptr<RandomAccessFile> && file, uint64_t file_size,
+                     unique_ptr<TableReader>* table_reader);
+
+  bool PrefixMayMatch(const Slice& internal_prefix) override;
+
+  Iterator* NewIterator(const ReadOptions&) override;
+
+  Status Get(
+      const ReadOptions&, const Slice& key, void* arg,
+      bool (*handle_result)(void* arg, const Slice& k, const Slice& v, bool),
+      void (*mark_key_may_exist)(void*) = nullptr) override;
+
+  uint64_t ApproximateOffsetOf(const Slice& key) override;
+
+  bool TEST_KeyInCache(const ReadOptions& options, const Slice& key) override;
+
+  void SetupForCompaction() override;
+
+  TableStats& GetTableStats() override;
+
+  ~SimpleTableReader();
+
+private:
+  struct Rep;
+  Rep* rep_;
+
+  explicit SimpleTableReader(Rep* rep) {
+    rep_ = rep;
+  }
+  friend class TableCache;
+  friend class SimpleTableIterator;
+
+  Status GetOffset(const Slice& target, uint64_t* offset);
+
+  // No copying allowed
+  explicit SimpleTableReader(const TableReader&) = delete;
+  void operator=(const TableReader&) = delete;
+};
+
+// Iterator to iterate SimpleTable
+class SimpleTableIterator: public Iterator {
+public:
+  explicit SimpleTableIterator(SimpleTableReader* table);
+  ~SimpleTableIterator();
+
+  bool Valid() const;
+
+  void SeekToFirst();
+
+  void SeekToLast();
+
+  void Seek(const Slice& target);
+
+  void Next();
+
+  void Prev();
+
+  Slice key() const;
+
+  Slice value() const;
+
+  Status status() const;
+
+private:
+  SimpleTableReader* table_;
+  uint64_t offset_;
+  uint64_t next_offset_;
+  Slice key_;
+  Slice value_;
+  char tmp_str_[4];
+  char* key_str_;
+  char* value_str_;
+  int value_str_len_;
+  Status status_;
+  // No copying allowed
+  SimpleTableIterator(const SimpleTableIterator&) = delete;
+  void operator=(const Iterator&) = delete;
+};
+
+struct SimpleTableReader::Rep {
+  ~Rep() {
+  }
+  Rep(const EnvOptions& storage_options, uint64_t index_start_offset,
+      int num_entries) :
+      soptions(storage_options), index_start_offset(index_start_offset),
+      num_entries(num_entries) {
+  }
+
+  Options options;
+  const EnvOptions& soptions;
+  Status status;
+  unique_ptr<RandomAccessFile> file;
+  uint64_t index_start_offset;
+  int num_entries;
+  TableStats table_stats;
+
+  const static int user_key_size = 16;
+  const static int offset_length = 8;
+  const static int key_footer_len = 8;
+
+  static int GetInternalKeyLength() {
+    return user_key_size + key_footer_len;
+  }
+};
+
+SimpleTableReader::~SimpleTableReader() {
+  delete rep_;
+}
+
+Status SimpleTableReader::Open(const Options& options,
+                               const EnvOptions& soptions,
+                               unique_ptr<RandomAccessFile> && file,
+                               uint64_t size,
+                               unique_ptr<TableReader>* table_reader) {
+  char footer_space[Rep::offset_length];
+  Slice footer_input;
+  Status s = file->Read(size - Rep::offset_length, Rep::offset_length,
+                        &footer_input, footer_space);
+  if (s.ok()) {
+    uint64_t index_start_offset = DecodeFixed64(footer_space);
+
+    int num_entries = (size - Rep::offset_length - index_start_offset)
+        / (Rep::GetInternalKeyLength() + Rep::offset_length);
+    SimpleTableReader::Rep* rep = new SimpleTableReader::Rep(soptions,
+                                                             index_start_offset,
+                                                             num_entries);
+
+    rep->file = std::move(file);
+    rep->options = options;
+    table_reader->reset(new SimpleTableReader(rep));
+  }
+  return s;
+}
+
+void SimpleTableReader::SetupForCompaction() {
+}
+
+TableStats& SimpleTableReader::GetTableStats() {
+  return rep_->table_stats;
+}
+
+bool SimpleTableReader::PrefixMayMatch(const Slice& internal_prefix) {
+  return true;
+}
+
+Iterator* SimpleTableReader::NewIterator(const ReadOptions& options) {
+  return new SimpleTableIterator(this);
+}
+
+Status SimpleTableReader::GetOffset(const Slice& target, uint64_t* offset) {
+  uint32_t left = 0;
+  uint32_t right = rep_->num_entries - 1;
+  char key_chars[Rep::GetInternalKeyLength()];
+  Slice tmp_slice;
+
+  uint32_t target_offset = 0;
+  while (left <= right) {
+    uint32_t mid = (left + right + 1) / 2;
+
+    uint64_t offset_to_read = rep_->index_start_offset
+        + (Rep::GetInternalKeyLength() + Rep::offset_length) * mid;
+    Status s = rep_->file->Read(offset_to_read, Rep::GetInternalKeyLength(),
+                                &tmp_slice, key_chars);
+    if (!s.ok()) {
+      return s;
+    }
+
+    int compare_result = rep_->options.comparator->Compare(tmp_slice, target);
+
+    if (compare_result < 0) {
+      if (left == right) {
+        target_offset = right + 1;
+        break;
+      }
+      left = mid;
+    } else {
+      if (left == right) {
+        target_offset = left;
+        break;
+      }
+      right = mid - 1;
+    }
+  }
+
+  if (target_offset >= (uint32_t) rep_->num_entries) {
+    *offset = rep_->index_start_offset;
+    return Status::OK();
+  }
+
+  char value_offset_chars[Rep::offset_length];
+
+  int64_t offset_for_value_offset = rep_->index_start_offset
+      + (Rep::GetInternalKeyLength() + Rep::offset_length) * target_offset
+      + Rep::GetInternalKeyLength();
+  Status s = rep_->file->Read(offset_for_value_offset, Rep::offset_length,
+                              &tmp_slice, value_offset_chars);
+  if (s.ok()) {
+    *offset = DecodeFixed64(value_offset_chars);
+  }
+  return s;
+}
+
+Status SimpleTableReader::Get(
+    const ReadOptions& options, const Slice& k, void* arg,
+    bool (*saver)(void*, const Slice&, const Slice&, bool),
+    void (*mark_key_may_exist)(void*)) {
+  Status s;
+  SimpleTableIterator* iter = new SimpleTableIterator(this);
+  for (iter->Seek(k); iter->Valid(); iter->Next()) {
+    if (!(*saver)(arg, iter->key(), iter->value(), true)) {
+      break;
+    }
+  }
+  s = iter->status();
+  delete iter;
+  return s;
+}
+
+bool SimpleTableReader::TEST_KeyInCache(const ReadOptions& options,
+                                        const Slice& key) {
+  return false;
+}
+
+uint64_t SimpleTableReader::ApproximateOffsetOf(const Slice& key) {
+  return 0;
+}
+
+SimpleTableIterator::SimpleTableIterator(SimpleTableReader* table) :
+    table_(table) {
+  key_str_ = new char[SimpleTableReader::Rep::GetInternalKeyLength()];
+  value_str_len_ = -1;
+  SeekToFirst();
+}
+
+SimpleTableIterator::~SimpleTableIterator() {
+ delete[] key_str_;
+ if (value_str_len_ >= 0) {
+   delete[] value_str_;
+ }
+}
+
+bool SimpleTableIterator::Valid() const {
+  return offset_ < table_->rep_->index_start_offset && offset_ >= 0;
+}
+
+void SimpleTableIterator::SeekToFirst() {
+  next_offset_ = 0;
+  Next();
+}
+
+void SimpleTableIterator::SeekToLast() {
+  assert(false);
+}
+
+void SimpleTableIterator::Seek(const Slice& target) {
+  Status s = table_->GetOffset(target, &next_offset_);
+  if (!s.ok()) {
+    status_ = s;
+  }
+  Next();
+}
+
+void SimpleTableIterator::Next() {
+  offset_ = next_offset_;
+  if (offset_ >= table_->rep_->index_start_offset) {
+    return;
+  }
+  Slice result;
+  int internal_key_size = SimpleTableReader::Rep::GetInternalKeyLength();
+
+  Status s = table_->rep_->file->Read(next_offset_, internal_key_size, &result,
+                                      key_str_);
+  next_offset_ += internal_key_size;
+  key_ = result;
+
+  Slice value_size_slice;
+  s = table_->rep_->file->Read(next_offset_, 4, &value_size_slice, tmp_str_);
+  next_offset_ += 4;
+  uint32_t value_size = DecodeFixed32(tmp_str_);
+
+  Slice value_slice;
+  if ((int) value_size > value_str_len_) {
+    if (value_str_len_ >= 0) {
+      delete[] value_str_;
+    }
+    value_str_ = new char[value_size];
+    value_str_len_ = value_size;
+  }
+  s = table_->rep_->file->Read(next_offset_, value_size, &value_slice,
+                               value_str_);
+  next_offset_ += value_size;
+  value_ = value_slice;
+}
+
+void SimpleTableIterator::Prev() {
+  assert(false);
+}
+
+Slice SimpleTableIterator::key() const {
+  Log(table_->rep_->options.info_log, "key!!!!");
+  return key_;
+}
+
+Slice SimpleTableIterator::value() const {
+  return value_;
+}
+
+Status SimpleTableIterator::status() const {
+  return status_;
+}
+
+class SimpleTableBuilder: public TableBuilder {
+public:
+  // Create a builder that will store the contents of the table it is
+  // building in *file.  Does not close the file.  It is up to the
+  // caller to close the file after calling Finish(). The output file
+  // will be part of level specified by 'level'.  A value of -1 means
+  // that the caller does not know which level the output file will reside.
+  SimpleTableBuilder(const Options& options, WritableFile* file,
+                     CompressionType compression_type);
+
+  // REQUIRES: Either Finish() or Abandon() has been called.
+  ~SimpleTableBuilder();
+
+  // Add key,value to the table being constructed.
+  // REQUIRES: key is after any previously added key according to comparator.
+  // REQUIRES: Finish(), Abandon() have not been called
+  void Add(const Slice& key, const Slice& value) override;
+
+  // Return non-ok iff some error has been detected.
+  Status status() const override;
+
+  // Finish building the table.  Stops using the file passed to the
+  // constructor after this function returns.
+  // REQUIRES: Finish(), Abandon() have not been called
+  Status Finish() override;
+
+  // Indicate that the contents of this builder should be abandoned.  Stops
+  // using the file passed to the constructor after this function returns.
+  // If the caller is not going to call Finish(), it must call Abandon()
+  // before destroying this builder.
+  // REQUIRES: Finish(), Abandon() have not been called
+  void Abandon() override;
+
+  // Number of calls to Add() so far.
+  uint64_t NumEntries() const override;
+
+  // Size of the file generated so far.  If invoked after a successful
+  // Finish() call, returns the size of the final generated file.
+  uint64_t FileSize() const override;
+
+private:
+  struct Rep;
+  Rep* rep_;
+
+  // No copying allowed
+  SimpleTableBuilder(const SimpleTableBuilder&) = delete;
+  void operator=(const SimpleTableBuilder&) = delete;
+};
+
+struct SimpleTableBuilder::Rep {
+  Options options;
+  WritableFile* file;
+  uint64_t offset = 0;
+  Status status;
+
+  uint64_t num_entries = 0;
+
+  bool closed = false;  // Either Finish() or Abandon() has been called.
+
+  const static int user_key_size = 16;
+  const static int offset_length = 8;
+  const static int key_footer_len = 8;
+
+  static int GetInternalKeyLength() {
+    return user_key_size + key_footer_len;
+  }
+
+  std::string index;
+
+  Rep(const Options& opt, WritableFile* f) :
+      options(opt), file(f) {
+  }
+  ~Rep() {
+  }
+};
+
+SimpleTableBuilder::SimpleTableBuilder(const Options& options,
+                                       WritableFile* file,
+                                       CompressionType compression_type) :
+    rep_(new SimpleTableBuilder::Rep(options, file)) {
+}
+
+SimpleTableBuilder::~SimpleTableBuilder() {
+  delete (rep_);
+}
+
+void SimpleTableBuilder::Add(const Slice& key, const Slice& value) {
+  assert((int ) key.size() == Rep::GetInternalKeyLength());
+
+  // Update index
+  rep_->index.append(key.data(), key.size());
+  PutFixed64(&(rep_->index), rep_->offset);
+
+  // Write key-value pair
+  rep_->file->Append(key);
+  rep_->offset += Rep::GetInternalKeyLength();
+
+  std::string size;
+  int value_size = value.size();
+  PutFixed32(&size, value_size);
+  Slice sizeSlice(size);
+  rep_->file->Append(sizeSlice);
+  rep_->file->Append(value);
+  rep_->offset += value_size + 4;
+
+  rep_->num_entries++;
+}
+
+Status SimpleTableBuilder::status() const {
+  return Status::OK();
+}
+
+Status SimpleTableBuilder::Finish() {
+  Rep* r = rep_;
+  assert(!r->closed);
+  r->closed = true;
+
+  uint64_t index_offset = rep_->offset;
+  Slice index_slice(rep_->index);
+  rep_->file->Append(index_slice);
+  rep_->offset += index_slice.size();
+
+  std::string index_offset_str;
+  PutFixed64(&index_offset_str, index_offset);
+  Slice foot_slice(index_offset_str);
+  rep_->file->Append(foot_slice);
+  rep_->offset += foot_slice.size();
+
+  return Status::OK();
+}
+
+void SimpleTableBuilder::Abandon() {
+  rep_->closed = true;
+}
+
+uint64_t SimpleTableBuilder::NumEntries() const {
+  return rep_->num_entries;
+}
+
+uint64_t SimpleTableBuilder::FileSize() const {
+  return rep_->offset;
+}
+
+class SimpleTableFactory: public TableFactory {
+public:
+  ~SimpleTableFactory() {
+  }
+  SimpleTableFactory() {
+  }
+  const char* Name() const override {
+    return "SimpleTable";
+  }
+  Status GetTableReader(const Options& options, const EnvOptions& soptions,
+                        unique_ptr<RandomAccessFile> && file,
+                        uint64_t file_size,
+                        unique_ptr<TableReader>* table_reader) const;
+
+  TableBuilder* GetTableBuilder(const Options& options, WritableFile* file,
+                                CompressionType compression_type) const;
+};
+
+Status SimpleTableFactory::GetTableReader(
+    const Options& options, const EnvOptions& soptions,
+    unique_ptr<RandomAccessFile> && file, uint64_t file_size,
+    unique_ptr<TableReader>* table_reader) const {
+
+  return SimpleTableReader::Open(options, soptions, std::move(file), file_size,
+                                 table_reader);
+}
+
+TableBuilder* SimpleTableFactory::GetTableBuilder(
+    const Options& options, WritableFile* file,
+    CompressionType compression_type) const {
+  return new SimpleTableBuilder(options, file, compression_type);
+}
+
+class SimpleTableDBTest {
+protected:
+public:
+  std::string dbname_;
+  Env* env_;
+  DB* db_;
+
+  Options last_options_;
+
+  SimpleTableDBTest() :
+      env_(Env::Default()) {
+    dbname_ = test::TmpDir() + "/simple_table_db_test";
+    ASSERT_OK(DestroyDB(dbname_, Options()));
+    db_ = nullptr;
+    Reopen();
+  }
+
+  ~SimpleTableDBTest() {
+    delete db_;
+    ASSERT_OK(DestroyDB(dbname_, Options()));
+  }
+
+  // Return the current option configuration.
+  Options CurrentOptions() {
+    Options options;
+    options.table_factory.reset(new SimpleTableFactory());
+    return options;
+  }
+
+  DBImpl* dbfull() {
+    return reinterpret_cast<DBImpl*>(db_);
+  }
+
+  void Reopen(Options* options = nullptr) {
+    ASSERT_OK(TryReopen(options));
+  }
+
+  void Close() {
+    delete db_;
+    db_ = nullptr;
+  }
+
+  void DestroyAndReopen(Options* options = nullptr) {
+    //Destroy using last options
+    Destroy(&last_options_);
+    ASSERT_OK(TryReopen(options));
+  }
+
+  void Destroy(Options* options) {
+    delete db_;
+    db_ = nullptr;
+    ASSERT_OK(DestroyDB(dbname_, *options));
+  }
+
+  Status PureReopen(Options* options, DB** db) {
+    return DB::Open(*options, dbname_, db);
+  }
+
+  Status TryReopen(Options* options = nullptr) {
+    delete db_;
+    db_ = nullptr;
+    Options opts;
+    if (options != nullptr) {
+      opts = *options;
+    } else {
+      opts = CurrentOptions();
+      opts.create_if_missing = true;
+    }
+    last_options_ = opts;
+
+    return DB::Open(opts, dbname_, &db_);
+  }
+
+  Status Put(const Slice& k, const Slice& v) {
+    return db_->Put(WriteOptions(), k, v);
+  }
+
+  Status Delete(const std::string& k) {
+    return db_->Delete(WriteOptions(), k);
+  }
+
+  std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
+    ReadOptions options;
+    options.snapshot = snapshot;
+    std::string result;
+    Status s = db_->Get(options, k, &result);
+    if (s.IsNotFound()) {
+      result = "NOT_FOUND";
+    } else if (!s.ok()) {
+      result = s.ToString();
+    }
+    return result;
+  }
+
+
+  int NumTableFilesAtLevel(int level) {
+    std::string property;
+    ASSERT_TRUE(
+        db_->GetProperty("rocksdb.num-files-at-level" + NumberToString(level),
+                         &property));
+    return atoi(property.c_str());
+  }
+
+  // Return spread of files per level
+  std::string FilesPerLevel() {
+    std::string result;
+    int last_non_zero_offset = 0;
+    for (int level = 0; level < db_->NumberLevels(); level++) {
+      int f = NumTableFilesAtLevel(level);
+      char buf[100];
+      snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
+      result += buf;
+      if (f > 0) {
+        last_non_zero_offset = result.size();
+      }
+    }
+    result.resize(last_non_zero_offset);
+    return result;
+  }
+
+  std::string IterStatus(Iterator* iter) {
+    std::string result;
+    if (iter->Valid()) {
+      result = iter->key().ToString() + "->" + iter->value().ToString();
+    } else {
+      result = "(invalid)";
+    }
+    return result;
+  }
+};
+
+TEST(SimpleTableDBTest, Empty) {
+  ASSERT_TRUE(db_ != nullptr);
+  ASSERT_EQ("NOT_FOUND", Get("0000000000000foo"));
+}
+
+TEST(SimpleTableDBTest, ReadWrite) {
+  ASSERT_OK(Put("0000000000000foo", "v1"));
+  ASSERT_EQ("v1", Get("0000000000000foo"));
+  ASSERT_OK(Put("0000000000000bar", "v2"));
+  ASSERT_OK(Put("0000000000000foo", "v3"));
+  ASSERT_EQ("v3", Get("0000000000000foo"));
+  ASSERT_EQ("v2", Get("0000000000000bar"));
+}
+
+TEST(SimpleTableDBTest, Flush) {
+  ASSERT_OK(Put("0000000000000foo", "v1"));
+  ASSERT_OK(Put("0000000000000bar", "v2"));
+  ASSERT_OK(Put("0000000000000foo", "v3"));
+  dbfull()->TEST_FlushMemTable();
+  ASSERT_EQ("v3", Get("0000000000000foo"));
+  ASSERT_EQ("v2", Get("0000000000000bar"));
+}
+
+TEST(SimpleTableDBTest, Flush2) {
+  ASSERT_OK(Put("0000000000000bar", "b"));
+  ASSERT_OK(Put("0000000000000foo", "v1"));
+  dbfull()->TEST_FlushMemTable();
+
+  ASSERT_OK(Put("0000000000000foo", "v2"));
+  dbfull()->TEST_FlushMemTable();
+  ASSERT_EQ("v2", Get("0000000000000foo"));
+
+  ASSERT_OK(Put("0000000000000eee", "v3"));
+  dbfull()->TEST_FlushMemTable();
+  ASSERT_EQ("v3", Get("0000000000000eee"));
+
+  ASSERT_OK(Delete("0000000000000bar"));
+  dbfull()->TEST_FlushMemTable();
+  ASSERT_EQ("NOT_FOUND", Get("0000000000000bar"));
+
+  ASSERT_OK(Put("0000000000000eee", "v5"));
+  dbfull()->TEST_FlushMemTable();
+  ASSERT_EQ("v5", Get("0000000000000eee"));
+}
+
+static std::string Key(int i) {
+  char buf[100];
+  snprintf(buf, sizeof(buf), "key_______%06d", i);
+  return std::string(buf);
+}
+
+static std::string RandomString(Random* rnd, int len) {
+  std::string r;
+  test::RandomString(rnd, len, &r);
+  return r;
+}
+
+TEST(SimpleTableDBTest, CompactionTrigger) {
+  Options options = CurrentOptions();
+  options.write_buffer_size = 100 << 10; //100KB
+  options.num_levels = 3;
+  options.max_mem_compaction_level = 0;
+  options.level0_file_num_compaction_trigger = 3;
+  Reopen(&options);
+
+  Random rnd(301);
+
+  for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
+      num++) {
+    std::vector<std::string> values;
+    // Write 120KB (12 values, each 10K)
+    for (int i = 0; i < 12; i++) {
+      values.push_back(RandomString(&rnd, 10000));
+      ASSERT_OK(Put(Key(i), values[i]));
+    }
+    dbfull()->TEST_WaitForFlushMemTable();
+    ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
+  }
+
+  //generate one more file in level-0, and should trigger level-0 compaction
+  std::vector<std::string> values;
+  for (int i = 0; i < 12; i++) {
+    values.push_back(RandomString(&rnd, 10000));
+    ASSERT_OK(Put(Key(i), values[i]));
+  }
+  dbfull()->TEST_WaitForCompact();
+
+  ASSERT_EQ(NumTableFilesAtLevel(0), 0);
+  ASSERT_EQ(NumTableFilesAtLevel(1), 1);
+}
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  return rocksdb::test::RunAllTests();
+}
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -25,9 +30,7 @@
 // ... prev vs. next pointer ordering ...
 //

-#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_
-#define STORAGE_LEVELDB_DB_SKIPLIST_H_
-
+#pragma once
 #include <assert.h>
 #include <stdlib.h>
 #include "port/port.h"
@@ -401,5 +404,3 @@ bool SkipList<Key,Comparator>::Contains(const Key& key) const {
 }

 }  // namespace rocksdb
-
-#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -1,10 +1,13 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_
-#define STORAGE_LEVELDB_DB_SNAPSHOT_H_
-
+#pragma once
 #include "rocksdb/db.h"

 namespace rocksdb {
@@ -81,5 +84,3 @@ class SnapshotList {
 };

 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_SNAPSHOT_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -7,15 +12,15 @@
 #include "db/filename.h"

 #include "rocksdb/statistics.h"
-#include "table/table.h"
+#include "rocksdb/table.h"
 #include "util/coding.h"
 #include "util/stop_watch.h"

 namespace rocksdb {

 static void DeleteEntry(const Slice& key, void* value) {
-  Table* table = reinterpret_cast<Table*>(value);
-  delete table;
+  TableReader* table_reader = reinterpret_cast<TableReader*>(value);
+  delete table_reader;
 }

 static void UnrefEntry(void* arg1, void* arg2) {
@@ -32,7 +37,10 @@ TableCache::TableCache(const std::string& dbname,
      dbname_(dbname),
      options_(options),
      storage_options_(storage_options),
-      cache_(NewLRUCache(entries, options->table_cache_numshardbits)) {}
+      cache_(
+        NewLRUCache(entries, options->table_cache_numshardbits,
+                    options->table_cache_remove_scan_count_limit)) {
+}

 TableCache::~TableCache() {
 }
@@ -55,7 +63,7 @@ Status TableCache::FindTable(const EnvOptions& toptions,
    }
    std::string fname = TableFileName(dbname_, file_number);
    unique_ptr<RandomAccessFile> file;
-    unique_ptr<Table> table;
+    unique_ptr<TableReader> table_reader;
    s = env_->NewRandomAccessFile(fname, &file, toptions);
    RecordTick(options_->statistics, NO_FILE_OPENS);
    if (s.ok()) {
@@ -63,17 +71,19 @@ Status TableCache::FindTable(const EnvOptions& toptions,
        file->Hint(RandomAccessFile::RANDOM);
      }
      StopWatch sw(env_, options_->statistics, TABLE_OPEN_IO_MICROS);
-      s = Table::Open(*options_, toptions, std::move(file), file_size, &table);
+      s = options_->table_factory->GetTableReader(*options_, toptions,
+                                                  std::move(file), file_size,
+                                                  &table_reader);
    }

    if (!s.ok()) {
-      assert(table == nullptr);
+      assert(table_reader == nullptr);
      RecordTick(options_->statistics, NO_FILE_ERRORS);
      // We do not cache error results so that if the error is transient,
      // or somebody repairs the file, we recover automatically.
    } else {
      assert(file.get() == nullptr);
-      *handle = cache_->Insert(key, table.release(), 1, &DeleteEntry);
+      *handle = cache_->Insert(key, table_reader.release(), 1, &DeleteEntry);
    }
  }
  return s;
@@ -83,10 +93,10 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
                                  const EnvOptions& toptions,
                                  uint64_t file_number,
                                  uint64_t file_size,
-                                  Table** tableptr,
+                                  TableReader** table_reader_ptr,
                                  bool for_compaction) {
-  if (tableptr != nullptr) {
-    *tableptr = nullptr;
+  if (table_reader_ptr != nullptr) {
+    *table_reader_ptr = nullptr;
  }

  Cache::Handle* handle = nullptr;
@@ -96,16 +106,16 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
    return NewErrorIterator(s);
  }

-  Table* table =
-    reinterpret_cast<Table*>(cache_->Value(handle));
-  Iterator* result = table->NewIterator(options);
+  TableReader* table_reader =
+    reinterpret_cast<TableReader*>(cache_->Value(handle));
+  Iterator* result = table_reader->NewIterator(options);
  result->RegisterCleanup(&UnrefEntry, cache_.get(), handle);
-  if (tableptr != nullptr) {
-    *tableptr = table;
+  if (table_reader_ptr != nullptr) {
+    *table_reader_ptr = table_reader;
  }

  if (for_compaction) {
-    table->SetupForCompaction();
+    table_reader->SetupForCompaction();
  }

  return result;
@@ -124,9 +134,9 @@ Status TableCache::Get(const ReadOptions& options,
                       &handle, table_io,
                       options.read_tier == kBlockCacheTier);
  if (s.ok()) {
-    Table* t =
-      reinterpret_cast<Table*>(cache_->Value(handle));
-    s = t->InternalGet(options, k, arg, saver, mark_key_may_exist);
+    TableReader* t =
+      reinterpret_cast<TableReader*>(cache_->Value(handle));
+    s = t->Get(options, k, arg, saver, mark_key_may_exist);
    cache_->Release(handle);
  } else if (options.read_tier && s.IsIncomplete()) {
    // Couldnt find Table in cache but treat as kFound if no_io set
@@ -146,8 +156,8 @@ bool TableCache::PrefixMayMatch(const ReadOptions& options,
                       file_size, &handle, table_io);
  bool may_match = true;
  if (s.ok()) {
-    Table* t =
-      reinterpret_cast<Table*>(cache_->Value(handle));
+    TableReader* t =
+      reinterpret_cast<TableReader*>(cache_->Value(handle));
    may_match = t->PrefixMayMatch(internal_prefix);
    cache_->Release(handle);
  }
@@ -1,19 +1,22 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 //
 // Thread-safe (provides internal synchronization)

-#ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_
-#define STORAGE_LEVELDB_DB_TABLE_CACHE_H_
-
+#pragma once
 #include <string>
 #include <stdint.h>
 #include "db/dbformat.h"
 #include "rocksdb/env.h"
 #include "rocksdb/cache.h"
 #include "port/port.h"
-#include "table/table.h"
+#include "rocksdb/table.h"

 namespace rocksdb {

@@ -36,7 +39,7 @@ class TableCache {
                        const EnvOptions& toptions,
                        uint64_t file_number,
                        uint64_t file_size,
-                        Table** tableptr = nullptr,
+                        TableReader** table_reader_ptr = nullptr,
                        bool for_compaction = false);

  // If a seek to internal key "k" in specified file finds an entry,
@@ -73,5 +76,3 @@ class TableCache {
 };

 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_TABLE_CACHE_H_
@@ -0,0 +1,55 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#include "db/table_stats_collector.h"
+
+#include "db/dbformat.h"
+#include "util/coding.h"
+
+namespace rocksdb {
+
+Status InternalKeyStatsCollector::Add(const Slice& key, const Slice& value) {
+  ParsedInternalKey ikey;
+  if (!ParseInternalKey(key, &ikey)) {
+    return Status::InvalidArgument("Invalid internal key");
+  }
+
+  if (ikey.type == ValueType::kTypeDeletion) {
+    ++deleted_keys_;
+  }
+
+  return Status::OK();
+}
+
+Status InternalKeyStatsCollector::Finish(
+    TableStats::UserCollectedStats* stats) {
+  assert(stats);
+  assert(stats->find(InternalKeyTableStatsNames::kDeletedKeys) == stats->end());
+  std::string val;
+
+  PutVarint64(&val, deleted_keys_);
+  stats->insert(std::make_pair(InternalKeyTableStatsNames::kDeletedKeys, val));
+
+  return Status::OK();
+}
+
+Status UserKeyTableStatsCollector::Add(const Slice& key, const Slice& value) {
+  ParsedInternalKey ikey;
+  if (!ParseInternalKey(key, &ikey)) {
+    return Status::InvalidArgument("Invalid internal key");
+  }
+
+  return collector_->Add(ikey.user_key, value);
+}
+
+Status UserKeyTableStatsCollector::Finish(
+    TableStats::UserCollectedStats* stats) {
+  return collector_->Finish(stats);
+}
+
+const std::string InternalKeyTableStatsNames::kDeletedKeys
+  = "rocksdb.deleted.keys";
+
+}  // namespace rocksdb
@@ -0,0 +1,58 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// This file defines a collection of statistics collectors.
+#pragma once
+
+#include "rocksdb/table_stats.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace rocksdb {
+
+struct InternalKeyTableStatsNames {
+  static const std::string kDeletedKeys;
+};
+
+// Collecting the statistics for internal keys. Visible only by internal
+// rocksdb modules.
+class InternalKeyStatsCollector : public TableStatsCollector {
+ public:
+  virtual Status Add(const Slice& key, const Slice& value);
+  virtual Status Finish(TableStats::UserCollectedStats* stats);
+  virtual const char* Name() const { return "InternalKeyStatsCollector"; }
+
+ private:
+  uint64_t deleted_keys_ = 0;
+};
+
+// When rocksdb creates a new table, it will encode all "user keys" into
+// "internal keys", which contains meta information of a given entry.
+//
+// This class extracts user key from the encoded internal key when Add() is
+// invoked.
+class UserKeyTableStatsCollector : public TableStatsCollector {
+ public:
+  explicit UserKeyTableStatsCollector(TableStatsCollector* collector):
+    UserKeyTableStatsCollector(
+        std::shared_ptr<TableStatsCollector>(collector)
+    ) {
+  }
+
+  explicit UserKeyTableStatsCollector(
+      std::shared_ptr<TableStatsCollector> collector) : collector_(collector) {
+  }
+  virtual ~UserKeyTableStatsCollector() { }
+  virtual Status Add(const Slice& key, const Slice& value);
+  virtual Status Finish(TableStats::UserCollectedStats* stats);
+  virtual const char* Name() const { return collector_->Name(); }
+
+ protected:
+  std::shared_ptr<TableStatsCollector> collector_;
+};
+
+}  // namespace rocksdb
@@ -0,0 +1,258 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#include <map>
+#include <memory>
+#include <string>
+
+#include "db/dbformat.h"
+#include "db/db_impl.h"
+#include "db/table_stats_collector.h"
+#include "rocksdb/table_stats.h"
+#include "rocksdb/table.h"
+#include "util/coding.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+
+namespace rocksdb {
+
+class TableStatsTest {
+ private:
+  unique_ptr<TableReader> table_reader_;
+};
+
+// TODO(kailiu) the following classes should be moved to some more general
+// places, so that other tests can also make use of them.
+// `FakeWritableFile` and `FakeRandomeAccessFile` bypass the real file system
+// and therefore enable us to quickly setup the tests.
+class FakeWritableFile : public WritableFile {
+ public:
+  ~FakeWritableFile() { }
+
+  const std::string& contents() const { return contents_; }
+
+  virtual Status Close() { return Status::OK(); }
+  virtual Status Flush() { return Status::OK(); }
+  virtual Status Sync() { return Status::OK(); }
+
+  virtual Status Append(const Slice& data) {
+    contents_.append(data.data(), data.size());
+    return Status::OK();
+  }
+
+ private:
+  std::string contents_;
+};
+
+
+class FakeRandomeAccessFile : public RandomAccessFile {
+ public:
+  explicit FakeRandomeAccessFile(const Slice& contents)
+      : contents_(contents.data(), contents.size()) {
+  }
+
+  virtual ~FakeRandomeAccessFile() { }
+
+  uint64_t Size() const { return contents_.size(); }
+
+  virtual Status Read(uint64_t offset, size_t n, Slice* result,
+                       char* scratch) const {
+    if (offset > contents_.size()) {
+      return Status::InvalidArgument("invalid Read offset");
+    }
+    if (offset + n > contents_.size()) {
+      n = contents_.size() - offset;
+    }
+    memcpy(scratch, &contents_[offset], n);
+    *result = Slice(scratch, n);
+    return Status::OK();
+  }
+
+ private:
+  std::string contents_;
+};
+
+
+class DumbLogger : public Logger {
+ public:
+  virtual void Logv(const char* format, va_list ap) { }
+  virtual size_t GetLogFileSize() const { return 0; }
+};
+
+// Utilities test functions
+void MakeBuilder(
+    const Options& options,
+    std::unique_ptr<FakeWritableFile>* writable,
+    std::unique_ptr<TableBuilder>* builder) {
+  writable->reset(new FakeWritableFile);
+  builder->reset(
+      options.table_factory->GetTableBuilder(options, writable->get(),
+                                             options.compression));
+}
+
+void OpenTable(
+    const Options& options,
+    const std::string& contents,
+    std::unique_ptr<TableReader>* table_reader) {
+  std::unique_ptr<RandomAccessFile> file(new FakeRandomeAccessFile(contents));
+  auto s = options.table_factory->GetTableReader(
+      options,
+      EnvOptions(),
+      std::move(file),
+      contents.size(),
+      table_reader
+  );
+  ASSERT_OK(s);
+}
+
+// Collects keys that starts with "A" in a table.
+class RegularKeysStartWithA: public TableStatsCollector {
+ public:
+   const char* Name() const { return "RegularKeysStartWithA"; }
+
+   Status Finish(TableStats::UserCollectedStats* stats) {
+     std::string encoded;
+     PutVarint32(&encoded, count_);
+     *stats = TableStats::UserCollectedStats {
+       { "TableStatsTest", "Rocksdb" },
+       { "Count", encoded }
+     };
+     return Status::OK();
+   }
+
+   Status Add(const Slice& user_key, const Slice& value) {
+     // simply asssume all user keys are not empty.
+     if (user_key.data()[0] == 'A') {
+       ++count_;
+     }
+     return Status::OK();
+   }
+
+ private:
+  uint32_t count_ = 0;
+};
+
+TEST(TableStatsTest, CustomizedTableStatsCollector) {
+  Options options;
+
+  // make sure the entries will be inserted with order.
+  std::map<std::string, std::string> kvs = {
+    {"About",     "val5"},  // starts with 'A'
+    {"Abstract",  "val2"},  // starts with 'A'
+    {"Around",    "val7"},  // starts with 'A'
+    {"Beyond",    "val3"},
+    {"Builder",   "val1"},
+    {"Cancel",    "val4"},
+    {"Find",      "val6"},
+  };
+
+  // Test stats collectors with internal keys or regular keys
+  for (bool encode_as_internal : { true, false }) {
+    // -- Step 1: build table
+    auto collector = new RegularKeysStartWithA();
+    if (encode_as_internal) {
+      options.table_stats_collectors = {
+        std::make_shared<UserKeyTableStatsCollector>(collector)
+      };
+    } else {
+      options.table_stats_collectors.resize(1);
+      options.table_stats_collectors[0].reset(collector);
+    }
+    std::unique_ptr<TableBuilder> builder;
+    std::unique_ptr<FakeWritableFile> writable;
+    MakeBuilder(options, &writable, &builder);
+
+    for (const auto& kv : kvs) {
+      if (encode_as_internal) {
+        InternalKey ikey(kv.first, 0, ValueType::kTypeValue);
+        builder->Add(ikey.Encode(), kv.second);
+      } else {
+        builder->Add(kv.first, kv.second);
+      }
+    }
+    ASSERT_OK(builder->Finish());
+
+    // -- Step 2: Open table
+    std::unique_ptr<TableReader> table_reader;
+    OpenTable(options, writable->contents(), &table_reader);
+    const auto& stats = table_reader->GetTableStats().user_collected_stats;
+
+    ASSERT_EQ("Rocksdb", stats.at("TableStatsTest"));
+
+    uint32_t starts_with_A = 0;
+    Slice key(stats.at("Count"));
+    ASSERT_TRUE(GetVarint32(&key, &starts_with_A));
+    ASSERT_EQ(3u, starts_with_A);
+  }
+}
+
+TEST(TableStatsTest, InternalKeyStatsCollector) {
+  InternalKey keys[] = {
+    InternalKey("A", 0, ValueType::kTypeValue),
+    InternalKey("B", 0, ValueType::kTypeValue),
+    InternalKey("C", 0, ValueType::kTypeValue),
+    InternalKey("W", 0, ValueType::kTypeDeletion),
+    InternalKey("X", 0, ValueType::kTypeDeletion),
+    InternalKey("Y", 0, ValueType::kTypeDeletion),
+    InternalKey("Z", 0, ValueType::kTypeDeletion),
+  };
+
+  for (bool sanitized : { false, true }) {
+    std::unique_ptr<TableBuilder> builder;
+    std::unique_ptr<FakeWritableFile> writable;
+    Options options;
+    if (sanitized) {
+      options.table_stats_collectors = {
+        std::make_shared<RegularKeysStartWithA>()
+      };
+      // with sanitization, even regular stats collector will be able to
+      // handle internal keys.
+      auto comparator = options.comparator;
+      // HACK: Set options.info_log to avoid writing log in
+      // SanitizeOptions().
+      options.info_log = std::make_shared<DumbLogger>();
+      options = SanitizeOptions(
+          "db",  // just a place holder
+          nullptr,  // with skip internal key comparator
+          nullptr,  // don't care filter policy
+          options
+      );
+      options.comparator = comparator;
+    } else {
+      options.table_stats_collectors = {
+        std::make_shared<InternalKeyStatsCollector>()
+      };
+    }
+
+    MakeBuilder(options, &writable, &builder);
+    for (const auto& k : keys) {
+      builder->Add(k.Encode(), "val");
+    }
+
+    ASSERT_OK(builder->Finish());
+
+    std::unique_ptr<TableReader> table_reader;
+    OpenTable(options, writable->contents(), &table_reader);
+    const auto& stats = table_reader->GetTableStats().user_collected_stats;
+
+    uint64_t deleted = 0;
+    Slice key(stats.at(InternalKeyTableStatsNames::kDeletedKeys));
+    ASSERT_TRUE(GetVarint64(&key, &deleted));
+    ASSERT_EQ(4u, deleted);
+
+    if (sanitized) {
+      uint32_t starts_with_A = 0;
+      Slice key(stats.at("Count"));
+      ASSERT_TRUE(GetVarint32(&key, &starts_with_A));
+      ASSERT_EQ(1u, starts_with_A);
+    }
+  }
+}
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  return rocksdb::test::RunAllTests();
+}
@@ -1,16 +1,21 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 #include "db/transaction_log_impl.h"
 #include "db/write_batch_internal.h"

 namespace rocksdb {

 TransactionLogIteratorImpl::TransactionLogIteratorImpl(
-                           const std::string& dbname,
+                           const std::string& dir,
                           const Options* options,
                           const EnvOptions& soptions,
                           const SequenceNumber seq,
                           std::unique_ptr<VectorLogPtr> files,
-                           SequenceNumber const * const lastFlushedSequence) :
-    dbname_(dbname),
+                           DBImpl const * const dbimpl) :
+    dir_(dir),
    options_(options),
    soptions_(soptions),
    startingSequenceNumber_(seq),
@@ -18,31 +23,34 @@ TransactionLogIteratorImpl::TransactionLogIteratorImpl(
    started_(false),
    isValid_(false),
    currentFileIndex_(0),
-    lastFlushedSequence_(lastFlushedSequence) {
-  assert(startingSequenceNumber_ <= *lastFlushedSequence_);
-  assert(files_.get() != nullptr);
+    currentBatchSeq_(0),
+    currentLastSeq_(0),
+    dbimpl_(dbimpl) {
+  assert(files_ != nullptr);
+  assert(dbimpl_ != nullptr);

  reporter_.env = options_->env;
  reporter_.info_log = options_->info_log.get();
+  SeekToStartSequence(); // Seek till starting sequence
 }

 Status TransactionLogIteratorImpl::OpenLogFile(
-  const LogFile* logFile,
-  unique_ptr<SequentialFile>* file) {
+    const LogFile* logFile,
+    unique_ptr<SequentialFile>* file) {
  Env* env = options_->env;
  if (logFile->Type() == kArchivedLogFile) {
-    std::string fname = ArchivedLogFileName(dbname_, logFile->LogNumber());
+    std::string fname = ArchivedLogFileName(dir_, logFile->LogNumber());
    return env->NewSequentialFile(fname, file, soptions_);
  } else {
-    std::string fname = LogFileName(dbname_, logFile->LogNumber());
+    std::string fname = LogFileName(dir_, logFile->LogNumber());
    Status status = env->NewSequentialFile(fname, file, soptions_);
    if (!status.ok()) {
      //  If cannot open file in DB directory.
      //  Try the archive dir, as it could have moved in the meanwhile.
-      fname = ArchivedLogFileName(dbname_, logFile->LogNumber());
+      fname = ArchivedLogFileName(dir_, logFile->LogNumber());
      status = env->NewSequentialFile(fname, file, soptions_);
      if (!status.ok()) {
-        return Status::IOError(" Requested file not present in the dir");
+        return Status::IOError("Requested file not present in the dir");
      }
    }
    return status;
@@ -52,7 +60,7 @@ Status TransactionLogIteratorImpl::OpenLogFile(
 BatchResult TransactionLogIteratorImpl::GetBatch()  {
  assert(isValid_);  //  cannot call in a non valid state.
  BatchResult result;
-  result.sequence = currentSequence_;
+  result.sequence = currentBatchSeq_;
  result.writeBatchPtr = std::move(currentBatch_);
  return result;
 }
@@ -65,93 +73,173 @@ bool TransactionLogIteratorImpl::Valid() {
  return started_ && isValid_;
 }

-void TransactionLogIteratorImpl::Next() {
-  LogFile* currentLogFile = files_.get()->at(currentFileIndex_).get();
+bool TransactionLogIteratorImpl::RestrictedRead(
+    Slice* record,
+    std::string* scratch) {
+  // Don't read if no more complete entries to read from logs
+  if (currentLastSeq_ >= dbimpl_->GetLatestSequenceNumber()) {
+    return false;
+  }
+  return currentLogReader_->ReadRecord(record, scratch);
+}

-//  First seek to the given seqNo. in the current file.
+void TransactionLogIteratorImpl::SeekToStartSequence(
+    uint64_t startFileIndex,
+    bool strict) {
  std::string scratch;
  Slice record;
-  if (!started_) {
-    started_ = true;  // this piece only runs onced.
-    isValid_ = false;
-    if (startingSequenceNumber_ > *lastFlushedSequence_) {
-      currentStatus_ = Status::IOError("Looking for a sequence, "
-                                        "which is not flushed yet.");
-      return;
+  started_ = false;
+  isValid_ = false;
+  if (files_->size() <= startFileIndex) {
+    return;
+  }
+  Status s = OpenLogReader(files_->at(startFileIndex).get());
+  if (!s.ok()) {
+    currentStatus_ = s;
+    return;
+  }
+  while (RestrictedRead(&record, &scratch)) {
+    if (record.size() < 12) {
+      reporter_.Corruption(
+        record.size(), Status::Corruption("very small log record"));
+      continue;
    }
-    Status s = OpenLogReader(currentLogFile);
-    if (!s.ok()) {
-      currentStatus_ = s;
+    UpdateCurrentWriteBatch(record);
+    if (currentLastSeq_ >= startingSequenceNumber_) {
+      if (strict && currentBatchSeq_ != startingSequenceNumber_) {
+        currentStatus_ = Status::Corruption("Gap in sequence number. Could not "
+                                            "seek to required sequence number");
+        reporter_.Info(currentStatus_.ToString().c_str());
+        return;
+      } else if (strict) {
+        reporter_.Info("Could seek required sequence number. Iterator will "
+                       "continue.");
+      }
+      isValid_ = true;
+      started_ = true; // set started_ as we could seek till starting sequence
+      return;
+    } else {
      isValid_ = false;
-      return;
-    }
-    while (currentLogReader_->ReadRecord(&record, &scratch)) {
-      if (record.size() < 12) {
-        reporter_.Corruption(
-          record.size(), Status::Corruption("log record too small"));
-        continue;
-      }
-      UpdateCurrentWriteBatch(record);
-      if (currentSequence_ >= startingSequenceNumber_) {
-        assert(currentSequence_ <= *lastFlushedSequence_);
-        isValid_ = true;
-        break;
-      } else {
-        isValid_ = false;
-      }
-    }
-    if (isValid_) {
-      // Done for this iteration
-      return;
    }
  }
-  bool openNextFile = true;
-  while(openNextFile) {
+
+  // Could not find start sequence in first file. Normally this must be the
+  // only file. Otherwise log the error and let the iterator return next entry
+  // If strict is set, we want to seek exactly till the start sequence and it
+  // should have been present in the file we scanned above
+  if (strict) {
+    currentStatus_ = Status::Corruption("Gap in sequence number. Could not "
+                                        "seek to required sequence number");
+    reporter_.Info(currentStatus_.ToString().c_str());
+  } else if (files_->size() != 1) {
+    currentStatus_ = Status::Corruption("Start sequence was not found, "
+                                        "skipping to the next available");
+    reporter_.Info(currentStatus_.ToString().c_str());
+    // Let NextImpl find the next available entry. started_ remains false
+    // because we don't want to check for gaps while moving to start sequence
+    NextImpl(true);
+  }
+}
+
+void TransactionLogIteratorImpl::Next() {
+  return NextImpl(false);
+}
+
+void TransactionLogIteratorImpl::NextImpl(bool internal) {
+  std::string scratch;
+  Slice record;
+  isValid_ = false;
+  if (!internal && !started_) {
+    // Runs every time until we can seek to the start sequence
+    return SeekToStartSequence();
+  }
+  while(true) {
    assert(currentLogReader_);
-    if (currentSequence_ < *lastFlushedSequence_) {
-      if (currentLogReader_->IsEOF()) {
-        currentLogReader_->UnmarkEOF();
-      }
-      while (currentLogReader_->ReadRecord(&record, &scratch)) {
-        if (record.size() < 12) {
-          reporter_.Corruption(
-            record.size(), Status::Corruption("log record too small"));
-          continue;
-        } else {
-          UpdateCurrentWriteBatch(record);
-          openNextFile = false;
-          break;
+    if (currentLogReader_->IsEOF()) {
+      currentLogReader_->UnmarkEOF();
+    }
+    while (RestrictedRead(&record, &scratch)) {
+      if (record.size() < 12) {
+        reporter_.Corruption(
+          record.size(), Status::Corruption("very small log record"));
+        continue;
+      } else {
+        // started_ should be true if called by application
+        assert(internal || started_);
+        // started_ should be false if called internally
+        assert(!internal || !started_);
+        UpdateCurrentWriteBatch(record);
+        if (internal && !started_) {
+          started_ = true;
        }
+        return;
      }
    }

-    if (openNextFile) {
-      if (currentFileIndex_ < files_.get()->size() - 1) {
-        ++currentFileIndex_;
-        Status status =OpenLogReader(files_.get()->at(currentFileIndex_).get());
-        if (!status.ok()) {
-          isValid_ = false;
-          currentStatus_ = status;
-          return;
-        }
-      } else {
+    // Open the next file
+    if (currentFileIndex_ < files_->size() - 1) {
+      ++currentFileIndex_;
+      Status status =OpenLogReader(files_->at(currentFileIndex_).get());
+      if (!status.ok()) {
        isValid_ = false;
-        openNextFile = false;
-        if (currentSequence_ == *lastFlushedSequence_) {
-          currentStatus_ = Status::OK();
-        } else {
-          currentStatus_ = Status::IOError(" NO MORE DATA LEFT");
-        }
+        currentStatus_ = status;
+        return;
      }
+    } else {
+      isValid_ = false;
+      if (currentLastSeq_ == dbimpl_->GetLatestSequenceNumber()) {
+        currentStatus_ = Status::OK();
+      } else {
+        currentStatus_ = Status::IOError("NO MORE DATA LEFT");
+      }
+      return;
    }
  }
 }

+bool TransactionLogIteratorImpl::IsBatchExpected(
+    const WriteBatch* batch,
+    const SequenceNumber expectedSeq) {
+  assert(batch);
+  SequenceNumber batchSeq = WriteBatchInternal::Sequence(batch);
+  if (batchSeq != expectedSeq) {
+    char buf[200];
+    snprintf(buf, sizeof(buf),
+             "Discontinuity in log records. Got seq=%lu, Expected seq=%lu, "
+             "Last flushed seq=%lu.Log iterator will reseek the correct batch.",
+             batchSeq, expectedSeq, dbimpl_->GetLatestSequenceNumber());
+    reporter_.Info(buf);
+    return false;
+  }
+  return true;
+}
+
 void TransactionLogIteratorImpl::UpdateCurrentWriteBatch(const Slice& record) {
-  WriteBatch* batch = new WriteBatch();
-  WriteBatchInternal::SetContents(batch, record);
-  currentSequence_ = WriteBatchInternal::Sequence(batch);
-  currentBatch_.reset(batch);
+  std::unique_ptr<WriteBatch> batch(new WriteBatch());
+  WriteBatchInternal::SetContents(batch.get(), record);
+
+  SequenceNumber expectedSeq = currentLastSeq_ + 1;
+  // If the iterator has started, then confirm that we get continuous batches
+  if (started_ && !IsBatchExpected(batch.get(), expectedSeq)) {
+    // Seek to the batch having expected sequence number
+    if (expectedSeq < files_->at(currentFileIndex_)->StartSequence()) {
+      // Expected batch must lie in the previous log file
+      currentFileIndex_--;
+      currentFileIndex_ = (currentFileIndex_ >= 0) ? currentFileIndex_ : 0;
+    }
+    startingSequenceNumber_ = expectedSeq;
+    // currentStatus_ will be set to Ok if reseek succeeds
+    currentStatus_ = Status::NotFound("Gap in sequence numbers");
+    return SeekToStartSequence(currentFileIndex_, true);
+  }
+
+  currentBatchSeq_ = WriteBatchInternal::Sequence(batch.get());
+  currentLastSeq_ = currentBatchSeq_ +
+                    WriteBatchInternal::Count(batch.get()) - 1;
+  // currentBatchSeq_ can only change here
+  assert(currentLastSeq_ <= dbimpl_->GetLatestSequenceNumber());
+
+  currentBatch_ = move(batch);
  isValid_ = true;
  currentStatus_ = Status::OK();
 }
@@ -1,13 +1,16 @@
-// Copyright 2008-present Facebook. All Rights Reserved.
-#ifndef STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
-#define STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
-
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+#pragma once
 #include <vector>

 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
 #include "rocksdb/types.h"
 #include "rocksdb/transaction_log.h"
+#include "db/db_impl.h"
 #include "db/log_reader.h"
 #include "db/filename.h"

@@ -19,6 +22,9 @@ struct LogReporter : public log::Reader::Reporter {
  virtual void Corruption(size_t bytes, const Status& s) {
    Log(info_log, "dropping %zu bytes; %s", bytes, s.ToString().c_str());
  }
+  virtual void Info(const char* s) {
+    Log(info_log, "%s", s);
+  }
 };

 class LogFileImpl : public LogFile {
@@ -60,12 +66,12 @@ class LogFileImpl : public LogFile {

 class TransactionLogIteratorImpl : public TransactionLogIterator {
 public:
-  TransactionLogIteratorImpl(const std::string& dbname,
+  TransactionLogIteratorImpl(const std::string& dir,
                             const Options* options,
                             const EnvOptions& soptions,
                             const SequenceNumber seqNum,
                             std::unique_ptr<VectorLogPtr> files,
-                             SequenceNumber const * const lastFlushedSequence);
+                             DBImpl const * const dbimpl);

  virtual bool Valid();

@@ -76,10 +82,10 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
  virtual BatchResult GetBatch();

 private:
-  const std::string& dbname_;
+  const std::string& dir_;
  const Options* options_;
  const EnvOptions& soptions_;
-  const SequenceNumber startingSequenceNumber_;
+  SequenceNumber startingSequenceNumber_;
  std::unique_ptr<VectorLogPtr> files_;
  bool started_;
  bool isValid_;  // not valid when it starts of.
@@ -89,15 +95,24 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
  unique_ptr<log::Reader> currentLogReader_;
  Status OpenLogFile(const LogFile* logFile, unique_ptr<SequentialFile>* file);
  LogReporter reporter_;
-  SequenceNumber const * const lastFlushedSequence_;
-  // represents the sequence number being read currently.
-  SequenceNumber currentSequence_;
+  SequenceNumber currentBatchSeq_; // sequence number at start of current batch
+  SequenceNumber currentLastSeq_; // last sequence in the current batch
+  DBImpl const * const dbimpl_; // The db on whose log files this iterates

+  // Reads from transaction log only if the writebatch record has been written
+  bool RestrictedRead(Slice* record, std::string* scratch);
+  // Seeks to startingSequenceNumber reading from startFileIndex in files_.
+  // If strict is set,then must get a batch starting with startingSequenceNumber
+  void SeekToStartSequence(uint64_t startFileIndex = 0, bool strict = false);
+  // Implementation of Next. SeekToStartSequence calls it internally with
+  // internal=true to let it find next entry even if it has to jump gaps because
+  // the iterator may start off from the first available entry but promises to
+  // be continuous after that
+  void NextImpl(bool internal = false);
+  // Check if batch is expected, else return false
+  bool IsBatchExpected(const WriteBatch* batch, SequenceNumber expectedSeq);
+  // Update current batch if a continuous batch is found, else return false
  void UpdateCurrentWriteBatch(const Slice& record);
  Status OpenLogReader(const LogFile* file);
 };
-
-
-
 }  //  namespace rocksdb
-#endif  //  STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -1,10 +1,13 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#ifndef STORAGE_LEVELDB_DB_VERSION_EDIT_H_
-#define STORAGE_LEVELDB_DB_VERSION_EDIT_H_
-
+#pragma once
 #include <set>
 #include <utility>
 #include <vector>
@@ -31,9 +34,9 @@ struct FileMetaData {

 class VersionEdit {
 public:
-  /* implicit */ VersionEdit(int number_levels) :
-  number_levels_(number_levels) {
-  Clear();
+  explicit VersionEdit(int number_levels) :
+      number_levels_(number_levels) {
+    Clear();
  }
  ~VersionEdit() { }

@@ -123,5 +126,3 @@ class VersionEdit {
 };

 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_VERSION_EDIT_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -14,7 +19,7 @@
 #include "db/table_cache.h"
 #include "rocksdb/env.h"
 #include "rocksdb/merge_operator.h"
-#include "rocksdb/table_builder.h"
+#include "rocksdb/table.h"
 #include "table/merger.h"
 #include "table/two_level_iterator.h"
 #include "util/coding.h"
@@ -289,11 +294,11 @@ struct Saver {
 };
 }

-// Called from TableCache::Get and InternalGet when file/block in which key may
-// exist are not there in TableCache/BlockCache respectively. In this case we
-// can't guarantee that key does not exist and are not permitted to do IO to be
-// certain.Set the status=kFound and value_found=false to let the caller know
-// that key may exist but is not there in memory
+// Called from TableCache::Get and Table::Get when file/block in which
+// key may  exist are not there in TableCache/BlockCache respectively. In this
+// case we  can't guarantee that key does not exist and are not permitted to do
+// IO to be  certain.Set the status=kFound and value_found=false to let the
+// caller know that key may exist but is not there in memory
 static void MarkKeyMayExist(void* arg) {
  Saver* s = reinterpret_cast<Saver*>(arg);
  s->state = kFound;
@@ -1243,18 +1248,8 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu,
  }

  if (!descriptor_log_ || new_descriptor_log) {
-    // No reason to unlock *mu here since we only hit this path in the
-    // first call to LogAndApply (when opening the database).
-    assert(!descriptor_log_ || new_descriptor_log);
    new_manifest_file = DescriptorFileName(dbname_, manifest_file_number_);
    edit->SetNextFile(next_file_number_);
-    unique_ptr<WritableFile> descriptor_file;
-    s = env_->NewWritableFile(new_manifest_file, &descriptor_file,
-                              storage_options_);
-    if (s.ok()) {
-      descriptor_log_.reset(new log::Writer(std::move(descriptor_file)));
-      s = WriteSnapshot(descriptor_log_.get());
-    }
  }

  // Unlock during expensive MANIFEST log write. New writes cannot get here
@@ -1266,6 +1261,18 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu,

    mu->Unlock();

+    // This is fine because everything inside of this block is serialized --
+    // only one thread can be here at the same time
+    if (!new_manifest_file.empty()) {
+      unique_ptr<WritableFile> descriptor_file;
+      s = env_->NewWritableFile(new_manifest_file, &descriptor_file,
+                                storage_options_);
+      if (s.ok()) {
+        descriptor_log_.reset(new log::Writer(std::move(descriptor_file)));
+        s = WriteSnapshot(descriptor_log_.get());
+      }
+    }
+
    // The calls to Finalize and UpdateFilesBySize are cpu-heavy
    // and is best called outside the mutex.
    Finalize(v, size_being_compacted);
@@ -1311,6 +1318,7 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu,
    // find offset in manifest file where this version is stored.
    new_manifest_file_size = descriptor_log_->file()->GetFileSize();

+    LogFlush(options_->info_log);
    mu->Lock();
    // cache the manifest_file_size so that it can be used to rollover in the
    // next call to LogAndApply
@@ -1430,9 +1438,9 @@ Status VersionSet::Recover() {
      if (s.ok()) {
        if (edit.has_comparator_ &&
            edit.comparator_ != icmp_.user_comparator()->Name()) {
-          s = Status::InvalidArgument(
-              edit.comparator_ + "does not match existing comparator ",
-              icmp_.user_comparator()->Name());
+          s = Status::InvalidArgument(icmp_.user_comparator()->Name(),
+                                      "does not match existing comparator " +
+                                      edit.comparator_);
        }
      }

@@ -1548,9 +1556,9 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
      if (s.ok()) {
        if (edit.has_comparator_ &&
            edit.comparator_ != icmp_.user_comparator()->Name()) {
-          s = Status::InvalidArgument(
-              edit.comparator_ + "does not match existing comparator ",
-              icmp_.user_comparator()->Name());
+          s = Status::InvalidArgument(icmp_.user_comparator()->Name(),
+                                      "does not match existing comparator " +
+                                      edit.comparator_);
        }
      }

@@ -1645,7 +1653,12 @@ void VersionSet::Finalize(Version* v,
  double max_score = 0;
  int max_score_level = 0;

-  for (int level = 0; level < NumberLevels()-1; level++) {
+  int num_levels_to_check =
+      (options_->compaction_style != kCompactionStyleUniversal) ?
+          NumberLevels() - 1 : 1;
+
+  for (int level = 0; level < num_levels_to_check; level++) {
+
    double score;
    if (level == 0) {
      // We treat level-0 specially by bounding the number of files
@@ -1910,12 +1923,12 @@ uint64_t VersionSet::ApproximateOffsetOf(Version* v, const InternalKey& ikey) {
      } else {
        // "ikey" falls in the range for this table.  Add the
        // approximate offset of "ikey" within the table.
-        Table* tableptr;
+        TableReader* table_reader_ptr;
        Iterator* iter = table_cache_->NewIterator(
            ReadOptions(), storage_options_, files[i]->number,
-            files[i]->file_size, &tableptr);
-        if (tableptr != nullptr) {
-          result += tableptr->ApproximateOffsetOf(ikey.Encode());
+            files[i]->file_size, &table_reader_ptr);
+        if (table_reader_ptr != nullptr) {
+          result += table_reader_ptr->ApproximateOffsetOf(ikey.Encode());
        }
        delete iter;
      }
@@ -2240,8 +2253,10 @@ Compaction* VersionSet::PickCompactionUniversalSizeAmp(
  assert(start_index >= 0 && start_index < file_by_time.size() - 1);

  // create a compaction request
+  // We always compact all the files, so always compress.
  Compaction* c = new Compaction(level, level, MaxFileSizeForLevel(level),
-                                 LLONG_MAX, NumberLevels());
+                                 LLONG_MAX, NumberLevels(), false,
+                                 true);
  c->score_ = score;
  for (unsigned int loop = start_index; loop < file_by_time.size(); loop++) {
    int index = file_by_time[loop];
@@ -2346,11 +2361,30 @@ Compaction* VersionSet::PickCompactionUniversalReadAmp(
  if (!done || candidate_count <= 1) {
    return nullptr;
  }
+  unsigned int first_index_after = start_index + candidate_count;
+  // Compression is enabled if files compacted earlier already reached
+  // size ratio of compression.
+  bool enable_compression = true;
+  int ratio_to_compress =
+      options_->compaction_options_universal.compression_size_percent;
+  if (ratio_to_compress >= 0) {
+    uint64_t total_size = TotalFileSize(current_->files_[level]);
+    uint64_t older_file_size = 0;
+    for (unsigned int i = file_by_time.size() - 1; i >= first_index_after;
+        i--) {
+      older_file_size += current_->files_[level][file_by_time[i]]->file_size;
+      if (older_file_size * 100L >= total_size * (long) ratio_to_compress) {
+        enable_compression = false;
+        break;
+      }
+    }
+  }
  Compaction* c = new Compaction(level, level, MaxFileSizeForLevel(level),
-                                 LLONG_MAX, NumberLevels());
+                                 LLONG_MAX, NumberLevels(), false,
+                                 enable_compression);
  c->score_ = score;

-  for (unsigned int i = start_index; i < start_index + candidate_count; i++) {
+  for (unsigned int i = start_index; i < first_index_after; i++) {
    int index = file_by_time[i];
    FileMetaData* f = current_->files_[level][index];
    c->inputs_[0].push_back(f);
@@ -2436,6 +2470,10 @@ Compaction* VersionSet::PickCompactionUniversal(int level, double score) {
  // remember this currently undergoing compaction
  compactions_in_progress_[level].insert(c);

+  // Record whether this compaction includes all sst files.
+  // For now, it is only relevant in universal compaction mode.
+  c->is_full_compaction_ = (c->inputs_[0].size() == current_->files_[0].size());
+
  return c;
 }

@@ -2874,7 +2912,7 @@ Compaction* VersionSet::CompactRange(

 Compaction::Compaction(int level, int out_level, uint64_t target_file_size,
  uint64_t max_grandparent_overlap_bytes, int number_levels,
-  bool seek_compaction)
+  bool seek_compaction, bool enable_compression)
    : level_(level),
      out_level_(out_level),
      max_output_file_size_(target_file_size),
@@ -2882,6 +2920,7 @@ Compaction::Compaction(int level, int out_level, uint64_t target_file_size,
      input_version_(nullptr),
      number_levels_(number_levels),
      seek_compaction_(seek_compaction),
+      enable_compression_(enable_compression),
      grandparent_index_(0),
      seen_key_(false),
      overlapped_bytes_(0),
@@ -2889,6 +2928,7 @@ Compaction::Compaction(int level, int out_level, uint64_t target_file_size,
      parent_index_(-1),
      score_(0),
      bottommost_level_(false),
+      is_full_compaction_(false),
      level_ptrs_(std::vector<size_t>(number_levels)) {
  edit_ = new VersionEdit(number_levels_);
  for (int i = 0; i < number_levels_; i++) {
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -12,9 +17,7 @@
 // Version,VersionSet are thread-compatible, but require external
 // synchronization on all accesses.

-#ifndef STORAGE_LEVELDB_DB_VERSION_SET_H_
-#define STORAGE_LEVELDB_DB_VERSION_SET_H_
-
+#pragma once
 #include <map>
 #include <memory>
 #include <set>
@@ -32,11 +35,9 @@ namespace log { class Writer; }
 class Compaction;
 class Iterator;
 class MemTable;
-class TableBuilder;
 class TableCache;
 class Version;
 class VersionSet;
-class WritableFile;

 // Return the smallest index i such that files[i]->largest >= key.
 // Return files.size() if there is no such file.
@@ -316,7 +317,17 @@ class VersionSet {
  // Returns true iff some level needs a compaction because it has
  // exceeded its target size.
  bool NeedsSizeCompaction() const {
-    for (int i = 0; i < NumberLevels()-1; i++) {
+    // In universal compaction case, this check doesn't really
+    // check the compaction condition, but checks num of files threshold
+    // only. We are not going to miss any compaction opportunity
+    // but it's likely that more compactions are scheduled but
+    // ending up with nothing to do. We can improve it later.
+    // TODO: improve this function to be accurate for universal
+    //       compactions.
+    int num_levels_to_check =
+        (options_->compaction_style != kCompactionStyleUniversal) ?
+            NumberLevels() - 1 : 1;
+    for (int i = 0; i < num_levels_to_check; i++) {
      if (current_->compaction_score_[i] >= 1) {
        return true;
      }
@@ -547,6 +558,9 @@ class Compaction {
  // Maximum size of files to build during this compaction.
  uint64_t MaxOutputFileSize() const { return max_output_file_size_; }

+  // Whether compression will be enabled for compaction outputs
+  bool enable_compression() const { return enable_compression_; }
+
  // Is this a trivial compaction that can be implemented by just
  // moving a single input file to the next level (no merging or splitting)
  bool IsTrivialMove() const;
@@ -575,13 +589,16 @@ class Compaction {
  // Is this compaction creating a file in the bottom most level?
  bool BottomMostLevel() { return bottommost_level_; }

+  // Does this compaction include all sst files?
+  bool IsFullCompaction() { return is_full_compaction_; }
+
 private:
  friend class Version;
  friend class VersionSet;

  explicit Compaction(int level, int out_level, uint64_t target_file_size,
    uint64_t max_grandparent_overlap_bytes, int number_levels,
-    bool seek_compaction = false);
+    bool seek_compaction = false, bool enable_compression = true);

  int level_;
  int out_level_; // levels to which output files are stored
@@ -592,6 +609,7 @@ class Compaction {
  int number_levels_;

  bool seek_compaction_;
+  bool enable_compression_;

  // Each compaction reads inputs from "level_" and "level_+1"
  std::vector<FileMetaData*> inputs_[2];      // The two sets of inputs
@@ -609,6 +627,8 @@ class Compaction {

  // Is this compaction creating a file in the bottom most level?
  bool bottommost_level_;
+  // Does this compaction include all sst files?
+  bool is_full_compaction_;

  // level_ptrs_ holds indices into input_version_->levels_: our state
  // is that we are positioned at one of the file ranges for each
@@ -628,5 +648,3 @@ class Compaction {
 };

 }  // namespace rocksdb
-
-#endif  // STORAGE_LEVELDB_DB_VERSION_SET_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2012 Facebook. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -183,7 +188,12 @@ class MemTableInserter : public WriteBatch::Handler {
  }

  virtual void Put(const Slice& key, const Slice& value) {
-    mem_->Add(sequence_, kTypeValue, key, value);
+    if (options_->inplace_update_support
+        && mem_->Update(sequence_, kTypeValue, key, value)) {
+      RecordTick(options_->statistics, NUMBER_KEYS_UPDATED);
+    } else {
+      mem_->Add(sequence_, kTypeValue, key, value);
+    }
    sequence_++;
  }
  virtual void Merge(const Slice& key, const Slice& value) {
@@ -1,10 +1,13 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
-#define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
-
+#pragma once
 #include "rocksdb/types.h"
 #include "rocksdb/write_batch.h"
 #include "rocksdb/db.h"
@@ -45,13 +48,10 @@ class WriteBatchInternal {
  // Drops deletes in batch if filter_del is set to true and
  // db->KeyMayExist returns false
  static Status InsertInto(const WriteBatch* batch, MemTable* memtable,
-                           const Options* opts = nullptr, DB* db = nullptr,
+                           const Options* opts, DB* db = nullptr,
                           const bool filter_del = false);

  static void Append(WriteBatch* dst, const WriteBatch* src);
 };

 }  // namespace rocksdb
-
-
-#endif  // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
@@ -1,3 +1,8 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -20,7 +25,8 @@ static std::string PrintContents(WriteBatch* b) {
  MemTable* mem = new MemTable(cmp, factory);
  mem->Ref();
  std::string state;
-  Status s = WriteBatchInternal::InsertInto(b, mem);
+  Options options;
+  Status s = WriteBatchInternal::InsertInto(b, mem, &options);
  int count = 0;
  Iterator* iter = mem->NewIterator();
  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
@@ -1,4 +1,4 @@
-This directory contains the hdfs extensions needed to make leveldb store
+This directory contains the hdfs extensions needed to make rocksdb store
 files in HDFS.

 The hdfs.h file is copied from the Apache Hadoop 1.0 source code. 
@@ -7,12 +7,12 @@ It defines the libhdfs library
 data in HDFS.  The libhdfs.a is copied from the Apache Hadoop 1.0 build. 
 It implements the API defined in hdfs.h. If your hadoop cluster is running
 a different hadoop release, then install these two files manually from your
-hadoop distribution and then recompile leveldb.
+hadoop distribution and then recompile rocksdb.

-The env_hdfs.h file defines the leveldb objects that are needed to talk to an
+The env_hdfs.h file defines the rocksdb objects that are needed to talk to an
 underlying filesystem. 

-If you want to compile leveldb with hdfs support, please set the following
+If you want to compile rocksdb with hdfs support, please set the following
 enviroment variables appropriately:
   USE_HDFS=1
   JAVA_HOME=/usr/local/jdk-6u22-64
@@ -1,25 +1,10 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-// Copyright (c) 2012 Facebook. All rights reserved.
-
-#ifndef LEVELDB_HDFS_FILE_H
-#define LEVELDB_HDFS_FILE_H
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//

+#pragma once
 #include <algorithm>
 #include <stdio.h>
 #include <sys/time.h>
@@ -55,7 +40,7 @@ private:
 };

 //
-// The HDFS environment for leveldb. This class overrides all the
+// The HDFS environment for rocksdb. This class overrides all the
 // file/dir access methods and delegates the thread-mgmt methods to the
 // default posix environment.
 //
@@ -81,6 +66,10 @@ class HdfsEnv : public Env {
  virtual Status NewWritableFile(const std::string& fname,
                                 WritableFile** result);

+  virtual Status NewRandomRWFile(const std::string& fname,
+                                 unique_ptr<RandomRWFile>* result,
+                                 const EnvOptions& options);
+
  virtual bool FileExists(const std::string& fname);

  virtual Status GetChildren(const std::string& path,
@@ -227,7 +216,7 @@ class HdfsEnv : public Env {

 public:
  HdfsEnv(const std::string& fsname) {
-    fprintf(stderr, "You have not build leveldb with HDFS support\n");
+    fprintf(stderr, "You have not build rocksdb with HDFS support\n");
    fprintf(stderr, "Please see hdfs/README for details\n");
    throw new std::exception();
  }
@@ -251,6 +240,12 @@ class HdfsEnv : public Env {
    return notsup;
  }

+  virtual Status NewRandomRWFile(const std::string& fname,
+                                 unique_ptr<RandomRWFile>* result,
+                                 const EnvOptions& options) {
+    return notsup;
+  }
+
  virtual bool FileExists(const std::string& fname){return false;}

  virtual Status GetChildren(const std::string& path,
@@ -305,5 +300,3 @@ class HdfsEnv : public Env {
 }

 #endif // USE_HDFS
-
-#endif // LEVELDB_HDFS_FILE_H
@@ -1,21 +1,8 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
 #ifndef LIBHDFS_HDFS_H
 #define LIBHDFS_HDFS_H

@@ -215,7 +215,7 @@ TEST(MemEnvTest, DBTest) {
  delete iterator;

  DBImpl* dbi = reinterpret_cast<DBImpl*>(db);
-  ASSERT_OK(dbi->TEST_CompactMemTable());
+  ASSERT_OK(dbi->TEST_FlushMemTable());

  for (size_t i = 0; i < 3; ++i) {
    std::string res;
@@ -38,6 +38,4 @@ class Arena {

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_ARENA_H_
@@ -28,10 +28,22 @@ using std::shared_ptr;

 class Cache;

-// Create a new cache with a fixed size capacity.  This implementation
-// of Cache uses a least-recently-used eviction policy.
+// Create a new cache with a fixed size capacity. The cache is sharded
+// to 2^numShardBits shards, by hash of the key. The total capacity
+// is divided and evenly assigned to each shard. Inside each shard,
+// the eviction is done in two passes: first try to free spaces by
+// evicting entries that are among the most least used removeScanCountLimit
+// entries and do not have reference other than by the cache itself, in
+// the least-used order. If not enough space is freed, further free the
+// entries in least used order.
+//
+// The functions without parameter numShardBits and/or removeScanCountLimit
+// use default values. removeScanCountLimit's default value is 0, which
+// means a strict LRU order inside each shard.
 extern shared_ptr<Cache> NewLRUCache(size_t capacity);
 extern shared_ptr<Cache> NewLRUCache(size_t capacity, int numShardBits);
+extern shared_ptr<Cache> NewLRUCache(size_t capacity, int numShardBits,
+                                     int removeScanCountLimit);

 class Cache {
 public:
@@ -103,6 +115,4 @@ class Cache {

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_UTIL_CACHE_H_
@@ -16,8 +16,14 @@ class Slice;

 class CompactionFilter {
 public:
-  virtual ~CompactionFilter() {}

+  // Context information of a compaction run
+  struct Context {
+    // Does this compaction run include all data files
+    bool is_full_compaction;
+  };
+
+  virtual ~CompactionFilter() {}

  // The compaction process invokes this
  // method for kv that is being compacted. A return value
@@ -44,30 +50,30 @@ class CompactionFilter {
 // Each compaction will create a new CompactionFilter allowing the
 // application to know about different campactions
 class CompactionFilterFactory {
-  public:
-    virtual ~CompactionFilterFactory() { };
-    virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter() = 0;
+ public:
+  virtual ~CompactionFilterFactory() { };

-    // Returns a name that identifies this compaction filter factory.
-    virtual const char* Name() const = 0;
+  virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
+    const CompactionFilter::Context& context) = 0;
+
+  // Returns a name that identifies this compaction filter factory.
+  virtual const char* Name() const = 0;
 };

 // Default implementaion of CompactionFilterFactory which does not
 // return any filter
 class DefaultCompactionFilterFactory : public CompactionFilterFactory {
-  public:
-    virtual std::unique_ptr<CompactionFilter>
-    CreateCompactionFilter() override {
-      return std::unique_ptr<CompactionFilter>(nullptr);
-    }
+ public:
+  virtual std::unique_ptr<CompactionFilter>
+  CreateCompactionFilter(const CompactionFilter::Context& context) override {
+    return std::unique_ptr<CompactionFilter>(nullptr);
+  }

-    virtual const char* Name() const override {
-      return "DefaultCompactionFilterFactory";
-    }
+  virtual const char* Name() const override {
+    return "DefaultCompactionFilterFactory";
+  }
 };

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_COMPACTION_FILTER_H_
@@ -13,7 +13,7 @@ class Slice;

 // A Comparator object provides a total order across slices that are
 // used as keys in an sstable or a database.  A Comparator implementation
-// must be thread-safe since leveldb may invoke its methods concurrently
+// must be thread-safe since rocksdb may invoke its methods concurrently
 // from multiple threads.
 class Comparator {
 public:
@@ -33,7 +33,7 @@ class Comparator {
  // the comparator implementation changes in a way that will cause
  // the relative ordering of any two keys to change.
  //
-  // Names starting with "leveldb." are reserved and should not be used
+  // Names starting with "rocksdb." are reserved and should not be used
  // by any clients of this package.
  virtual const char* Name() const = 0;

@@ -60,6 +60,4 @@ extern const Comparator* BytewiseComparator();

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_COMPARATOR_H_
@@ -81,8 +81,8 @@ class DB {
  DB() { }
  virtual ~DB();

-  // Set the database entry for "key" to "value".  Returns OK on success,
-  // and a non-OK status on error.
+  // Set the database entry for "key" to "value".
+  // Returns OK on success, and a non-OK status on error.
  // Note: consider setting options.sync = true.
  virtual Status Put(const WriteOptions& options,
                     const Slice& key,
@@ -175,11 +175,11 @@ class DB {
  //
  // Valid property names include:
  //
-  //  "leveldb.num-files-at-level<N>" - return the number of files at level <N>,
+  //  "rocksdb.num-files-at-level<N>" - return the number of files at level <N>,
  //     where <N> is an ASCII representation of a level number (e.g. "0").
-  //  "leveldb.stats" - returns a multi-line string that describes statistics
+  //  "rocksdb.stats" - returns a multi-line string that describes statistics
  //     about the internal operation of the DB.
-  //  "leveldb.sstables" - returns a multi-line string that describes all
+  //  "rocksdb.sstables" - returns a multi-line string that describes all
  //     of the sstables that make up the db contents.
  virtual bool GetProperty(const Slice& property, std::string* value) = 0;

@@ -257,32 +257,24 @@ class DB {
  // Retrieve the sorted list of all wal files with earliest file first
  virtual Status GetSortedWalFiles(VectorLogPtr& files) = 0;

-  // Delete wal files in files. These can be either live or archived.
-  // Returns Status::OK if all files could be deleted, otherwise Status::IOError
-  virtual Status DeleteWalFiles(const VectorLogPtr& files) = 0;
-
  // The sequence number of the most recent transaction.
-  virtual SequenceNumber GetLatestSequenceNumber() = 0;
+  virtual SequenceNumber GetLatestSequenceNumber() const = 0;

-  // Return's an iterator for all writes since the sequence number
-  // Status::ok if iterator is valid.
-  // The iterator internally holds references to the available log files.
-  // It automatically takes care of closing a file with no-updates left, and
-  // opening the next one.
-  // If the sequence number is non existent. it returns an iterator at a seq_no
-  // just greater than the requested seq_no.
-  // Must set WAL_ttl_seconds to a large value to use this api.
-  // else the WAL files will get
+  // Sets iter to an iterator that is positioned at a write-batch containing
+  // seq_number. If the sequence number is non existent, it returns an iterator
+  // at the first available seq_no after the requested seq_no
+  // Returns Status::Ok if iterator is valid
+  // Must set WAL_ttl_seconds or WAL_size_limit_MB to large values to
+  // use this api, else the WAL files will get
  // cleared aggressively and the iterator might keep getting invalid before
  // an update is read.
  virtual Status GetUpdatesSince(SequenceNumber seq_number,
                                 unique_ptr<TransactionLogIterator>* iter) = 0;

-  // Delete the file name from the db directory and update the internal
-  // state to reflect that.
-  virtual Status DeleteFile(std::string name) {
-    return Status::OK();
-  }
+  // Delete the file name from the db directory and update the internal state to
+  // reflect that. Supports deletion of sst and log files only. 'name' must be
+  // path relative to the db directory. eg. 000001.sst, /archive/000003.log
+  virtual Status DeleteFile(std::string name) = 0;

  // Returns a list of all table files with their level, start key
  // and end key
@@ -308,6 +300,4 @@ Status RepairDB(const std::string& dbname, const Options& options);

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_DB_H_
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 //
-// An Env is an interface used by the leveldb implementation to access
+// An Env is an interface used by the rocksdb implementation to access
 // operating system functionality like the filesystem etc.  Callers
 // may wish to provide a custom Env object when opening a database to
 // get fine gain control; e.g., to rate limit file system operations.
@@ -28,6 +28,7 @@ class RandomAccessFile;
 class SequentialFile;
 class Slice;
 class WritableFile;
+class RandomRWFile;
 class Options;

 using std::unique_ptr;
@@ -72,7 +73,7 @@ class Env {
  // system.  Sophisticated users may wish to provide their own Env
  // implementation instead of relying on this default environment.
  //
-  // The result of Default() belongs to leveldb and must never be deleted.
+  // The result of Default() belongs to rocksdb and must never be deleted.
  static Env* Default();

  // Create a brand new sequentially-readable file with the specified name.
@@ -109,6 +110,15 @@ class Env {
                                 unique_ptr<WritableFile>* result,
                                 const EnvOptions& options) = 0;

+  // Create an object that both reads and writes to a file on
+  // specified offsets (random access). If file already exists,
+  // does not overwrite it. On success, stores a pointer to the
+  // new file in *result and returns OK. On failure stores nullptr
+  // in *result and returns non-OK.
+  virtual Status NewRandomRWFile(const std::string& fname,
+                                 unique_ptr<RandomRWFile>* result,
+                                 const EnvOptions& options) = 0;
+
  // Returns true iff the named file exists.
  virtual bool FileExists(const std::string& fname) = 0;

@@ -223,6 +233,9 @@ class Env {
  // Converts seconds-since-Jan-01-1970 to a printable string
  virtual std::string TimeToString(uint64_t time) = 0;

+  // Generates a unique id that can be used to identify a db
+  virtual std::string GenerateUniqueId();
+
 private:
  // No copying allowed
  Env(const Env&);
@@ -329,7 +342,7 @@ class WritableFile {

  /*
   * Sync data and/or metadata as well.
-   * By default, sync only metadata.
+   * By default, sync only data.
   * Override this method for environments where we need to sync
   * metadata as well.
   */
@@ -360,6 +373,11 @@ class WritableFile {
    *block_size = preallocation_block_size_;
  }

+  // For documentation, refer to RandomAccessFile::GetUniqueId()
+  virtual size_t GetUniqueId(char* id, size_t max_size) const {
+    return 0; // Default implementation to prevent issues with backwards
+  }
+
  // Remove any kind of caching of data from the offset to offset+length
  // of this file. If the length is 0, then it refers to the end of file.
  // If the system is not caching the file contents, then this is a noop.
@@ -418,6 +436,55 @@ class WritableFile {
  void operator=(const WritableFile&);
 };

+// A file abstraction for random reading and writing.
+class RandomRWFile {
+ public:
+  RandomRWFile() {}
+  virtual ~RandomRWFile() {}
+
+  // Write data from Slice data to file starting from offset
+  // Returns IOError on failure, but does not guarantee
+  // atomicity of a write.  Returns OK status on success.
+  //
+  // Safe for concurrent use.
+  virtual Status Write(uint64_t offset, const Slice& data) = 0;
+  // Read up to "n" bytes from the file starting at "offset".
+  // "scratch[0..n-1]" may be written by this routine.  Sets "*result"
+  // to the data that was read (including if fewer than "n" bytes were
+  // successfully read).  May set "*result" to point at data in
+  // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
+  // "*result" is used.  If an error was encountered, returns a non-OK
+  // status.
+  //
+  // Safe for concurrent use by multiple threads.
+  virtual Status Read(uint64_t offset, size_t n, Slice* result,
+                      char* scratch) const = 0;
+  virtual Status Close() = 0; // closes the file
+  virtual Status Sync() = 0; // sync data
+
+  /*
+   * Sync data and/or metadata as well.
+   * By default, sync only data.
+   * Override this method for environments where we need to sync
+   * metadata as well.
+   */
+  virtual Status Fsync() {
+    return Sync();
+  }
+
+  /*
+   * Pre-allocate space for a file.
+   */
+  virtual Status Allocate(off_t offset, off_t len) {
+    return Status::OK();
+  }
+
+ private:
+  // No copying allowed
+  RandomRWFile(const RandomRWFile&);
+  void operator=(const RandomRWFile&);
+};
+
 // An interface for writing log messages.
 class Logger {
 public:
@@ -430,6 +497,8 @@ class Logger {
  virtual size_t GetLogFileSize() const {
    return DO_NOT_SUPPORT_GET_LOG_FILE_SIZE;
  }
+  // Flush to the OS buffers
+  virtual void Flush() {}

 private:
  // No copying allowed
@@ -449,6 +518,9 @@ class FileLock {
  void operator=(const FileLock&);
 };

+
+extern void LogFlush(const shared_ptr<Logger>& info_log);
+
 // Log the specified data to *info_log if info_log is non-nullptr.
 extern void Log(const shared_ptr<Logger>& info_log, const char* format, ...)
 #   if defined(__GNUC__) || defined(__clang__)
@@ -456,6 +528,8 @@ extern void Log(const shared_ptr<Logger>& info_log, const char* format, ...)
 #   endif
    ;

+extern void LogFlush(Logger *info_log);
+
 extern void Log(Logger* info_log, const char* format, ...)
 #   if defined(__GNUC__) || defined(__clang__)
    __attribute__((__format__ (__printf__, 2, 3)))
@@ -497,6 +571,10 @@ class EnvWrapper : public Env {
                         const EnvOptions& options) {
    return target_->NewWritableFile(f, r, options);
  }
+  Status NewRandomRWFile(const std::string& f, unique_ptr<RandomRWFile>* r,
+                         const EnvOptions& options) {
+    return target_->NewRandomRWFile(f, r, options);
+  }
  bool FileExists(const std::string& f) { return target_->FileExists(f); }
  Status GetChildren(const std::string& dir, std::vector<std::string>* r) {
    return target_->GetChildren(dir, r);
@@ -565,6 +643,4 @@ class EnvWrapper : public Env {

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_ENV_H_
@@ -4,8 +4,8 @@
 //
 // A database can be configured with a custom FilterPolicy object.
 // This object is responsible for creating a small filter from a set
-// of keys.  These filters are stored in leveldb and are consulted
-// automatically by leveldb to decide whether or not to read some
+// of keys.  These filters are stored in rocksdb and are consulted
+// automatically by rocksdb to decide whether or not to read some
 // information from disk. In many cases, a filter can cut down the
 // number of disk seeks form a handful to a single disk seek per
 // DB::Get() call.
@@ -66,6 +66,5 @@ class FilterPolicy {
 extern const FilterPolicy* NewBloomFilterPolicy(int bits_per_key);

 }
-#include "rocksdb/rocksdb_to_leveldb.h"

 #endif  // STORAGE_ROCKSDB_INCLUDE_FILTER_POLICY_H_
@@ -99,6 +99,4 @@ extern Iterator* NewErrorIterator(const Status& status);

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_ITERATOR_H_
@@ -10,11 +10,6 @@ class LDBTool {
  void Run(int argc, char** argv, Options = Options());
 };

-namespace leveldb = rocksdb;
-
-
 } // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif // STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
@@ -10,16 +10,16 @@
 // The liberal use of assertions is encouraged to enforce (1).
 //
 // The factory will be passed an Arena object when a new MemTableRep is
-// requested. The API for this object is in leveldb/arena.h.
+// requested. The API for this object is in rocksdb/arena.h.
 //
 // Users can implement their own memtable representations. We include four
 // types built in:
 //  - SkipListRep: This is the default; it is backed by a skip list.
-//  - TransformRep: This is backed by an std::unordered_map<Slice,
-// std::set>. On construction, they are given a SliceTransform object. This
+//  - TransformRep: This is backed by an custom hash map.
+// On construction, they are given a SliceTransform object. This
 // object is applied to the user key of stored items which indexes into the
-// unordered map to yield a set containing all records that share the same user
-// key under the transform function.
+// hash map to yield a skiplist containing all records that share the same
+// user key under the transform function.
 //  - UnsortedRep: A subclass of TransformRep where the transform function is
 // the identity function. Optimized for point lookups.
 //  - PrefixHashRep: A subclass of TransformRep where the transform function is
@@ -130,6 +130,12 @@ class MemTableRep {
    return GetIterator();
  }

+  // Return an iterator that has a special Seek semantics. The result of
+  // a Seek might only include keys with the same prefix as the target key.
+  virtual std::shared_ptr<Iterator> GetDynamicPrefixIterator() {
+    return GetIterator();
+  }
+
 protected:
  // When *key is an internal key concatenated with the value, returns the
  // user key.
@@ -177,7 +183,7 @@ public:

 // TransformReps are backed by an unordered map of buffers to buckets. When
 // looking up a key, the user key is extracted and a user-supplied transform
-// function (see leveldb/slice_transform.h) is applied to get the key into the
+// function (see rocksdb/slice_transform.h) is applied to get the key into the
 // unordered map. This allows the user to bin user keys based on arbitrary
 // criteria. Two example implementations are UnsortedRepFactory and
 // PrefixHashRepFactory.
@@ -254,8 +260,68 @@ public:
  }
 };

+
+// NO LOCKS VERSION
+
+// The same as TransformRepFactory except it doesn't use locks.
+// Experimental, will replace TransformRepFactory once we are sure
+// it performs better
+class TransformRepNoLockFactory : public MemTableRepFactory {
+ public:
+  explicit TransformRepNoLockFactory(const SliceTransform* transform,
+    size_t bucket_count)
+    : transform_(transform),
+      bucket_count_(bucket_count) { }
+
+  virtual ~TransformRepNoLockFactory() { delete transform_; }
+
+  virtual std::shared_ptr<MemTableRep> CreateMemTableRep(
+    MemTableRep::KeyComparator&, Arena*) override;
+
+  virtual const char* Name() const override {
+    return "TransformRepNoLockFactory";
+  }
+
+  const SliceTransform* GetTransform() { return transform_; }
+
+ protected:
+  const SliceTransform* transform_;
+  const size_t bucket_count_;
+};
+
+// UnsortedReps bin user keys based on an identity function transform -- that
+// is, transform(key) = key. This optimizes for point look-ups.
+//
+// Parameters: See TransformRepNoLockFactory.
+class UnsortedRepNoLockFactory : public TransformRepNoLockFactory {
+public:
+  explicit UnsortedRepNoLockFactory(size_t bucket_count = 1000000)
+    : TransformRepNoLockFactory(NewNoopTransform(),
+                                bucket_count) { }
+  virtual const char* Name() const override {
+    return "UnsortedRepNoLockFactory";
+  }
+};
+
+// PrefixHashReps bin user keys based on a fixed-size prefix. This optimizes for
+// short ranged scans over a given prefix.
+//
+// Parameters: See TransformRepNoLockFactory.
+class PrefixHashRepNoLockFactory : public TransformRepNoLockFactory {
+public:
+  explicit PrefixHashRepNoLockFactory(const SliceTransform* prefix_extractor,
+    size_t bucket_count = 1000000)
+    : TransformRepNoLockFactory(prefix_extractor, bucket_count)
+    { }
+
+  virtual std::shared_ptr<MemTableRep> CreateMemTableRep(
+    MemTableRep::KeyComparator&, Arena*) override;
+
+  virtual const char* Name() const override {
+    return "PrefixHashRepNoLockFactory";
+  }
+};
+
 }

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif // STORAGE_ROCKSDB_DB_MEMTABLEREP_H_
@@ -7,7 +7,7 @@

 #include <string>
 #include <deque>
-#include "rocksdb/slice.h"  // TODO: Remove this when migration is done;
+#include "rocksdb/slice.h"

 namespace rocksdb {

@@ -144,6 +144,4 @@ class AssociativeMergeOperator : public MergeOperator {

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_MERGE_OPERATOR_H_
@@ -10,8 +10,12 @@
 #include <memory>
 #include <vector>
 #include <stdint.h>
+
+#include "rocksdb/memtablerep.h"
 #include "rocksdb/slice.h"
+#include "rocksdb/slice_transform.h"
 #include "rocksdb/statistics.h"
+#include "rocksdb/table_stats.h"
 #include "rocksdb/universal_compaction.h"
 #include "rocksdb/memtablerep.h"
 #include "rocksdb/slice_transform.h"
@@ -27,6 +31,7 @@ class MergeOperator;
 class Snapshot;
 class CompactionFilter;
 class CompactionFilterFactory;
+class TableFactory;

 using std::shared_ptr;

@@ -34,7 +39,7 @@ using std::shared_ptr;
 // sequence of key,value pairs.  Each block may be compressed before
 // being stored in a file.  The following enum describes which
 // compression method (if any) is used to compress a block.
-enum CompressionType {
+enum CompressionType : char {
  // NOTE: do not change the values of existing entries, as these are
  // part of the persistent format on disk.
  kNoCompression     = 0x0,
@@ -43,7 +48,7 @@ enum CompressionType {
  kBZip2Compression = 0x3
 };

-enum CompactionStyle {
+enum CompactionStyle : char {
  kCompactionStyleLevel       = 0x0, // level based compaction style
  kCompactionStyleUniversal   = 0x1  // Universal compaction style
 };
@@ -109,6 +114,9 @@ struct Options {
  // errors.  This may have unforeseen ramifications: for example, a
  // corruption of one DB entry may cause a large number of entries to
  // become unreadable or for the entire DB to become unopenable.
+  // If any of the  writes to the database fails (Put, Delete, Merge, Write),
+  // the database will switch to read-only mode and fail all other
+  // Write operations.
  // Default: false
  bool paranoid_checks;

@@ -165,10 +173,15 @@ struct Options {
  // a block is the unit of reading from disk).

  // If non-NULL use the specified cache for blocks.
-  // If NULL, leveldb will automatically create and use an 8MB internal cache.
+  // If NULL, rocksdb will automatically create and use an 8MB internal cache.
  // Default: nullptr
  shared_ptr<Cache> block_cache;

+  // If non-NULL use the specified cache for compressed blocks.
+  // If NULL, rocksdb will not use a compressed block cache.
+  // Default: nullptr
+  shared_ptr<Cache> block_cache_compressed;
+
  // Approximate size of user data packed per block.  Note that the
  // block size specified here corresponds to uncompressed data.  The
  // actual size of the unit read from disk may be smaller if
@@ -351,13 +364,21 @@ struct Options {
  // Default value is 1800 (half an hour).
  int db_stats_log_interval;

-  // This specifies the log dir.
+  // This specifies the info LOG dir.
  // If it is empty, the log files will be in the same dir as data.
  // If it is non empty, the log files will be in the specified dir,
  // and the db data dir's absolute path will be used as the log file
  // name's prefix.
  std::string db_log_dir;

+  // This specifies the absolute dir path for write-ahead logs (WAL).
+  // If it is empty, the log files will be in the same dir as data,
+  //   dbname is used as the data dir by default
+  // If it is non empty, the log files will be in kept the specified dir.
+  // When destroying the db,
+  //   all log files in wal_dir and the dir itself is deleted
+  std::string wal_dir;
+
  // Disable compaction triggered by seek.
  // With bloomfilter and fast storage, a miss on one level
  // is very cheap if the file handle is cached in table cache
@@ -434,6 +455,15 @@ struct Options {
  // Number of shards used for table cache.
  int table_cache_numshardbits;

+  // During data eviction of table's LRU cache, it would be inefficient
+  // to strictly follow LRU because this piece of memory will not really
+  // be released unless its refcount falls to zero. Instead, make two
+  // passes: the first pass will release items with refcount = 1,
+  // and if not enough space releases after scanning the number of
+  // elements specified by this parameter, we will remove items in LRU
+  // order.
+  int table_cache_remove_scan_count_limit;
+
  // size of one block in arena memory allocation.
  // If <= 0, a proper value is automatically calculated (usually 1/10 of
  // writer_buffer_size).
@@ -459,15 +489,20 @@ struct Options {
  // be issued on this database.
  bool disable_auto_compactions;

-  // The number of seconds a WAL(write ahead log) should be kept after it has
-  // been marked as Not Live. If the value is set. The WAL files are moved to
-  // the archive direcotory and deleted after the given TTL.
-  // If set to 0, WAL files are deleted as soon as they are not required by
-  // the database.
-  // If set to std::numeric_limits<uint64_t>::max() the WAL files will never be
-  // deleted.
-  // Default : 0
+  // The following two fields affect how archived logs will be deleted.
+  // 1. If both set to 0, logs will be deleted asap and will not get into
+  //    the archive.
+  // 2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0,
+  //    WAL files will be checked every 10 min and if total size is greater
+  //    then WAL_size_limit_MB, they will be deleted starting with the
+  //    earliest until size_limit is met. All empty files will be deleted.
+  // 3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then
+  //    WAL files will be checked every WAL_ttl_secondsi / 2 and those which
+  //    are older than WAL_ttl_seconds will be deleted.
+  // 4. If both are not 0, WAL files will be checked every 10 min and both
+  //    checks will be performed with ttl being first.
  uint64_t WAL_ttl_seconds;
+  uint64_t WAL_size_limit_MB;

  // Number of bytes to preallocate (via fallocate) the manifest
  // files.  Default is 4mb, which is reasonable to reduce random IO
@@ -497,7 +532,7 @@ struct Options {
  // Default: false
  bool skip_log_error_on_recovery;

-  // if not zero, dump leveldb.stats to LOG every stats_dump_period_sec
+  // if not zero, dump rocksdb.stats to LOG every stats_dump_period_sec
  // Default: 3600 (1 hour)
  unsigned int stats_dump_period_sec;

@@ -557,6 +592,11 @@ struct Options {
  // MemTableRep.
  std::shared_ptr<MemTableRepFactory> memtable_factory;

+  // This is a factory that provides TableFactory objects.
+  // Default: a factory that provides a default implementation of
+  // Table and TableBuilder.
+  std::shared_ptr<TableFactory> table_factory;
+
  // This is a factory that provides compaction filter objects which allow
  // an application to modify/delete a key-value during background compaction.
  // Default: a factory that doesn't provide any object
@@ -566,6 +606,23 @@ struct Options {
  // to data file.
  // Default: true
  bool purge_log_after_memtable_flush;
+
+  // This option allows user to to collect their own interested statistics of
+  // the tables.
+  // Default: emtpy vector -- no user-defined statistics collection will be
+  // performed.
+  std::vector<std::shared_ptr<TableStatsCollector>> table_stats_collectors;
+
+  // Allows thread-safe inplace updates. Requires Updates iff
+  // * key exists in current memtable
+  // * new sizeof(new_value) <= sizeof(old_value)
+  // * old_value for that key is a put i.e. kTypeValue
+  // Default: false.
+  bool inplace_update_support;
+
+  // Number of locks used for inplace update
+  // Default: 10000, if inplace_update_support = true, else 0.
+  size_t inplace_update_num_locks;
 };

 //
@@ -592,6 +649,10 @@ struct ReadOptions {
  // Default: true
  bool fill_cache;

+  // If this option is set and memtable implementation allows, Seek
+  // might only return keys with the same prefix as the seek-key
+  bool prefix_seek;
+
  // If "snapshot" is non-nullptr, read as of the supplied snapshot
  // (which must belong to the DB that is being read and which must
  // not have been released).  If "snapshot" is nullptr, use an impliicit
@@ -621,13 +682,14 @@ struct ReadOptions {
  ReadOptions()
      : verify_checksums(false),
        fill_cache(true),
+        prefix_seek(false),
        snapshot(nullptr),
        prefix(nullptr),
        read_tier(kReadAllTier) {
  }
  ReadOptions(bool cksum, bool cache) :
              verify_checksums(cksum), fill_cache(cache),
-              snapshot(nullptr), prefix(nullptr),
+              prefix_seek(false), snapshot(nullptr), prefix(nullptr),
              read_tier(kReadAllTier) {
  }
 };
@@ -675,6 +737,4 @@ struct FlushOptions {

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_
@@ -22,20 +22,22 @@ struct PerfContext {
  void Reset(); // reset all performance counters to zero

  uint64_t user_key_comparison_count; // total number of user key comparisons
-  uint64_t block_cache_hit_count;
-  uint64_t block_read_count;
-  uint64_t block_read_byte;
-  uint64_t block_read_time;
-  uint64_t block_checksum_time;
-  uint64_t block_decompress_time;
+  uint64_t block_cache_hit_count;     // total number of block cache hits
+  uint64_t block_read_count;          // total number of block reads (with IO)
+  uint64_t block_read_byte;           // total number of bytes from block reads
+  uint64_t block_read_time;           // total time spent on block reads
+  uint64_t block_checksum_time;       // total time spent on block checksum
+  uint64_t block_decompress_time;     // total time spent on block decompression
+  // total number of internal keys skipped over during iteration (overwritten or
+  // deleted, to be more specific, hidden by a put or delete of the same key)
  uint64_t internal_key_skipped_count;
+  // total number of deletes skipped over during iteration
  uint64_t internal_delete_skipped_count;
+  uint64_t wal_write_time;            // total time spent on writing to WAL
 };

 extern __thread PerfContext perf_context;

 }

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif
@@ -1,9 +0,0 @@
-// Copyright (c) 2013 Facebook.
-
-#pragma once
-
-//
-// This is for backward compatibility with applications that use the
-// 'leveldb' namespace. This file will be deleted in a future release.
-//
-namespace leveldb = rocksdb;
@@ -91,7 +91,7 @@ class Slice {
            (memcmp(data_, x.data_, x.size_) == 0));
  }

- // private: make these public for leveldbjni access
+ // private: make these public for rocksdbjni access
  const char* data_;
  size_t size_;

@@ -119,6 +119,4 @@ inline int Slice::compare(const Slice& b) const {

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_SLICE_H_
@@ -40,6 +40,4 @@ extern const SliceTransform* NewNoopTransform();

 }

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_SLICE_TRANSFORM_H_
@@ -39,6 +39,8 @@ enum Tickers {
  NUMBER_KEYS_WRITTEN,
  // Number of Keys read,
  NUMBER_KEYS_READ,
+  // Number keys updated, if inplace update is enabled
+  NUMBER_KEYS_UPDATED,
  // Bytes written / read
  BYTES_WRITTEN,
  BYTES_READ,
@@ -80,6 +82,9 @@ enum Tickers {
  // transaction log iterator refreshes
  GET_UPDATES_SINCE_CALLS,

+  BLOCK_CACHE_COMPRESSED_MISS, // miss in the compressed block cache
+  BLOCK_CACHE_COMPRESSED_HIT,  // hit in the compressed block cache
+
  TICKER_ENUM_MAX
 };

@@ -94,6 +99,7 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
  { COMPACTION_KEY_DROP_USER, "rocksdb.compaction.key.drop.user" },
  { NUMBER_KEYS_WRITTEN, "rocksdb.number.keys.written" },
  { NUMBER_KEYS_READ, "rocksdb.number.keys.read" },
+  { NUMBER_KEYS_UPDATED, "rocksdb.number.keys.updated" },
  { BYTES_WRITTEN, "rocksdb.bytes.written" },
  { BYTES_READ, "rocksdb.bytes.read" },
  { NO_FILE_CLOSES, "rocksdb.no.file.closes" },
@@ -113,7 +119,9 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
  { BLOOM_FILTER_PREFIX_CHECKED, "rocksdb.bloom.filter.prefix.checked" },
  { BLOOM_FILTER_PREFIX_USEFUL, "rocksdb.bloom.filter.prefix.useful" },
  { NUMBER_OF_RESEEKS_IN_ITERATION, "rocksdb.number.reseeks.iteration" },
-  { GET_UPDATES_SINCE_CALLS, "rocksdb.getupdatessince.calls" }
+  { GET_UPDATES_SINCE_CALLS, "rocksdb.getupdatessince.calls" },
+  { BLOCK_CACHE_COMPRESSED_MISS, "rocksdb.block.cachecompressed.miss" },
+  { BLOCK_CACHE_COMPRESSED_HIT, "rocksdb.block.cachecompressed.hit" }
 };

 /**
@@ -144,7 +152,7 @@ enum Histograms {
  HARD_RATE_LIMIT_DELAY_COUNT,
  SOFT_RATE_LIMIT_DELAY_COUNT,
  NUM_FILES_IN_SINGLE_COMPACTION,
-  HISTOGRAM_ENUM_MAX
+  HISTOGRAM_ENUM_MAX,
 };

 const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
@@ -165,7 +173,7 @@ const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
  { STALL_L0_NUM_FILES_COUNT, "rocksdb.num.files.stall.count"},
  { HARD_RATE_LIMIT_DELAY_COUNT, "rocksdb.hard.rate.limit.delay.count"},
  { SOFT_RATE_LIMIT_DELAY_COUNT, "rocksdb.soft.rate.limit.delay.count"},
-  { NUM_FILES_IN_SINGLE_COMPACTION, "rocksdb.numfiles.in.singlecompaction" }
+  { NUM_FILES_IN_SINGLE_COMPACTION, "rocksdb.numfiles.in.singlecompaction" },
 };

 struct HistogramData {
@@ -198,7 +206,7 @@ class Histogram {

 /**
 * A dumb ticker which keeps incrementing through its life time.
- * Not thread safe. Locking is currently managed by external leveldb lock
+ * Thread safe. Locking managed by implementation of this interface.
 */
 class Ticker {
 public:
@@ -260,6 +268,4 @@ inline void SetTickerCount(std::shared_ptr<Statistics> statistics,

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_STATISTICS_H_
@@ -123,6 +123,4 @@ inline void Status::operator=(const Status& s) {

 }  // namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_STATUS_H_
@@ -0,0 +1,180 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <memory>
+#include <stdint.h>
+#include "rocksdb/env.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/table_stats.h"
+#include "rocksdb/options.h"
+
+namespace rocksdb {
+
+struct Options;
+class RandomAccessFile;
+struct ReadOptions;
+class TableCache;
+class WritableFile;
+
+using std::unique_ptr;
+
+// TableBuilder provides the interface used to build a Table
+// (an immutable and sorted map from keys to values).
+//
+// Multiple threads can invoke const methods on a TableBuilder without
+// external synchronization, but if any of the threads may call a
+// non-const method, all threads accessing the same TableBuilder must use
+// external synchronization.
+class TableBuilder {
+ public:
+  // REQUIRES: Either Finish() or Abandon() has been called.
+  virtual ~TableBuilder() {}
+
+  // Add key,value to the table being constructed.
+  // REQUIRES: key is after any previously added key according to comparator.
+  // REQUIRES: Finish(), Abandon() have not been called
+  virtual void Add(const Slice& key, const Slice& value) = 0;
+
+  // Return non-ok iff some error has been detected.
+  virtual Status status() const = 0;
+
+  // Finish building the table.
+  // REQUIRES: Finish(), Abandon() have not been called
+  virtual Status Finish() = 0;
+
+  // Indicate that the contents of this builder should be abandoned.
+  // If the caller is not going to call Finish(), it must call Abandon()
+  // before destroying this builder.
+  // REQUIRES: Finish(), Abandon() have not been called
+  virtual void Abandon() = 0;
+
+  // Number of calls to Add() so far.
+  virtual uint64_t NumEntries() const = 0;
+
+  // Size of the file generated so far.  If invoked after a successful
+  // Finish() call, returns the size of the final generated file.
+  virtual uint64_t FileSize() const = 0;
+};
+
+// A Table is a sorted map from strings to strings.  Tables are
+// immutable and persistent.  A Table may be safely accessed from
+// multiple threads without external synchronization.
+class TableReader {
+ public:
+  virtual ~TableReader() {}
+
+  // Determine whether there is a chance that the current table file
+  // contains the key a key starting with iternal_prefix. The specific
+  // table implementation can use bloom filter and/or other heuristic
+  // to filter out this table as a whole.
+  virtual bool PrefixMayMatch(const Slice& internal_prefix) = 0;
+
+  // Returns a new iterator over the table contents.
+  // The result of NewIterator() is initially invalid (caller must
+  // call one of the Seek methods on the iterator before using it).
+  virtual Iterator* NewIterator(const ReadOptions&) = 0;
+
+  // Given a key, return an approximate byte offset in the file where
+  // the data for that key begins (or would begin if the key were
+  // present in the file).  The returned value is in terms of file
+  // bytes, and so includes effects like compression of the underlying data.
+  // E.g., the approximate offset of the last key in the table will
+  // be close to the file length.
+  virtual uint64_t ApproximateOffsetOf(const Slice& key) = 0;
+
+  // Returns true if the block for the specified key is in cache.
+  // REQUIRES: key is in this table.
+  virtual bool TEST_KeyInCache(const ReadOptions& options,
+                               const Slice& key) = 0;
+
+  // Set up the table for Compaction. Might change some parameters with
+  // posix_fadvise
+  virtual void SetupForCompaction() = 0;
+
+  virtual TableStats& GetTableStats() = 0;
+
+  // Calls (*result_handler)(handle_context, ...) repeatedly, starting with
+  // the entry found after a call to Seek(key), until result_handler returns
+  // false, where k is the actual internal key for a row found and v as the
+  // value of the key. didIO is true if I/O is involved in the operation. May
+  // not make such a call if filter policy says that key is not present.
+  //
+  // mark_key_may_exist_handler needs to be called when it is configured to be
+  // memory only and the key is not found in the block cache, with
+  // the parameter to be handle_context.
+  //
+  // readOptions is the options for the read
+  // key is the key to search for
+  virtual Status Get(
+      const ReadOptions& readOptions,
+      const Slice& key,
+      void* handle_context,
+      bool (*result_handler)(void* handle_context, const Slice& k,
+                             const Slice& v, bool didIO),
+      void (*mark_key_may_exist_handler)(void* handle_context) = nullptr) = 0;
+};
+
+// A base class for table factories
+class TableFactory {
+ public:
+  virtual ~TableFactory() {}
+
+  // The type of the table.
+  //
+  // The client of this package should switch to a new name whenever
+  // the table format implementation changes.
+  //
+  // Names starting with "rocksdb." are reserved and should not be used
+  // by any clients of this package.
+  virtual const char* Name() const = 0;
+
+  // Returns a Table object table that can fetch data from file specified
+  // in parameter file. It's the caller's responsibility to make sure
+  // file is in the correct format.
+  //
+  // GetTableReader() is called in two places:
+  // (1) TableCache::FindTable() calls the function when table cache miss
+  //     and cache the table object returned.
+  // (1) SstFileReader (for SST Dump) opens the table and dump the table
+  //     contents using the interator of the table.
+  // options and soptions are options. options is the general options.
+  // Multiple configured can be accessed from there, including and not
+  // limited to block cache and key comparators.
+  // file is a file handler to handle the file for the table
+  // file_size is the physical file size of the file
+  // table_reader is the output table reader
+  virtual Status GetTableReader(
+      const Options& options, const EnvOptions& soptions,
+      unique_ptr<RandomAccessFile> && file, uint64_t file_size,
+      unique_ptr<TableReader>* table_reader) const = 0;
+
+  // Return a table builder to write to a file for this table type.
+  //
+  // It is called in several places:
+  // (1) When flushing memtable to a level-0 output file, it creates a table
+  //     builder (In DBImpl::WriteLevel0Table(), by calling BuildTable())
+  // (2) During compaction, it gets the builder for writing compaction output
+  //     files in DBImpl::OpenCompactionOutputFile().
+  // (3) When recovering from transaction logs, it creates a table builder to
+  //     write to a level-0 output file (In DBImpl::WriteLevel0TableForRecovery,
+  //     by calling BuildTable())
+  // (4) When running Repairer, it creates a table builder to convert logs to
+  //     SST files (In Repairer::ConvertLogToTable() by calling BuildTable())
+  //
+  // options is the general options. Multiple configured can be acceseed from
+  // there, including and not limited to compression options.
+  // file is a handle of a writable file. It is the caller's responsibility to
+  // keep the file open and close the file after closing the table builder.
+  // compression_type is the compression type to use in this table.
+  virtual TableBuilder* GetTableBuilder(
+      const Options& options, WritableFile* file,
+      CompressionType compression_type) const = 0;
+};
+}  // namespace rocksdb
@@ -0,0 +1,66 @@
+// Copyright (c) 2013 Facebook
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+#include "rocksdb/status.h"
+
+namespace rocksdb {
+
+// TableStats contains a bunch of read-only stats of its associated
+// table.
+struct TableStats {
+ public:
+  // Other than basic table stats, each table may also have the user
+  // collected stats.
+  // The value of the user-collected stats are encoded as raw bytes --
+  // users have to interprete these values by themselves.
+  typedef
+    std::unordered_map<std::string, std::string>
+    UserCollectedStats;
+
+  // the total size of all data blocks.
+  uint64_t data_size = 0;
+  // the total size of all index blocks.
+  uint64_t index_size = 0;
+  // total raw key size
+  uint64_t raw_key_size = 0;
+  // total raw value size
+  uint64_t raw_value_size = 0;
+  // the number of blocks in this table
+  uint64_t num_data_blocks = 0;
+  // the number of entries in this table
+  uint64_t num_entries = 0;
+
+  // The name of the filter policy used in this table.
+  // If no filter policy is used, `filter_policy_name` will be an empty string.
+  std::string filter_policy_name;
+
+  // user collected stats
+  UserCollectedStats user_collected_stats;
+};
+
+// `TableStatsCollector` provides the mechanism for users to collect their own
+// interested stats. This class is essentially a collection of callback
+// functions that will be invoked during table building.
+class TableStatsCollector {
+ public:
+  virtual ~TableStatsCollector() { }
+  // Add() will be called when a new key/value pair is inserted into the table.
+  // @params key    the original key that is inserted into the table.
+  // @params value  the original value that is inserted into the table.
+  virtual Status Add(const Slice& key, const Slice& value) = 0;
+
+  // Finish() will be called when a table has already been built and is ready
+  // for writing the stats block.
+  // @params stats  User will add their collected statistics to `stats`.
+  virtual Status Finish(TableStats::UserCollectedStats* stats) = 0;
+
+  // The name of the stats collector can be used for debugging purpose.
+  virtual const char* Name() const = 0;
+};
+
+}  // namespace rocksdb
@@ -5,6 +5,8 @@
 #include "rocksdb/status.h"
 #include "rocksdb/types.h"
 #include "rocksdb/write_batch.h"
+#include <memory>
+#include <vector>

 namespace rocksdb {

@@ -14,7 +16,9 @@ typedef std::vector<std::unique_ptr<LogFile>> VectorLogPtr;
 enum  WalFileType {
  /* Indicates that WAL file is in archive directory. WAL files are moved from
   * the main db directory to archive directory once they are not live and stay
-   * there for a duration of WAL_ttl_seconds which can be set in Options
+   * there until cleaned up. Files are cleaned depending on archive size
+   * (Options::WAL_size_limit_MB) and time since last cleaning
+   * (Options::WAL_ttl_seconds).
   */
  kArchivedLogFile = 0,

@@ -52,7 +56,9 @@ struct BatchResult {
  std::unique_ptr<WriteBatch> writeBatchPtr;
 };

-//  A TransactionLogIterator is used to iterate over the Transaction's in a db.
+// A TransactionLogIterator is used to iterate over the transactions in a db.
+// One run of the iterator is continuous, i.e. the iterator will stop at the
+// beginning of any gap in sequences
 class TransactionLogIterator {
 public:
  TransactionLogIterator() {}
@@ -67,17 +73,15 @@ class TransactionLogIterator {
  // REQUIRES: Valid() to be true.
  virtual void Next() = 0;

-  // Return's ok if the iterator is valid.
-  // Return the Error when something has gone wrong.
+  // Returns ok if the iterator is valid.
+  // Returns the Error when something has gone wrong.
  virtual Status status() = 0;

  // If valid return's the current write_batch and the sequence number of the
-  // latest transaction contained in the batch.
+  // earliest transaction contained in the batch.
  // ONLY use if Valid() is true and status() is OK.
  virtual BatchResult GetBatch() = 0;
 };
 } //  namespace rocksdb

-#include "rocksdb/rocksdb_to_leveldb.h"
-
 #endif  // STORAGE_ROCKSDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_