Add universal compaction to db_stress nightly build

Summary:
Most code change in this diff is code cleanup/rewrite. The logic changes include:

(1) add universal compaction to db_crashtest2.py
(2) randomly set --test_batches_snapshots to be 0 or 1 in db_crashtest2.py. Old codes always use 1.
(3) use different tmp directory as db directory in different runs. I saw some intermittent errors in my local tests. Use of different tmp directory seems to be able to solve the issue.

Test Plan: Have run "make crashtest" for multiple times. Also run "make all check"

Reviewers: emayanke, dhruba, haobo

Reviewed By: emayanke

Differential Revision: https://reviews.facebook.net/D12369

.arcconfig

@@ -1,5 +1,10 @@
 {
   "project_id" : "leveldb",
   "conduit_uri" : "https://reviews.facebook.net/",
-  "copyright_holder" : ""
+  "copyright_holder" : "",
+  "load" : [
+    "linters/src/"
+  ],
+  "lint.engine" : "FacebookFbcodeLintEngine",
+  "lint.engine.single.linter" : "FbcodeCppLinter"
 }

.gitignore

@@ -1,12 +1,16 @@
 build_config.mk
 *.a
-*.o
+*.arc
+*.d
 *.dylib*
+*.gcda
+*.gcno
+*.o
 *.so
 *.so.*
 *_test
-*.arc
 db_bench
+db_repl_stress
 db_stress
 ldb
 leveldb_server
@@ -14,4 +18,6 @@ leveldb_shell
 manifest_dump
 sst_dump
 util/build_version.cc
-db_repl_stress
+build_tools/VALGRIND_LOGS/
+coverage/COVERAGE_REPORT
+util/build_version.cc.tmp

AUTHORS

@@ -1,8 +0,0 @@
-# Names should be added to this file like so:
-# Name or Organization <email address>
-
-Google Inc.
-
-# Initial version authors:
-Jeffrey Dean <jeff@google.com>
-Sanjay Ghemawat <sanjay@google.com>

Makefile

@@ -9,18 +9,18 @@ INSTALL_PATH ?= $(CURDIR)
 # Uncomment exactly one of the lines labelled (A), (B), and (C) below
 # to switch between compilation modes.
 
-# OPT ?= -O2 -DNDEBUG       # (A) Production use (optimized mode)
-# OPT ?= -g2              # (B) Debug mode, w/ full line-level debugging symbols
-OPT ?= -O2 -g2 -DNDEBUG -Wall # (C) Profiling mode: opt, but w/debugging symbols
+# OPT ?= -DNDEBUG     # (A) Production use (optimized mode)
+OPT += -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer
 #-----------------------------------------------
 
 # detect what platform we're building on
-$(shell ./build_detect_platform build_config.mk)
+$(shell (cd build_tools/; ROCKSDB_ROOT=.. ./build_detect_platform ../build_config.mk))
 # this file is generated by the previous line to set build flags and sources
 include build_config.mk
 
-CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
-CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT)
+WARNING_FLAGS = -Wall -Werror
+CFLAGS += -g $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
+CXXFLAGS += -g $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -std=gnu++0x -Woverloaded-virtual
 
 LDFLAGS += $(PLATFORM_LDFLAGS)
 
@@ -30,10 +30,15 @@ MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
 
 TESTUTIL = ./util/testutil.o
 TESTHARNESS = ./util/testharness.o $(TESTUTIL)
-
+VALGRIND_ERROR = 2
+VALGRIND_DIR = build_tools/VALGRIND_LOGS
+VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)
+VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full
 
 TESTS = \
 	arena_test \
+	auto_roll_logger_test \
+	block_test \
 	bloom_test \
 	c_test \
 	cache_test \
@@ -43,31 +48,37 @@ TESTS = \
 	db_test \
 	dbformat_test \
 	env_test \
+	filelock_test \
 	filename_test \
 	filter_block_test \
+	histogram_test \
 	log_test \
+	manual_compaction_test \
 	memenv_test \
+	merge_test \
+	redis_test \
+	reduce_levels_test \
 	skiplist_test \
+	stringappend_test \
 	table_test \
-	block_test \
+	ttl_test \
 	version_edit_test \
 	version_set_test \
-	reduce_levels_test \
-	write_batch_test \
-	filelock_test
+	write_batch_test
 
 TOOLS = \
-	manifest_dump \
         sst_dump \
         db_stress \
         ldb \
 	db_repl_stress
 
-PROGRAMS = db_bench $(TESTS) $(TOOLS)
+PROGRAMS = db_bench signal_test $(TESTS) $(TOOLS)
 BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
-VERSIONFILE=util/build_version.cc
 
-LIBRARY = libleveldb.a
+# The library name is configurable since we are maintaining libraries of both
+# debug/release mode.
+LIBNAME = librocksdb
+LIBRARY = ${LIBNAME}.a
 MEMENVLIBRARY = libmemenv.a
 
 default: all
@@ -76,15 +87,15 @@ default: all
 ifneq ($(PLATFORM_SHARED_EXT),)
 
 ifneq ($(PLATFORM_SHARED_VERSIONED),true)
-SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
+SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT)
 SHARED2 = $(SHARED1)
 SHARED3 = $(SHARED1)
 SHARED = $(SHARED1)
 else
 # Update db.h if you change these.
-SHARED_MAJOR = 1
-SHARED_MINOR = 5
-SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
+SHARED_MAJOR = 2
+SHARED_MINOR = 0
+SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT)
 SHARED2 = $(SHARED1).$(SHARED_MAJOR)
 SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
 SHARED = $(SHARED1) $(SHARED2) $(SHARED3)
@@ -95,131 +106,197 @@ $(SHARED2): $(SHARED3)
 endif
 
 $(SHARED3):
-	$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3)
+	$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(COVERAGEFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3)
 
 endif  # PLATFORM_SHARED_EXT
 
-all: $(VERSIONFILE) $(SHARED) $(LIBRARY) $(TOOLS)
+all: $(SHARED) $(LIBRARY) $(PROGRAMS)
 
-check: all $(PROGRAMS) $(TESTS) $(TOOLS)
+.PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests \
+	release tags valgrind_check whitebox_crash_test
+
+release:
+	$(MAKE) clean
+	OPT=-DNDEBUG $(MAKE) -j32
+
+coverage:
+	$(MAKE) clean
+	COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" $(MAKE) all check
+	(cd coverage; ./coverage_test.sh)
+	# Delete intermediate files
+	find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
+
+check: all $(PROGRAMS) $(TESTS) $(TOOLS) ldb_tests
 	for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
 
-clean:
-	-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) $(THRIFTSERVER) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o build_config.mk $(VERSIONFILE)
-	-rm -rf ios-x86/* ios-arm/*
+ldb_tests: all $(PROGRAMS) $(TOOLS)
+	python tools/ldb_test.py
 
+crash_test: blackbox_crash_test whitebox_crash_test
+
+blackbox_crash_test: db_stress
+	python -u tools/db_crashtest.py
+
+whitebox_crash_test: db_stress
+	python -u tools/db_crashtest2.py
+
+valgrind_check: all $(PROGRAMS) $(TESTS)
+	mkdir -p $(VALGRIND_DIR)
+	echo TESTS THAT HAVE VALGRIND ERRORS > $(VALGRIND_DIR)/valgrind_failed_tests; \
+	echo TIMES in seconds TAKEN BY TESTS ON VALGRIND > $(VALGRIND_DIR)/valgrind_tests_times; \
+	for t in $(filter-out skiplist_test,$(TESTS)); do \
+		stime=`date '+%s'`; \
+		$(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \
+		if [ $$? -eq $(VALGRIND_ERROR) ] ; then \
+			echo $$t >> $(VALGRIND_DIR)/valgrind_failed_tests; \
+		fi; \
+		etime=`date '+%s'`; \
+		echo $$t $$((etime - stime)) >> $(VALGRIND_DIR)/valgrind_tests_times; \
+	done
+
+clean:
+	-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) $(THRIFTSERVER) build_config.mk
+	-rm -rf ios-x86/* ios-arm/*
+	-find . -name "*.[od]" | xargs --no-run-if-empty rm
+	-find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
+tags:
+	ctags * -R
+	cscope -b `find . -name '*.cc'` `find . -name '*.h'`
+
+# ---------------------------------------------------------------------------
+# 	Unit tests and tools
+# ---------------------------------------------------------------------------
 $(LIBRARY): $(LIBOBJECTS)
 	rm -f $@
 	$(AR) -rs $@ $(LIBOBJECTS)
 
-
 db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS)
+	$(CXX) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 
 db_stress: tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS)
+	$(CXX) tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 
 db_repl_stress: tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS)
+	$(CXX) tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 
 db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LDFLAGS) -lsqlite3
+	$(CXX) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lsqlite3 $(COVERAGEFLAGS)
 
 db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LDFLAGS) -lkyotocabinet
+	$(CXX) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lkyotocabinet $(COVERAGEFLAGS)
+
+signal_test: util/signal_test.o $(LIBOBJECTS)
+	$(CXX) util/signal_test.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
+stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
+redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
+histogram_test: util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o$@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 corruption_test: db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
+perf_context_test: db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+
+ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 
 dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 block_test: table/block_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) table/block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) table/block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 reduce_levels_test: tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
+merge_test: db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 $(MEMENVLIBRARY) : $(MEMENVOBJECTS)
 	rm -f $@
 	$(AR) -rs $@ $(MEMENVOBJECTS)
 
 memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS)
-	$(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
-leveldb_shell: tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o tools/shell/ShellContext.h tools/shell/ShellState.h tools/shell/DBClientProxy.h $(LIBOBJECTS)
-	$(CXX) tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o $(LIBOBJECTS) -o $@ $(LDFLAGS)
+manual_compaction_test: util/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) util/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
-DBClientProxy_test: tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY) 
-	$(CXX) tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS)
+rocksdb_shell: tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o tools/shell/ShellContext.h tools/shell/ShellState.h tools/shell/DBClientProxy.h $(LIBOBJECTS)
+	$(CXX) tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
-manifest_dump: tools/manifest_dump.o $(LIBOBJECTS)
-	$(CXX) tools/manifest_dump.o $(LIBOBJECTS) -o $@ $(LDFLAGS)
+DBClientProxy_test: tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY)
+	$(CXX) tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY) $(EXEC_LDFLAGS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 
 filelock_test: util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LDFLAGS)
+	$(CXX) util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
+auto_roll_logger_test: util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
-	$(CXX) tools/sst_dump.o $(LIBOBJECTS) -o $@ $(LDFLAGS)
+	$(CXX) tools/sst_dump.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
 ldb: tools/ldb.o $(LIBOBJECTS)
-	$(CXX) tools/ldb.o $(LIBOBJECTS) -o $@ $(LDFLAGS)
+	$(CXX) tools/ldb.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
-# recreate the version file with the latest git revision
-$(VERSIONFILE):	build_detect_version
-	$(shell ./build_detect_platform build_config.mk)
-
-# recreate the version file with the latest git revision
-$(VERSIONFILE):	build_detect_version
-	$(shell ./build_detect_platform build_config.mk)
+# ---------------------------------------------------------------------------
+#  	Platform-specific compilation
+# ---------------------------------------------------------------------------
 
 ifeq ($(PLATFORM), IOS)
 # For iOS, create universal object files to be used on both the simulator and
@@ -231,9 +308,9 @@ IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/versionCFBun
 
 .cc.o:
 	mkdir -p ios-x86/$(dir $@)
-	$(SIMULATORROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
+	$(SIMULATORROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@ $(COVERAGEFLAGS)
 	mkdir -p ios-arm/$(dir $@)
-	$(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
+	$(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@ $(COVERAGEFLAGS)
 	lipo ios-x86/$@ ios-arm/$@ -create -output $@
 
 .c.o:
@@ -245,8 +322,31 @@ IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/versionCFBun
 
 else
 .cc.o:
-	$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
+	$(CXX) $(CXXFLAGS) -c $< -o $@ $(COVERAGEFLAGS)
 
 .c.o:
-	$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
+	$(CC) $(CFLAGS) -c $< -o $@
+endif
+
+# ---------------------------------------------------------------------------
+#  	Source files dependencies detection
+# ---------------------------------------------------------------------------
+
+# Add proper dependency support so changing a .h file forces a .cc file to
+# rebuild.
+
+# The .d file indicates .cc file's dependencies on .h files. We generate such
+# dependency by g++'s -MM option, whose output is a make dependency rule.
+# The sed command makes sure the "target" file in the generated .d file has
+# the correct path prefix.
+%.d: %.cc
+	$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -MM $< -o $@
+	@sed -i -e 's|.*:|$*.o:|' $@
+
+DEPFILES = $(filter-out util/build_version.d,$(SOURCES:.cc=.d))
+
+depend: $(DEPFILES)
+
+ifneq ($(MAKECMDGOALS),clean)
+-include $(DEPFILES)
 endif

NEWS

@@ -1,17 +0,0 @@
-Release 1.2 2011-05-16
-----------------------
-
-Fixes for larger databases (tested up to one billion 100-byte entries,
-i.e., ~100GB).
-
-(1) Place hard limit on number of level-0 files.  This fixes errors
-of the form "too many open files".
-
-(2) Fixed memtable management.  Before the fix, a heavy write burst
-could cause unbounded memory usage.
-
-A fix for a logging bug where the reader would incorrectly complain
-about corruption.
-
-Allow public access to WriteBatch contents so that users can easily
-wrap a DB.

README

@@ -1,7 +1,9 @@
 rocksdb: A persistent key-value store for flash storage
-Authors: The Facebook Database Engineering Team
+Authors: * The Facebook Database Engineering Team
+         * Build on earlier work on leveldb by Sanjay Ghemawat 
+           (sanjay@google.com) and Jeff Dean (jeff@google.com)
 
-This code is a library that forms the core building block for a fast 
+This code is a library that forms the core building block for a fast
 key value server, especially suited for storing data on flash drives.
 It has an Log-Stuctured-Merge-Database (LSM) design with flexible tradeoffs
 between Write-Amplification-Factor(WAF), Read-Amplification-Factor (RAF)
@@ -11,9 +13,6 @@ single database.
 
 The core of this code has been derived from open-source leveldb.
 
-leveldb: A key-value store
-Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
-
 The code under this directory implements a system for maintaining a
 persistent key/value store.
 
@@ -59,6 +58,25 @@ include/env.h
     Abstraction of the OS environment.  A posix implementation of
     this interface is in util/env_posix.cc
 
-include/table.h
 include/table_builder.h
     Lower-level modules that most clients probably won't use directly
+
+include/cache.h
+    An API for the block cache.
+
+include/compaction_filter.h
+    An API for a application filter invoked on every compaction.
+
+include/filter_policy.h
+    An API for configuring a bloom filter.
+
+include/memtablerep.h
+    An API for implementing a memtable.
+
+include/statistics.h
+    An API to retrieve various database statistics.
+
+include/transaction_log_iterator.h
+    An API to retrieve transaction logs from a database.
+
+

README.fb

@@ -1,12 +1,3 @@
-* How to compile using fbcode and jemalloc
-  source fbcode.sh
-  make 
+* Detailed instructions on how to compile using fbcode and jemalloc
 
-* Compiling for CPUs with SSE4 support
-
-  make OPT='-O2 -DNDEBUG -msse -msse4.2'
-  
-  This makes CRC computation much faster, but
-  binaries won't run on CPUs that don't support it.
-
-* Latest release is 1.5.7.fb
+* Latest release is 2.1.fb

TODO

@@ -1,13 +0,0 @@
-ss
-- Stats
-
-db
-- Maybe implement DB::BulkDeleteForRange(start_key, end_key)
-  that would blow away files whose ranges are entirely contained
-  within [start_key..end_key]?  For Chrome, deletion of obsolete
-  object stores, etc. can be done in the background anyway, so
-  probably not that important.
-
-After a range is completely deleted, what gets rid of the
-corresponding files if we do no future changes to that range.  Make
-the conditions for triggering compactions fire in more situations?

build_detect_version

@@ -1,24 +0,0 @@
-#!/bin/sh
-#
-# Record the version of the source that we are compiling.
-# We keep a record of the git revision in util/version.cc. This source file
-# is then built as a regular source file as part of the compilation process.
-# One can run "strings executable_filename | grep _build_" to find the version of
-# the source that we used to build the executable file.
-#
-
-# create git version file
-VFILE=util/build_version.cc
-
-# check to see if git is in the path
-which git > /dev/null
-
-if [ "$?" = 0 ]; then
-  git rev-parse HEAD | awk ' BEGIN {print "#include \"build_version.h\""} {print "const char * leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";"} END {}' >  ${VFILE}
-else
-  echo "git not found"| awk ' BEGIN {print "#include \"build_version.h\""} {print "const char * leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";"} END {}' > ${VFILE}
-fi
-
-date | awk 'BEGIN {} {print "const char * leveldb_build_git_datetime = \"leveldb_build_git_datetime:"$0"\";"} END {} ' >> ${VFILE}
-echo "const char * leveldb_build_compile_date = __DATE__;" >> ${VFILE}
-echo "const char * leveldb_build_compile_time = __TIME__;" >> ${VFILE}

build_tools/build_detect_platform

@@ -28,6 +28,15 @@ if test -z "$OUTPUT"; then
   exit 1
 fi
 
+# Default to fbcode gcc on internal fb machines
+if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
+    if [ -z "$USE_CLANG" ]; then
+        source ./fbcode.gcc471.sh
+    else
+        source ./fbcode.clang31.sh
+    fi
+fi
+
 # Delete existing output, if it exists
 rm -f $OUTPUT
 touch $OUTPUT
@@ -45,66 +54,72 @@ if test -z "$TARGET_OS"; then
     TARGET_OS=`uname -s`
 fi
 
-COMMON_FLAGS=
+COMMON_FLAGS="${CFLAGS}"
 CROSS_COMPILE=
 PLATFORM_CCFLAGS=
-PLATFORM_CXXFLAGS=
-PLATFORM_LDFLAGS=
+PLATFORM_CXXFLAGS="${CXXFLAGS}"
+PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS"
 PLATFORM_SHARED_EXT="so"
-PLATFORM_SHARED_LDFLAGS="-shared -Wl,-soname -Wl,"
+PLATFORM_SHARED_LDFLAGS="${EXEC_LDFLAGS_SHARED} -shared -Wl,-soname -Wl,"
 PLATFORM_SHARED_CFLAGS="-fPIC"
 PLATFORM_SHARED_VERSIONED=true
 
+# generic port files (working on all platform by #ifdef) go directly in /port
+GENERIC_PORT_FILES=`cd $ROCKSDB_ROOT; find port -name '*.cc' | tr "\n" " "`
+
 # On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
 case "$TARGET_OS" in
     Darwin)
         PLATFORM=OS_MACOSX
-        COMMON_FLAGS="-fno-builtin-memcmp -DOS_MACOSX"
+        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -DOS_MACOSX"
         PLATFORM_SHARED_EXT=dylib
         PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
-        PORT_FILE=port/port_posix.cc
+        # PORT_FILES=port/darwin/darwin_specific.cc
         ;;
     Linux)
         PLATFORM=OS_LINUX
-        COMMON_FLAGS="-I/usr/include -fno-builtin-memcmp -pthread -DOS_LINUX -fPIC"
-        PLATFORM_LDFLAGS="-pthread"
-        PORT_FILE=port/port_posix.cc
+        COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX"
+        if [ -z "$USE_CLANG" ]; then
+            COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
+        fi
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
+        # PORT_FILES=port/linux/linux_specific.cc
         ;;
     SunOS)
         PLATFORM=OS_SOLARIS
-        COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS"
-        PLATFORM_LDFLAGS="-lpthread -lrt"
-        PORT_FILE=port/port_posix.cc
+        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
+        # PORT_FILES=port/sunos/sunos_specific.cc
         ;;
     FreeBSD)
         PLATFORM=OS_FREEBSD
-        COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
-        PLATFORM_LDFLAGS="-lpthread"
-        PORT_FILE=port/port_posix.cc
+        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
+        # PORT_FILES=port/freebsd/freebsd_specific.cc
         ;;
     NetBSD)
         PLATFORM=OS_NETBSD
-        COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
-        PLATFORM_LDFLAGS="-lpthread -lgcc_s"
-        PORT_FILE=port/port_posix.cc
+        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lgcc_s"
+        # PORT_FILES=port/netbsd/netbsd_specific.cc
         ;;
     OpenBSD)
         PLATFORM=OS_OPENBSD
-        COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
-        PLATFORM_LDFLAGS="-pthread"
-        PORT_FILE=port/port_posix.cc
+        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread"
+        # PORT_FILES=port/openbsd/openbsd_specific.cc
         ;;
     DragonFly)
         PLATFORM=OS_DRAGONFLYBSD
-        COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD"
-        PLATFORM_LDFLAGS="-lpthread"
-        PORT_FILE=port/port_posix.cc
+        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
+        # PORT_FILES=port/dragonfly/dragonfly_specific.cc
         ;;
     OS_ANDROID_CROSSCOMPILE)
         PLATFORM=OS_ANDROID
-	COMMON_FLAGS="-fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
-	PLATFORM_LDFLAGS=""  # All pthread features are in the Android C library
-        PORT_FILE=port/port_posix.cc
+	COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
+	PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS "  # All pthread features are in the Android C library
+        # PORT_FILES=port/android/android.cc
         CROSS_COMPILE=true
         ;;
     *)
@@ -118,7 +133,7 @@ esac
 # except for the test and benchmark files. By default, find will output a list
 # of all files matching either rule, so we need to append -print to make the
 # prune take effect.
-DIRS="util db table"
+DIRS="util db table utilities"
 if test "$USE_THRIFT"; then
   DIRS="$DIRS thrift/server_utils.cpp thrift/gen-cpp "
   THRIFTSERVER=leveldb_server
@@ -131,13 +146,13 @@ fi
 set -f # temporarily disable globbing so that our patterns arent expanded
 PRUNE_TEST="-name *test*.cc -prune"
 PRUNE_BENCH="-name *_bench.cc -prune"
-PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
-PORTABLE_CPP=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
+PORTABLE_FILES=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
+PORTABLE_CPP=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
 set +f # re-enable globbing
 
 # The sources consist of the portable files, plus the platform-specific port
 # file.
-echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT
+echo "SOURCES=$PORTABLE_FILES $GENERIC_PORT_FILES $PORT_FILES" >> $OUTPUT
 echo "SOURCESCPP=$PORTABLE_CPP" >> $OUTPUT
 echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT
 
@@ -226,10 +241,13 @@ COMMON_FLAGS="$COMMON_FLAGS $USE_SSE"
 PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
 
+VALGRIND_VER="$VALGRIND_VER"
+
 echo "CC=$CC" >> $OUTPUT
 echo "CXX=$CXX" >> $OUTPUT
 echo "PLATFORM=$PLATFORM" >> $OUTPUT
 echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> $OUTPUT
+echo "VALGRIND_VER=$VALGRIND_VER" >> $OUTPUT
 echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> $OUTPUT
 echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> $OUTPUT
 echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
@@ -237,3 +255,4 @@ echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT
 echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT
 echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT
 echo "THRIFTSERVER=$THRIFTSERVER" >> $OUTPUT
+echo "EXEC_LDFLAGS=$EXEC_LDFLAGS" >> $OUTPUT

build_tools/build_detect_version

@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+# Record the version of the source that we are compiling.
+# We keep a record of the git revision in util/version.cc. This source file
+# is then built as a regular source file as part of the compilation process.
+# One can run "strings executable_filename | grep _build_" to find the version of
+# the source that we used to build the executable file.
+#
+
+# create git version file
+VFILE=$ROCKSDB_ROOT/util/build_version.cc.tmp
+trap "rm $VFILE" EXIT
+
+# check to see if git is in the path
+which git > /dev/null
+
+if [ "$?" = 0 ]; then
+  env -i git rev-parse HEAD |
+  awk '
+  BEGIN {
+    print "#include \"build_version.h\"\n"
+  } 
+  { print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";" }
+  ' >  ${VFILE}
+else
+  echo "git not found" |
+  awk '
+    BEGIN {
+      print "#include \"build_version.h\"" 
+    } 
+    { print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";" }
+    ' > ${VFILE}
+fi
+
+echo "const char* leveldb_build_git_datetime = \"leveldb_build_git_datetime:$(date)\";" >> ${VFILE}
+echo "const char* leveldb_build_compile_date = __DATE__;" >> ${VFILE}
+echo "const char* leveldb_build_compile_time = __TIME__;" >> ${VFILE}
+
+OUTFILE=$ROCKSDB_ROOT/util/build_version.cc
+if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
+    cp $VFILE $OUTFILE
+fi

build_tools/fbcode.clang31.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+#
+# Set environment variables so that we can compile leveldb using
+# fbcode settings.  It uses the latest g++ compiler and also
+# uses jemalloc
+
+TOOLCHAIN_REV=f365dbeae46a30414a2874a6f45e73e10f1caf7d
+TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
+TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
+TOOL_JEMALLOC=jemalloc-3.0.0/2f45f3a
+GLIBC_RUNTIME_PATH=/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1
+
+# location of snappy headers and libraries
+SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include"
+SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
+
+# location of boost headers and libraries
+THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/include -std=gnu++0x"
+THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp"
+THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/lib"
+
+# location of libevent
+LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include"
+LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
+
+# use Intel SSE support for checksum calculations
+export USE_SSE=" -msse -msse4.2 "
+
+CC="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang $CLANG_INCLUDES"
+CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
+AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
+RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
+
+CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin -nostdlib -nostdinc -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1 -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1/x86_64-facebook-linux -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1/backward -isystem $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/include -isystem $TOOLCHAIN_LIB_BASE/clang/clang-3.1/c8f7279/lib/clang/3.1/include -isystem $TOOLCHAIN_LIB_BASE/kernel-headers/kernel-headers-3.2.18_70_fbk11_00129_gc8882d0/da39a3e/include/linux -isystem $TOOLCHAIN_LIB_BASE/kernel-headers/kernel-headers-3.2.18_70_fbk11_00129_gc8882d0/da39a3e/include -Wall -Wno-sign-compare -Wno-unused-variable -Winvalid-pch -Wno-deprecated -Woverloaded-virtual"
+CXXFLAGS="$CFLAGS -nostdinc++ -std=gnu++0x"
+
+CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
+
+EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
+EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a"
+EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,$GLIBC_RUNTIME_PATH/lib/ld-linux-x86-64.so.2"
+EXEC_LDFLAGS+=" -B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin"
+
+PLATFORM_LDFLAGS="-L$TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/lib -L$TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS"
+SNAPPY_LDFLAGS="$SNAPPY_LIBS"
+
+export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED SNAPPY_LDFLAGS

build_tools/fbcode.gcc471.sh

@@ -3,12 +3,11 @@
 # Set environment variables so that we can compile leveldb using
 # fbcode settings.  It uses the latest g++ compiler and also
 # uses jemalloc
-# This is compiled with gcc version 4.7.1 for zeus proxy
 
-TOOLCHAIN_REV=20d3328ac30f633840ce819ad03019f415267a86
+TOOLCHAIN_REV=83eb773e262fa705eaebbf3de40db71d53fabf2e
 TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
 TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
-TOOL_JEMALLOC=jemalloc-3.0.0/69dc57c
+TOOL_JEMALLOC=jemalloc-3.3.1/2f45f3a
 
 # location of libhdfs libraries
 if test "$USE_HDFS"; then
@@ -24,10 +23,14 @@ fi
 SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include"
 SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
 
+# location of zlib headers and libraries
+ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include"
+ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a"
+
 # location of boost headers and libraries
-THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/include -std=gnu++0x"
+THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/include"
 THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp"
-THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/lib"
+THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/lib"
 
 # location of libevent
 LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include"
@@ -37,7 +40,7 @@ LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
 export USE_SSE=" -msse -msse4.2 "
 
 CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/gcc"
-CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
+CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
 AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
 RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
 
@@ -45,9 +48,14 @@ CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/bin/gold -m64 -mtune=g
 CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
 
 EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
-EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-20100812_experimental/91ddd43/lib/libunwind.a"
-EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.so"
-SNAPPY_LDFLAGS="$SNAPPY_LIBS"
+EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a"
+EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
 
-export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED SNAPPY_LDFLAGS
+PLATFORM_LDFLAGS="-L$TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/lib -L$TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS"
+SNAPPY_LDFLAGS="$SNAPPY_LIBS $ZLIB_LIBS"
+
+VALGRIND_VER="$TOOLCHAIN_LIB_BASE/valgrind/valgrind-3.8.1/91ddd43/bin/"
+
+export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED SNAPPY_LDFLAGS VALGRIND_VER

build_tools/regression_build_test.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+set -e
+
+NUM=10000000
+
+if [ $# -eq 1 ];then
+  DATA_DIR=$1
+elif [ $# -eq 2 ];then
+  DATA_DIR=$1
+  STAT_FILE=$2
+fi
+
+# On the production build servers, set data and stat
+# files/directories not in /tmp or else the tempdir cleaning
+# scripts will make you very unhappy.
+DATA_DIR=${DATA_DIR:-$(mktemp -t -d rocksdb_XXXX)}
+STAT_FILE=${STAT_FILE:-$(mktemp -t -u rocksdb_test_stats_XXXX)}
+
+function cleanup {
+  rm -rf $DATA_DIR
+  rm -f $STAT_FILE.fillseq
+  rm -f $STAT_FILE.readrandom
+  rm -f $STAT_FILE.overwrite
+}
+
+trap cleanup EXIT
+
+function send_to_ods {
+  key="$1"
+  value="$2"
+
+  if [ -z "$value" ];then
+    echo >&2 "ERROR: Key $key doesn't have a value."
+    return
+  fi
+  curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=$key&value=$value" \
+    --connect-timeout 60
+}
+
+make clean
+make db_bench -j$(nproc)
+
+./db_bench \
+    --benchmarks=fillseq \
+    --db=$DATA_DIR \
+    --use_existing_db=0 \
+    --bloom_bits=10 \
+    --num=$NUM \
+    --writes=$NUM \
+    --cache_size=6442450944 \
+    --cache_numshardbits=6 \
+    --open_files=55000 \
+    --statistics=1 \
+    --histogram=1 \
+    --disable_data_sync=1 \
+    --disable_wal=1 \
+    --sync=0  > ${STAT_FILE}.fillseq
+
+./db_bench \
+    --benchmarks=overwrite \
+    --db=$DATA_DIR \
+    --use_existing_db=1 \
+    --bloom_bits=10 \
+    --num=$NUM \
+    --writes=$((NUM / 2)) \
+    --cache_size=6442450944 \
+    --cache_numshardbits=6  \
+    --open_files=55000 \
+    --statistics=1 \
+    --histogram=1 \
+    --disable_data_sync=1 \
+    --disable_wal=1 \
+    --sync=0 \
+    --threads=8 > ${STAT_FILE}.overwrite
+
+./db_bench \
+    --benchmarks=readrandom \
+    --db=$DATA_DIR \
+    --use_existing_db=1 \
+    --bloom_bits=10 \
+    --num=$NUM \
+    --reads=$((NUM / 100)) \
+    --cache_size=6442450944 \
+    --cache_numshardbits=6 \
+    --open_files=55000 \
+    --statistics=1 \
+    --histogram=1 \
+    --disable_data_sync=1 \
+    --disable_wal=1 \
+    --sync=0 \
+    --threads=128 > ${STAT_FILE}.readrandom
+
+OVERWRITE_OPS=$(awk '/overwrite/ {print $5}' $STAT_FILE.overwrite)
+FILLSEQ_OPS=$(awk '/fillseq/ {print $5}' $STAT_FILE.fillseq)
+READRANDOM_OPS=$(awk '/readrandom/ {print $5}' $STAT_FILE.readrandom)
+
+send_to_ods rocksdb.build.overwrite.qps $OVERWRITE_OPS
+send_to_ods rocksdb.build.fillseq.qps $FILLSEQ_OPS
+send_to_ods rocksdb.build.readrandom.qps $READRANDOM_OPS

build_tools/valgrind_test.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+#A shell script for Jenknis to run valgrind on rocksdb tests
+#Returns 0 on success when there are no failed tests 
+
+VALGRIND_DIR=build_tools/VALGRIND_LOGS
+make clean
+make -j$(nproc) valgrind_check
+NUM_FAILED_TESTS=$((`wc -l $VALGRIND_DIR/valgrind_failed_tests | awk '{print $1}'` - 1))
+if [ $NUM_FAILED_TESTS -lt 1 ]; then
+  echo No tests have valgrind errors
+  exit 0
+else
+  cat $VALGRIND_DIR/valgrind_failed_tests
+  exit 1
+fi

coverage/coverage_test.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Exit on error.
+set -e
+
+if [ -n "$USE_CLANG" ]; then
+  echo "Error: Coverage test is supported only for gcc."
+  exit 1
+fi
+
+ROOT=".."
+# Fetch right version of gcov
+if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
+  source $ROOT/build_tools/fbcode.gcc471.sh
+  GCOV=$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1/cc6c9dc/bin/gcov
+else
+  GCOV=$(which gcov)
+fi
+
+COVERAGE_DIR="$PWD/COVERAGE_REPORT"
+mkdir -p $COVERAGE_DIR
+
+# Find all gcno files to generate the coverage report
+
+GCNO_FILES=`find $ROOT -name "*.gcno"`
+$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
+  # Parse the raw gcov report to more human readable form.
+  python $ROOT/coverage/parse_gcov_output.py |
+  # Write the output to both stdout and report file.
+  tee $COVERAGE_DIR/coverage_report_all.txt &&
+echo -e "Generated coverage report for all files: $COVERAGE_DIR/coverage_report_all.txt\n"
+
+# TODO: we also need to get the files of the latest commits.
+# Get the most recently committed files.
+LATEST_FILES=`
+  git show --pretty="format:" --name-only HEAD |
+  grep -v "^$" |
+  paste -s -d,`
+RECENT_REPORT=$COVERAGE_DIR/coverage_report_recent.txt
+
+echo -e "Recently updated files: $LATEST_FILES\n" > $RECENT_REPORT
+$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
+  python $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES |
+  tee -a $RECENT_REPORT &&
+echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n"
+
+# Generate the html report. If we cannot find lcov in this machine, we'll simply
+# skip this step.
+echo "Generating the html coverage report..."
+
+LCOV=$(which lcov || true 2>/dev/null)
+if [ -z $LCOV ]
+then
+  echo "Skip: Cannot find lcov to generate the html report."
+  exit 0
+fi
+
+(cd $ROOT; lcov --no-external \
+     --capture  \
+     --directory $PWD \
+     --gcov-tool $GCOV \
+     --output-file $COVERAGE_DIR/coverage.info &>/dev/null)
+
+genhtml $COVERAGE_DIR/coverage.info -o $COVERAGE_DIR &>/dev/null
+
+echo "HTML Coverage report is generated in $COVERAGE_DIR"

coverage/parse_gcov_output.py

@@ -0,0 +1,118 @@
+import optparse
+import re
+import sys
+
+from optparse import OptionParser
+
+# the gcov report follows certain pattern. Each file will have two lines
+# of report, from which we can extract the file name, total lines and coverage
+# percentage.
+def parse_gcov_report(gcov_input):
+    per_file_coverage = {}
+    total_coverage = None
+
+    for line in sys.stdin:
+        line = line.strip()
+
+        # --First line of the coverage report (with file name in it)?
+        match_obj = re.match("^File '(.*)'$", line)
+        if match_obj:
+            # fetch the file name from the first line of the report.
+            current_file = match_obj.group(1)
+            continue
+
+        # -- Second line of the file report (with coverage percentage)
+        match_obj = re.match("^Lines executed:(.*)% of (.*)", line)
+
+        if match_obj:
+            coverage = float(match_obj.group(1))
+            lines = int(match_obj.group(2))
+
+            if current_file is not None:
+                per_file_coverage[current_file] = (coverage, lines)
+                current_file = None
+            else:
+                # If current_file is not set, we reach the last line of report,
+                # which contains the summarized coverage percentage.
+                total_coverage = (coverage, lines)
+            continue
+
+        # If the line's pattern doesn't fall into the above categories. We
+        # can simply ignore them since they're either empty line or doesn't
+        # find executable lines of the given file.
+        current_file = None
+
+    return per_file_coverage, total_coverage
+
+def get_option_parser():
+    usage = "Parse the gcov output and generate more human-readable code " +\
+            "coverage report."
+    parser = OptionParser(usage)
+
+    parser.add_option(
+        "--interested-files", "-i",
+        dest="filenames",
+        help="Comma separated files names. if specified, we will display " +
+             "the coverage report only for interested source files. " +
+             "Otherwise we will display the coverage report for all " +
+             "source files."
+    )
+    return parser
+
+def display_file_coverage(per_file_coverage, total_coverage):
+    # To print out auto-adjustable column, we need to know the longest
+    # length of file names.
+    max_file_name_length = max(
+        len(fname) for fname in per_file_coverage.keys()
+    )
+
+    # -- Print header
+    # size of separator is determined by 3 column sizes:
+    # file name, coverage percentage and lines.
+    header_template = \
+        "%" + str(max_file_name_length) + "s\t%s\t%s"
+    separator = "-" * (max_file_name_length + 10 + 20)
+    print header_template % ("Filename", "Coverage", "Lines")
+    print separator
+
+    # -- Print body
+    # template for printing coverage report for each file.
+    record_template = "%" + str(max_file_name_length) + "s\t%5.2f%%\t%10d"
+
+    for fname, coverage_info in per_file_coverage.items():
+        coverage, lines = coverage_info
+        print record_template % (fname, coverage, lines)
+
+    # -- Print footer
+    if total_coverage:
+        print separator
+        print record_template % ("Total", total_coverage[0], total_coverage[1])
+
+def report_coverage():
+    parser = get_option_parser()
+    (options, args) = parser.parse_args()
+
+    interested_files = set()
+    if options.filenames is not None:
+        interested_files = set(f.strip() for f in options.filenames.split(','))
+
+    # To make things simple, right now we only read gcov report from the input
+    per_file_coverage, total_coverage = parse_gcov_report(sys.stdin)
+
+    # Check if we need to display coverage info for interested files.
+    if len(interested_files):
+        per_file_coverage = dict(
+            (fname, per_file_coverage[fname]) for fname in interested_files
+            if fname in per_file_coverage
+        )
+        # If we only interested in several files, it makes no sense to report
+        # the total_coverage
+        total_coverage = None
+
+    if not len(per_file_coverage):
+        print >> sys.stderr, "Cannot find coverage info for the given files."
+        return
+    display_file_coverage(per_file_coverage, total_coverage)
+
+if __name__ == "__main__":
+    report_coverage()

db/builder.cc

@@ -6,38 +6,152 @@
 
 #include "db/filename.h"
 #include "db/dbformat.h"
+#include "db/merge_helper.h"
 #include "db/table_cache.h"
 #include "db/version_edit.h"
 #include "leveldb/db.h"
 #include "leveldb/env.h"
 #include "leveldb/iterator.h"
+#include "util/stop_watch.h"
 
 namespace leveldb {
 
 Status BuildTable(const std::string& dbname,
                   Env* env,
                   const Options& options,
+                  const EnvOptions& soptions,
                   TableCache* table_cache,
                   Iterator* iter,
-                  FileMetaData* meta) {
+                  FileMetaData* meta,
+                  const Comparator* user_comparator,
+                  const SequenceNumber newest_snapshot,
+                  const SequenceNumber earliest_seqno_in_memtable) {
   Status s;
   meta->file_size = 0;
+  meta->smallest_seqno = meta->largest_seqno = 0;
   iter->SeekToFirst();
 
+  // If the sequence number of the smallest entry in the memtable is
+  // smaller than the most recent snapshot, then we do not trigger
+  // removal of duplicate/deleted keys as part of this builder.
+  bool purge = options.purge_redundant_kvs_while_flush;
+  if (earliest_seqno_in_memtable <= newest_snapshot) {
+    purge = false;
+  }
+
   std::string fname = TableFileName(dbname, meta->number);
   if (iter->Valid()) {
-    WritableFile* file;
-    s = env->NewWritableFile(fname, &file);
+    unique_ptr<WritableFile> file;
+    s = env->NewWritableFile(fname, &file, soptions);
     if (!s.ok()) {
       return s;
     }
+    TableBuilder* builder = new TableBuilder(options, file.get(), 0);
 
-    TableBuilder* builder = new TableBuilder(options, file, 0);
-    meta->smallest.DecodeFrom(iter->key());
-    for (; iter->Valid(); iter->Next()) {
-      Slice key = iter->key();
-      meta->largest.DecodeFrom(key);
-      builder->Add(key, iter->value());
+    // the first key is the smallest key
+    Slice key = iter->key();
+    meta->smallest.DecodeFrom(key);
+    meta->smallest_seqno = GetInternalKeySeqno(key);
+    meta->largest_seqno = meta->smallest_seqno;
+
+    MergeHelper merge(user_comparator, options.merge_operator.get(),
+                      options.info_log.get(),
+                      true /* internal key corruption is not ok */);
+
+    if (purge) {
+      // Ugly walkaround to avoid compiler error for release build
+      bool ok __attribute__((unused)) = true;
+
+      // Will write to builder if current key != prev key
+      ParsedInternalKey prev_ikey;
+      std::string prev_key;
+      bool is_first_key = true;    // Also write if this is the very first key
+
+      while (iter->Valid()) {
+        bool iterator_at_next = false;
+
+        // Get current key
+        ParsedInternalKey this_ikey;
+        Slice key = iter->key();
+        Slice value = iter->value();
+
+        // In-memory key corruption is not ok;
+        // TODO: find a clean way to treat in memory key corruption
+        ok = ParseInternalKey(key, &this_ikey);
+        assert(ok);
+        assert(this_ikey.sequence >= earliest_seqno_in_memtable);
+
+        // If the key is the same as the previous key (and it is not the
+        // first key), then we skip it, since it is an older version.
+        // Otherwise we output the key and mark it as the "new" previous key.
+        if (!is_first_key && !user_comparator->Compare(prev_ikey.user_key,
+                                                       this_ikey.user_key)) {
+          // seqno within the same key are in decreasing order
+          assert(this_ikey.sequence < prev_ikey.sequence);
+        } else {
+          is_first_key = false;
+
+          if (this_ikey.type == kTypeMerge) {
+            // Handle merge-type keys using the MergeHelper
+            merge.MergeUntil(iter, 0 /* don't worry about snapshot */);
+            iterator_at_next = true;
+            if (merge.IsSuccess()) {
+              // Merge completed correctly.
+              // Add the resulting merge key/value and continue to next
+              builder->Add(merge.key(), merge.value());
+              prev_key.assign(merge.key().data(), merge.key().size());
+              ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
+              assert(ok);
+            } else {
+              // Merge did not find a Put/Delete.
+              // Can not compact these merges into a kValueType.
+              // Write them out one-by-one. (Proceed back() to front())
+              const std::deque<std::string>& keys = merge.keys();
+              const std::deque<std::string>& values = merge.values();
+              assert(keys.size() == values.size() && keys.size() >= 1);
+              std::deque<std::string>::const_reverse_iterator key_iter;
+              std::deque<std::string>::const_reverse_iterator value_iter;
+              for (key_iter=keys.rbegin(), value_iter = values.rbegin();
+                   key_iter != keys.rend() && value_iter != values.rend();
+                   ++key_iter, ++value_iter) {
+
+                builder->Add(Slice(*key_iter), Slice(*value_iter));
+              }
+
+              // Sanity check. Both iterators should end at the same time
+              assert(key_iter == keys.rend() && value_iter == values.rend());
+
+              prev_key.assign(keys.front());
+              ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
+              assert(ok);
+            }
+          } else {
+            // Handle Put/Delete-type keys by simply writing them
+            builder->Add(key, value);
+            prev_key.assign(key.data(), key.size());
+            ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
+            assert(ok);
+          }
+        }
+
+        if (!iterator_at_next) iter->Next();
+      }
+
+      // The last key is the largest key
+      meta->largest.DecodeFrom(Slice(prev_key));
+      SequenceNumber seqno = GetInternalKeySeqno(Slice(prev_key));
+      meta->smallest_seqno = std::min(meta->smallest_seqno, seqno);
+      meta->largest_seqno = std::max(meta->largest_seqno, seqno);
+
+    } else {
+      for (; iter->Valid(); iter->Next()) {
+        Slice key = iter->key();
+        meta->largest.DecodeFrom(key);
+        builder->Add(key, iter->value());
+        SequenceNumber seqno = GetInternalKeySeqno(key);
+        meta->smallest_seqno = std::min(meta->smallest_seqno, seqno);
+        meta->largest_seqno = std::max(meta->largest_seqno, seqno);
+      }
     }
 
     // Finish and check for builder errors
@@ -55,20 +169,21 @@ Status BuildTable(const std::string& dbname,
     // Finish and check for file errors
     if (s.ok() && !options.disableDataSync) {
       if (options.use_fsync) {
+        StopWatch sw(env, options.statistics, TABLE_SYNC_MICROS);
         s = file->Fsync();
       } else {
+        StopWatch sw(env, options.statistics, TABLE_SYNC_MICROS);
         s = file->Sync();
       }
     }
     if (s.ok()) {
       s = file->Close();
     }
-    delete file;
-    file = NULL;
 
     if (s.ok()) {
       // Verify that the table is usable
       Iterator* it = table_cache->NewIterator(ReadOptions(),
+                                              soptions,
                                               meta->number,
                                               meta->file_size);
       s = it->status();

db/builder.h

@@ -5,7 +5,9 @@
 #ifndef STORAGE_LEVELDB_DB_BUILDER_H_
 #define STORAGE_LEVELDB_DB_BUILDER_H_
 
+#include "leveldb/comparator.h"
 #include "leveldb/status.h"
+#include "leveldb/types.h"
 
 namespace leveldb {
 
@@ -13,6 +15,7 @@ struct Options;
 struct FileMetaData;
 
 class Env;
+class EnvOptions;
 class Iterator;
 class TableCache;
 class VersionEdit;
@@ -25,9 +28,13 @@ class VersionEdit;
 extern Status BuildTable(const std::string& dbname,
                          Env* env,
                          const Options& options,
+                         const EnvOptions& soptions,
                          TableCache* table_cache,
                          Iterator* iter,
-                         FileMetaData* meta);
+                         FileMetaData* meta,
+                         const Comparator* user_comparator,
+                         const SequenceNumber newest_snapshot,
+                         const SequenceNumber earliest_seqno_in_memtable);
 
 }  // namespace leveldb
 

db/c.cc

@@ -39,6 +39,8 @@ using leveldb::WritableFile;
 using leveldb::WriteBatch;
 using leveldb::WriteOptions;
 
+using std::shared_ptr;
+
 extern "C" {
 
 struct leveldb_t              { DB*               rep; };
@@ -48,12 +50,12 @@ struct leveldb_snapshot_t     { const Snapshot*   rep; };
 struct leveldb_readoptions_t  { ReadOptions       rep; };
 struct leveldb_writeoptions_t { WriteOptions      rep; };
 struct leveldb_options_t      { Options           rep; };
-struct leveldb_cache_t        { Cache*            rep; };
 struct leveldb_seqfile_t      { SequentialFile*   rep; };
 struct leveldb_randomfile_t   { RandomAccessFile* rep; };
 struct leveldb_writablefile_t { WritableFile*     rep; };
-struct leveldb_logger_t       { Logger*           rep; };
 struct leveldb_filelock_t     { FileLock*         rep; };
+struct leveldb_logger_t       { shared_ptr<Logger>  rep; };
+struct leveldb_cache_t        { shared_ptr<Cache>   rep; };
 
 struct leveldb_comparator_t : public Comparator {
   void* state_;
@@ -421,7 +423,9 @@ void leveldb_options_set_env(leveldb_options_t* opt, leveldb_env_t* env) {
 }
 
 void leveldb_options_set_info_log(leveldb_options_t* opt, leveldb_logger_t* l) {
-  opt->rep.info_log = (l ? l->rep : NULL);
+  if (l) {
+    opt->rep.info_log = l->rep;
+  }
 }
 
 void leveldb_options_set_write_buffer_size(leveldb_options_t* opt, size_t s) {
@@ -433,7 +437,9 @@ void leveldb_options_set_max_open_files(leveldb_options_t* opt, int n) {
 }
 
 void leveldb_options_set_cache(leveldb_options_t* opt, leveldb_cache_t* c) {
-  opt->rep.block_cache = c->rep;
+  if (c) {
+    opt->rep.block_cache = c->rep;
+  }
 }
 
 void leveldb_options_set_block_size(leveldb_options_t* opt, size_t s) {
@@ -502,6 +508,16 @@ void leveldb_options_set_compression(leveldb_options_t* opt, int t) {
   opt->rep.compression = static_cast<CompressionType>(t);
 }
 
+void leveldb_options_set_compression_per_level(leveldb_options_t* opt,
+                                               int* level_values,
+                                               size_t num_levels) {
+  opt->rep.compression_per_level.resize(num_levels);
+  for (size_t i = 0; i < num_levels; ++i) {
+    opt->rep.compression_per_level[i] =
+      static_cast<CompressionType>(level_values[i]);
+  }
+}
+
 void leveldb_options_set_compression_options(
     leveldb_options_t* opt, int w_bits, int level, int strategy) {
   opt->rep.compression_opts.window_bits = w_bits;
@@ -647,7 +663,6 @@ leveldb_cache_t* leveldb_cache_create_lru(size_t capacity) {
 }
 
 void leveldb_cache_destroy(leveldb_cache_t* cache) {
-  delete cache->rep;
   delete cache;
 }
 

db/c_test.c

@@ -188,6 +188,9 @@ int main(int argc, char** argv) {
   leveldb_options_set_block_restart_interval(options, 8);
   leveldb_options_set_compression(options, leveldb_no_compression);
   leveldb_options_set_compression_options(options, -14, -1, 0);
+  int compression_levels[] = {leveldb_no_compression, leveldb_no_compression,
+                              leveldb_no_compression, leveldb_no_compression};
+  leveldb_options_set_compression_per_level(options, compression_levels, 4);
 
   roptions = leveldb_readoptions_create();
   leveldb_readoptions_set_verify_checksums(roptions, 1);

db/corruption_test.cc

@@ -10,12 +10,12 @@
 #include <sys/types.h>
 #include "leveldb/cache.h"
 #include "leveldb/env.h"
-#include "leveldb/table.h"
 #include "leveldb/write_batch.h"
 #include "db/db_impl.h"
 #include "db/filename.h"
 #include "db/log_format.h"
 #include "db/version_set.h"
+#include "table/table.h"
 #include "util/logging.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
@@ -28,7 +28,7 @@ class CorruptionTest {
  public:
   test::ErrorEnv env_;
   std::string dbname_;
-  Cache* tiny_cache_;
+  shared_ptr<Cache> tiny_cache_;
   Options options_;
   DB* db_;
 
@@ -38,8 +38,9 @@ class CorruptionTest {
     dbname_ = test::TmpDir() + "/db_test";
     DestroyDB(dbname_, options_);
 
-    db_ = NULL;
+    db_ = nullptr;
     options_.create_if_missing = true;
+    options_.block_size_deviation = 0; // make unit test pass for now
     Reopen();
     options_.create_if_missing = false;
   }
@@ -47,25 +48,26 @@ class CorruptionTest {
   ~CorruptionTest() {
      delete db_;
      DestroyDB(dbname_, Options());
-     delete tiny_cache_;
   }
 
-  Status TryReopen(Options* options = NULL) {
+  Status TryReopen(Options* options = nullptr) {
     delete db_;
-    db_ = NULL;
+    db_ = nullptr;
     Options opt = (options ? *options : options_);
     opt.env = &env_;
     opt.block_cache = tiny_cache_;
+    opt.block_size_deviation = 0;
+    opt.arena_block_size = 4096;
     return DB::Open(opt, dbname_, &db_);
   }
 
-  void Reopen(Options* options = NULL) {
+  void Reopen(Options* options = nullptr) {
     ASSERT_OK(TryReopen(options));
   }
 
   void RepairDB() {
     delete db_;
-    db_ = NULL;
+    db_ = nullptr;
     ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
   }
 
@@ -229,8 +231,8 @@ TEST(CorruptionTest, TableFile) {
   Build(100);
   DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
   dbi->TEST_CompactMemTable();
-  dbi->TEST_CompactRange(0, NULL, NULL);
-  dbi->TEST_CompactRange(1, NULL, NULL);
+  dbi->TEST_CompactRange(0, nullptr, nullptr);
+  dbi->TEST_CompactRange(1, nullptr, nullptr);
 
   Corrupt(kTableFile, 100, 1);
   Check(99, 99);
@@ -278,7 +280,7 @@ TEST(CorruptionTest, CorruptedDescriptor) {
   ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
   DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
   dbi->TEST_CompactMemTable();
-  dbi->TEST_CompactRange(0, NULL, NULL);
+  dbi->TEST_CompactRange(0, nullptr, nullptr);
 
   Corrupt(kDescriptorFile, 0, 1000);
   Status s = TryReopen();

db/db_filesnapshot.cc

@@ -2,10 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#include "db/db_impl.h"
-#include "db/filename.h"
+#include <algorithm>
 #include <string>
 #include <stdint.h>
+#include "db/db_impl.h"
+#include "db/filename.h"
 #include "db/version_set.h"
 #include "leveldb/db.h"
 #include "leveldb/env.h"
@@ -17,12 +18,14 @@ namespace leveldb {
 Status DBImpl::DisableFileDeletions() {
   MutexLock l(&mutex_);
   disable_delete_obsolete_files_ = true;
+  Log(options_.info_log, "File Deletions Disabled");
   return Status::OK();
 }
 
 Status DBImpl::EnableFileDeletions() {
   MutexLock l(&mutex_);
   disable_delete_obsolete_files_ = false;
+  Log(options_.info_log, "File Deletions Enabled");
   return Status::OK();
 }
 
@@ -64,4 +67,47 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
   return Status::OK();
 }
 
+Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
+  // First get sorted files in archive dir, then append sorted files from main
+  // dir to maintain sorted order
+
+  //  list wal files in archive dir.
+  Status s;
+  std::string archivedir = ArchivalDirectory(dbname_);
+  if (env_->FileExists(archivedir)) {
+    s = AppendSortedWalsOfType(archivedir, files, kArchivedLogFile);
+    if (!s.ok()) {
+      return s;
+    }
+  }
+  // list wal files in main db dir.
+  s = AppendSortedWalsOfType(dbname_, files, kAliveLogFile);
+  if (!s.ok()) {
+    return s;
+  }
+  return s;
+}
+
+Status DBImpl::DeleteWalFiles(const VectorLogPtr& files) {
+  Status s;
+  std::string archivedir = ArchivalDirectory(dbname_);
+  std::string files_not_deleted;
+  for (const auto& wal : files) {
+    /* Try deleting in archive dir. If fails, try deleting in main db dir.
+     * This is efficient because all except for very few wal files will be in
+     * archive. Checking for WalType is not much helpful because alive wal could
+       be archived now.
+     */
+    if (!env_->DeleteFile(archivedir + "/" + wal->Filename()).ok() &&
+        !env_->DeleteFile(dbname_ + "/" + wal->Filename()).ok()) {
+      files_not_deleted.append(wal->Filename());
+    }
+  }
+  if (!files_not_deleted.empty()) {
+    return Status::IOError("Deleted all requested files except: " +
+                           files_not_deleted);
+  }
+  return Status::OK();
+}
+
 }

db/db_impl.h

@@ -5,14 +5,17 @@
 #ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_
 #define STORAGE_LEVELDB_DB_DB_IMPL_H_
 
+#include <atomic>
 #include <deque>
 #include <set>
+#include <vector>
 #include "db/dbformat.h"
-#include "db/log_file.h"
 #include "db/log_writer.h"
 #include "db/snapshot.h"
 #include "leveldb/db.h"
 #include "leveldb/env.h"
+#include "leveldb/memtablerep.h"
+#include "leveldb/transaction_log.h"
 #include "port/port.h"
 #include "util/stats_logger.h"
 #include "memtablelist.h"
@@ -36,17 +39,32 @@ class DBImpl : public DB {
 
   // Implementations of the DB interface
   virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value);
+  virtual Status Merge(const WriteOptions&, const Slice& key,
+                       const Slice& value);
   virtual Status Delete(const WriteOptions&, const Slice& key);
   virtual Status Write(const WriteOptions& options, WriteBatch* updates);
   virtual Status Get(const ReadOptions& options,
                      const Slice& key,
                      std::string* value);
+  virtual std::vector<Status> MultiGet(const ReadOptions& options,
+                                       const std::vector<Slice>& keys,
+                                       std::vector<std::string>* values);
+
+  // Returns false if key doesn't exist in the database and true if it may.
+  // If value_found is not passed in as null, then return the value if found in
+  // memory. On return, if value was found, then value_found will be set to true
+  // , otherwise false.
+  virtual bool KeyMayExist(const ReadOptions& options,
+                           const Slice& key,
+                           std::string* value,
+                           bool* value_found = nullptr);
   virtual Iterator* NewIterator(const ReadOptions&);
   virtual const Snapshot* GetSnapshot();
   virtual void ReleaseSnapshot(const Snapshot* snapshot);
   virtual bool GetProperty(const Slice& property, std::string* value);
   virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
-  virtual void CompactRange(const Slice* begin, const Slice* end);
+  virtual void CompactRange(const Slice* begin, const Slice* end,
+                            bool reduce_level = false);
   virtual int NumberLevels();
   virtual int MaxMemCompactionLevel();
   virtual int Level0StopWriteTrigger();
@@ -55,13 +73,15 @@ class DBImpl : public DB {
   virtual Status EnableFileDeletions();
   virtual Status GetLiveFiles(std::vector<std::string>&,
                               uint64_t* manifest_file_size);
+  virtual Status GetSortedWalFiles(VectorLogPtr& files);
+  virtual Status DeleteWalFiles(const VectorLogPtr& files);
   virtual SequenceNumber GetLatestSequenceNumber();
   virtual Status GetUpdatesSince(SequenceNumber seq_number,
-                                 TransactionLogIterator ** iter);
+                                 unique_ptr<TransactionLogIterator>* iter);
 
   // Extra methods (for testing) that are not in the public DB interface
 
-  // Compact any files in the named level that overlap [*begin,*end]
+  // Compact any files in the named level that overlap [*begin, *end]
   void TEST_CompactRange(int level, const Slice* begin, const Slice* end);
 
   // Force current memtable contents to be compacted.
@@ -85,19 +105,29 @@ class DBImpl : public DB {
   // Simulate a db crash, no elegant closing of database.
   void TEST_Destroy_DBImpl();
 
+  // Return the current manifest file no.
+  uint64_t TEST_Current_Manifest_FileNo();
+
+  // Trigger's a background call for testing.
+  void TEST_PurgeObsoleteteWAL();
+
  protected:
   Env* const env_;
   const std::string dbname_;
-  VersionSet* versions_;
+  unique_ptr<VersionSet> versions_;
   const InternalKeyComparator internal_comparator_;
   const Options options_;  // options_.comparator == &internal_comparator_
 
   const Comparator* user_comparator() const {
     return internal_comparator_.user_comparator();
   }
+
   MemTable* GetMemTable() {
     return mem_;
-  } 
+  }
+
+  Iterator* NewInternalIterator(const ReadOptions&,
+                                SequenceNumber* latest_snapshot);
 
  private:
   friend class DB;
@@ -105,15 +135,12 @@ class DBImpl : public DB {
   struct Writer;
   struct DeletionState;
 
-  Iterator* NewInternalIterator(const ReadOptions&,
-                                SequenceNumber* latest_snapshot);
-
   Status NewDB();
 
   // Recover the descriptor from persistent storage.  May do a significant
   // amount of work to recover recently logged updates.  Any changes to
   // be made to the descriptor are added to *edit.
-  Status Recover(VersionEdit* edit, MemTable* external_table = NULL,
+  Status Recover(VersionEdit* edit, MemTable* external_table = nullptr,
       bool error_if_log_file_exist = false);
 
   void MaybeIgnoreError(Status* s) const;
@@ -125,7 +152,7 @@ class DBImpl : public DB {
 
   // Compact the in-memory write buffer to disk.  Switches to a new
   // log-file/memtable and writes a new descriptor iff successful.
-  Status CompactMemTable(bool* madeProgress = NULL);
+  Status CompactMemTable(bool* madeProgress = nullptr);
 
   Status RecoverLogFile(uint64_t log_number,
                         VersionEdit* edit,
@@ -138,9 +165,10 @@ class DBImpl : public DB {
   // for the entire period. The second method WriteLevel0Table supports
   // concurrent flush memtables to storage.
   Status WriteLevel0TableForRecovery(MemTable* mem, VersionEdit* edit);
-  Status WriteLevel0Table(MemTable* mem, VersionEdit* edit,
+  Status WriteLevel0Table(std::vector<MemTable*> &mems, VersionEdit* edit,
                                 uint64_t* filenumber);
 
+  uint64_t SlowdownAmount(int n, int top, int bottom);
   Status MakeRoomForWrite(bool force /* compact even if there is room? */);
   WriteBatch* BuildBatchGroup(Writer** last_writer);
 
@@ -157,7 +185,7 @@ class DBImpl : public DB {
   void MaybeScheduleCompaction();
   static void BGWork(void* db);
   void BackgroundCall();
-  Status BackgroundCompaction(bool* madeProgress, DeletionState& deletion_state);
+  Status BackgroundCompaction(bool* madeProgress,DeletionState& deletion_state);
   void CleanupCompaction(CompactionState* compact);
   Status DoCompactionWork(CompactionState* compact);
 
@@ -182,47 +210,62 @@ class DBImpl : public DB {
 
   void PurgeObsoleteWALFiles();
 
-  Status ListAllWALFiles(const std::string& path,
-                         std::vector<LogFile>* logFiles,
-                         WalFileType type);
+  Status AppendSortedWalsOfType(const std::string& path,
+                                VectorLogPtr& log_files,
+                                WalFileType type);
 
-  //  Find's all the log files which contain updates with seq no.
-  //  Greater Than or Equal to the requested SequenceNumber
-  Status FindProbableWALFiles(std::vector<LogFile>* const allLogs,
-                              std::vector<LogFile>* const result,
-                              const SequenceNumber target);
-
-
-  Status ReadFirstRecord(const LogFile& file, WriteBatch* const result);
+  // Requires: all_logs should be sorted with earliest log file first
+  // Retains all log files in all_logs which contain updates with seq no.
+  // Greater Than or Equal to the requested SequenceNumber.
+  Status RetainProbableWalFiles(VectorLogPtr& all_logs,
+                                const SequenceNumber target);
+  //  return true if
+  bool CheckWalFileExistsAndEmpty(const WalFileType type,
+                                  const uint64_t number);
 
+  Status ReadFirstRecord(const WalFileType type, const uint64_t number,
+                         WriteBatch* const result);
 
   Status ReadFirstLine(const std::string& fname, WriteBatch* const batch);
+
+  void PrintStatistics();
+
+  // dump leveldb.stats to LOG
+  void MaybeDumpStats();
+
+  // Return the minimum empty level that could hold the total data in the
+  // input level. Return the input level, if such level could not be found.
+  int FindMinimumEmptyLevelFitting(int level);
+
+  // Move the files in the input level to the minimum level that could hold
+  // the data set.
+  void ReFitLevel(int level);
+
   // Constant after construction
   const InternalFilterPolicy internal_filter_policy_;
   bool owns_info_log_;
-  bool owns_cache_;
 
   // table_cache_ provides its own synchronization
-  TableCache* table_cache_;
+  unique_ptr<TableCache> table_cache_;
 
-  // Lock over the persistent DB state.  Non-NULL iff successfully acquired.
+  // Lock over the persistent DB state.  Non-nullptr iff successfully acquired.
   FileLock* db_lock_;
 
   // State below is protected by mutex_
   port::Mutex mutex_;
   port::AtomicPointer shutting_down_;
   port::CondVar bg_cv_;          // Signalled when background work finishes
+  std::shared_ptr<MemTableRepFactory> mem_rep_factory_;
   MemTable* mem_;
   MemTableList imm_;             // Memtable that are not changing
-  WritableFile* logfile_;
   uint64_t logfile_number_;
-  log::Writer* log_;
+  unique_ptr<log::Writer> log_;
 
   std::string host_name_;
 
   // Queue of writers.
   std::deque<Writer*> writers_;
-  WriteBatch* tmp_batch_;
+  WriteBatch tmp_batch_;
 
   SnapshotList snapshots_;
 
@@ -241,8 +284,8 @@ class DBImpl : public DB {
     int level;
     bool done;
     bool in_progress;           // compaction request being processed?
-    const InternalKey* begin;   // NULL means beginning of key range
-    const InternalKey* end;     // NULL means end of key range
+    const InternalKey* begin;   // nullptr means beginning of key range
+    const InternalKey* end;     // nullptr means end of key range
     InternalKey tmp_storage;    // Used to keep track of compaction progress
   };
   ManualCompaction* manual_compaction_;
@@ -250,7 +293,7 @@ class DBImpl : public DB {
   // Have we encountered a background error in paranoid mode?
   Status bg_error_;
 
-  StatsLogger* logger_;
+  std::unique_ptr<StatsLogger> logger_;
 
   int64_t volatile last_log_ts;
 
@@ -260,11 +303,21 @@ class DBImpl : public DB {
   // last time when DeleteObsoleteFiles was invoked
   uint64_t delete_obsolete_files_last_run_;
 
+  // last time when PurgeObsoleteWALFiles ran.
+  uint64_t purge_wal_files_last_run_;
+
+  // last time stats were dumped to LOG
+  std::atomic<uint64_t> last_stats_dump_time_microsec_;
+
   // These count the number of microseconds for which MakeRoomForWrite stalls.
   uint64_t stall_level0_slowdown_;
   uint64_t stall_memtable_compaction_;
   uint64_t stall_level0_num_files_;
-  uint64_t stall_leveln_slowdown_;
+  std::vector<uint64_t> stall_leveln_slowdown_;
+  uint64_t stall_level0_slowdown_count_;
+  uint64_t stall_memtable_compaction_count_;
+  uint64_t stall_level0_num_files_count_;
+  std::vector<uint64_t> stall_leveln_slowdown_count_;
 
   // Time at which this instance was started.
   const uint64_t started_at_;
@@ -274,7 +327,7 @@ class DBImpl : public DB {
   // Per level compaction stats.  stats_[level] stores the stats for
   // compactions that produced data for the specified "level".
   struct CompactionStats {
-    int64_t micros;
+    uint64_t micros;
 
     // Bytes read from level N during compaction between levels N and N+1
     int64_t bytes_readn;
@@ -314,7 +367,20 @@ class DBImpl : public DB {
     }
   };
 
-  CompactionStats* stats_;
+  std::vector<CompactionStats> stats_;
+
+  // Used to compute per-interval statistics
+  struct StatsSnapshot {
+    uint64_t bytes_read_;
+    uint64_t bytes_written_;
+    uint64_t bytes_new_;
+    double   seconds_up_;
+
+    StatsSnapshot() : bytes_read_(0), bytes_written_(0),
+                      bytes_new_(0), seconds_up_(0) {}
+  };
+
+  StatsSnapshot last_stats_;
 
   static const int KEEP_LOG_FILE_NUM = 1000;
   std::string db_absolute_path_;
@@ -322,6 +388,19 @@ class DBImpl : public DB {
   // count of the number of contiguous delaying writes
   int delayed_writes_;
 
+  // store the last flushed sequence.
+  // Used by transaction log iterator.
+  SequenceNumber last_flushed_sequence_;
+
+  // The options to access storage files
+  const EnvOptions storage_options_;
+
+  // A value of true temporarily disables scheduling of background work
+  bool bg_work_gate_closed_;
+
+  // Guard against multiple concurrent refitting
+  bool refitting_level_;
+
   // No copying allowed
   DBImpl(const DBImpl&);
   void operator=(const DBImpl&);
@@ -329,9 +408,20 @@ class DBImpl : public DB {
   // dump the delayed_writes_ to the log file and reset counter.
   void DelayLoggingAndReset();
 
-  // find the earliest snapshot where seqno is visible
-  inline SequenceNumber findEarliestVisibleSnapshot(SequenceNumber in,
-    std::vector<SequenceNumber>& snapshots);
+  // Return the earliest snapshot where seqno is visible.
+  // Store the snapshot right before that, if any, in prev_snapshot
+  inline SequenceNumber findEarliestVisibleSnapshot(
+    SequenceNumber in,
+    std::vector<SequenceNumber>& snapshots,
+    SequenceNumber* prev_snapshot);
+
+  // Function that Get and KeyMayExist call with no_io true or false
+  // Note: 'value_found' from KeyMayExist propagates here
+  Status GetImpl(const ReadOptions& options,
+                 const Slice& key,
+                 std::string* value,
+                 const bool no_io = false,
+                 bool* value_found = nullptr);
 };
 
 // Sanitize db options.  The caller should delete result.info_log if

db/db_impl_readonly.cc

@@ -24,11 +24,11 @@
 #include "leveldb/db.h"
 #include "leveldb/env.h"
 #include "leveldb/status.h"
-#include "leveldb/table.h"
 #include "leveldb/table_builder.h"
 #include "port/port.h"
 #include "table/block.h"
 #include "table/merger.h"
+#include "table/table.h"
 #include "table/two_level_iterator.h"
 #include "util/coding.h"
 #include "util/logging.h"
@@ -50,28 +50,33 @@ Status DBImplReadOnly::Get(const ReadOptions& options,
                    const Slice& key,
                    std::string* value) {
   Status s;
+  MemTable* mem = GetMemTable();
   Version* current = versions_->current();
   SequenceNumber snapshot = versions_->LastSequence();
+  std::deque<std::string> merge_operands;
   LookupKey lkey(key, snapshot);
-  Version::GetStats stats;
-  s = current->Get(options, lkey, value, &stats);
+  if (mem->Get(lkey, value, &s, &merge_operands, options_)) {
+  } else {
+    Version::GetStats stats;
+    current->Get(options, lkey, value, &s, &merge_operands, &stats, options_);
+  }
   return s;
 }
 
 Iterator* DBImplReadOnly::NewIterator(const ReadOptions& options) {
-  std::vector<Iterator*> list;
-  versions_->current()->AddIterators(options, &list);
-  Iterator* internal_iter =
-      NewMergingIterator(&internal_comparator_, &list[0], list.size());
+  SequenceNumber latest_snapshot;
+  Iterator* internal_iter = NewInternalIterator(options, &latest_snapshot);
   return NewDBIterator(
-      &dbname_, env_, user_comparator(), internal_iter,
-      reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_);
+    &dbname_, env_, options_,  user_comparator(),internal_iter,
+      (options.snapshot != nullptr
+      ? reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_
+      : latest_snapshot));
 }
 
 
 Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
                 DB** dbptr, bool error_if_log_file_exist) {
-  *dbptr = NULL;
+  *dbptr = nullptr;
 
   DBImplReadOnly* impl = new DBImplReadOnly(options, dbname);
   impl->mutex_.Lock();

db/db_impl_readonly.h

@@ -32,18 +32,26 @@ public:
  virtual Status Get(const ReadOptions& options,
                     const Slice& key,
                     std::string* value);
+
+ // TODO: Implement ReadOnly MultiGet?
+
  virtual Iterator* NewIterator(const ReadOptions&);
 
  virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value) {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
+ virtual Status Merge(const WriteOptions&, const Slice& key,
+                      const Slice& value) {
+   return Status::NotSupported("Not supported operation in read only mode.");
+ }
  virtual Status Delete(const WriteOptions&, const Slice& key) {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
  virtual Status Write(const WriteOptions& options, WriteBatch* updates) {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
- virtual void CompactRange(const Slice* begin, const Slice* end) {
+ virtual void CompactRange(const Slice* begin, const Slice* end,
+                           bool reduce_level = false) {
  }
  virtual Status DisableFileDeletions() {
    return Status::NotSupported("Not supported operation in read only mode.");

db/db_iter.cc

@@ -3,11 +3,15 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include "db/db_iter.h"
+#include <stdexcept>
+#include <deque>
 
 #include "db/filename.h"
 #include "db/dbformat.h"
 #include "leveldb/env.h"
+#include "leveldb/options.h"
 #include "leveldb/iterator.h"
+#include "leveldb/merge_operator.h"
 #include "port/port.h"
 #include "util/logging.h"
 #include "util/mutexlock.h"
@@ -36,6 +40,7 @@ namespace {
 // numbers, deletion markers, overwrites, etc.
 class DBIter: public Iterator {
  public:
+  // The following is grossly complicated. TODO: clean it up
   // Which direction is the iterator currently moving?
   // (1) When moving forward, the internal iterator is positioned at
   //     the exact entry that yields this->key(), this->value()
@@ -46,27 +51,34 @@ class DBIter: public Iterator {
     kReverse
   };
 
-  DBIter(const std::string* dbname, Env* env,
+  DBIter(const std::string* dbname, Env* env, const Options& options,
          const Comparator* cmp, Iterator* iter, SequenceNumber s)
       : dbname_(dbname),
         env_(env),
+        logger_(options.info_log),
         user_comparator_(cmp),
+        user_merge_operator_(options.merge_operator.get()),
         iter_(iter),
         sequence_(s),
         direction_(kForward),
-        valid_(false) {
+        valid_(false),
+        current_entry_is_merged_(false),
+        statistics_(options.statistics) {
+    RecordTick(statistics_, NO_ITERATORS, 1);
   }
   virtual ~DBIter() {
+    RecordTick(statistics_, NO_ITERATORS, -1);
     delete iter_;
   }
   virtual bool Valid() const { return valid_; }
   virtual Slice key() const {
     assert(valid_);
-    return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_;
+    return saved_key_;
   }
   virtual Slice value() const {
     assert(valid_);
-    return (direction_ == kForward) ? iter_->value() : saved_value_;
+    return (direction_ == kForward && !current_entry_is_merged_) ?
+      iter_->value() : saved_value_;
   }
   virtual Status status() const {
     if (status_.ok()) {
@@ -83,9 +95,10 @@ class DBIter: public Iterator {
   virtual void SeekToLast();
 
  private:
-  void FindNextUserEntry(bool skipping, std::string* skip);
+  void FindNextUserEntry(bool skipping);
   void FindPrevUserEntry();
   bool ParseKey(ParsedInternalKey* key);
+  void MergeValuesNewToOld();
 
   inline void SaveKey(const Slice& k, std::string* dst) {
     dst->assign(k.data(), k.size());
@@ -102,15 +115,20 @@ class DBIter: public Iterator {
 
   const std::string* const dbname_;
   Env* const env_;
+  shared_ptr<Logger> logger_;
   const Comparator* const user_comparator_;
+  const MergeOperator* const user_merge_operator_;
   Iterator* const iter_;
   SequenceNumber const sequence_;
 
   Status status_;
   std::string saved_key_;     // == current key when direction_==kReverse
   std::string saved_value_;   // == current raw value when direction_==kReverse
+  std::string skip_key_;
   Direction direction_;
   bool valid_;
+  bool current_entry_is_merged_;
+  std::shared_ptr<Statistics> statistics_;
 
   // No copying allowed
   DBIter(const DBIter&);
@@ -120,6 +138,8 @@ class DBIter: public Iterator {
 inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
   if (!ParseInternalKey(iter_->key(), ikey)) {
     status_ = Status::Corruption("corrupted internal key in DBIter");
+    Log(logger_, "corrupted internal key in DBIter: %s",
+        iter_->key().ToString(true).c_str());
     return false;
   } else {
     return true;
@@ -146,47 +166,156 @@ void DBIter::Next() {
     }
   }
 
-  // Temporarily use saved_key_ as storage for key to skip.
-  std::string* skip = &saved_key_;
-  SaveKey(ExtractUserKey(iter_->key()), skip);
-  FindNextUserEntry(true, skip);
+  // If the current value is merged, we might already hit end of iter_
+  if (!iter_->Valid()) {
+    valid_ = false;
+    return;
+  }
+  FindNextUserEntry(true /* skipping the current user key */);
 }
 
-void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
+
+// PRE: saved_key_ has the current user key if skipping
+// POST: saved_key_ should have the next user key if valid_,
+//       if the current entry is a result of merge
+//           current_entry_is_merged_ => true
+//           saved_value_             => the merged value
+//
+// NOTE: In between, saved_key_ can point to a user key that has
+//       a delete marker
+void DBIter::FindNextUserEntry(bool skipping) {
   // Loop until we hit an acceptable entry to yield
   assert(iter_->Valid());
   assert(direction_ == kForward);
+  current_entry_is_merged_ = false;
   do {
     ParsedInternalKey ikey;
     if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
-      switch (ikey.type) {
-        case kTypeDeletion:
-          // Arrange to skip all upcoming entries for this key since
-          // they are hidden by this deletion.
-          SaveKey(ikey.user_key, skip);
-          skipping = true;
-          break;
-        case kTypeValue:
-          if (skipping &&
-              user_comparator_->Compare(ikey.user_key, *skip) <= 0) {
-            // Entry hidden
-          } else {
+      if (skipping &&
+          user_comparator_->Compare(ikey.user_key, saved_key_) <= 0) {
+        // skip this entry
+      } else {
+        skipping = false;
+        switch (ikey.type) {
+          case kTypeDeletion:
+            // Arrange to skip all upcoming entries for this key since
+            // they are hidden by this deletion.
+            SaveKey(ikey.user_key, &saved_key_);
+            skipping = true;
+            break;
+          case kTypeValue:
             valid_ = true;
-            saved_key_.clear();
+            SaveKey(ikey.user_key, &saved_key_);
             return;
-          }
-          break;
+          case kTypeMerge:
+            // By now, we are sure the current ikey is going to yield a value
+            SaveKey(ikey.user_key, &saved_key_);
+            current_entry_is_merged_ = true;
+            valid_ = true;
+            MergeValuesNewToOld();  // Go to a different state machine
+            return;
+          case kTypeLogData:
+            assert(false);
+            break;
+        }
       }
     }
     iter_->Next();
   } while (iter_->Valid());
-  saved_key_.clear();
   valid_ = false;
 }
 
+// Merge values of the same user key starting from the current iter_ position
+// Scan from the newer entries to older entries.
+// PRE: iter_->key() points to the first merge type entry
+//      saved_key_ stores the user key
+// POST: saved_value_ has the merged value for the user key
+//       iter_ points to the next entry (or invalid)
+void DBIter::MergeValuesNewToOld() {
+  if (!user_merge_operator_) {
+    Log(logger_, "Options::merge_operator is null.");
+    throw std::logic_error("DBIter::MergeValuesNewToOld() with"
+                           " Options::merge_operator null");
+  }
+
+  // Start the merge process by pushing the first operand
+  std::deque<std::string> operands;
+  operands.push_front(iter_->value().ToString());
+
+  std::string merge_result;   // Temporary string to hold merge result later
+  ParsedInternalKey ikey;
+  for (iter_->Next(); iter_->Valid(); iter_->Next()) {
+    if (!ParseKey(&ikey)) {
+      // skip corrupted key
+      continue;
+    }
+
+    if (user_comparator_->Compare(ikey.user_key, saved_key_) != 0) {
+      // hit the next user key, stop right here
+      break;
+    }
+
+    if (kTypeDeletion == ikey.type) {
+      // hit a delete with the same user key, stop right here
+      // iter_ is positioned after delete
+      iter_->Next();
+      break;
+    }
+
+    if (kTypeValue == ikey.type) {
+      // hit a put, merge the put value with operands and store the
+      // final result in saved_value_. We are done!
+      // ignore corruption if there is any.
+      const Slice value = iter_->value();
+      user_merge_operator_->FullMerge(ikey.user_key, &value, operands,
+                                      &saved_value_, logger_.get());
+      // iter_ is positioned after put
+      iter_->Next();
+      return;
+    }
+
+    if (kTypeMerge == ikey.type) {
+      // hit a merge, add the value as an operand and run associative merge.
+      // when complete, add result to operands and continue.
+      const Slice& value = iter_->value();
+      operands.push_front(value.ToString());
+      while(operands.size() >= 2) {
+        // Call user associative-merge until it returns false
+        if (user_merge_operator_->PartialMerge(ikey.user_key,
+                                               Slice(operands[0]),
+                                               Slice(operands[1]),
+                                               &merge_result,
+                                               logger_.get())) {
+          operands.pop_front();
+          swap(operands.front(), merge_result);
+        } else {
+          // Associative merge returns false ==> stack the operands
+          break;
+        }
+      }
+
+    }
+  }
+
+  // we either exhausted all internal keys under this user key, or hit
+  // a deletion marker.
+  // feed null as the existing value to the merge operator, such that
+  // client can differentiate this scenario and do things accordingly.
+  user_merge_operator_->FullMerge(saved_key_, nullptr, operands,
+                                  &saved_value_, logger_.get());
+}
+
 void DBIter::Prev() {
   assert(valid_);
 
+  // Throw an exception now if merge_operator is provided
+  // TODO: support backward iteration
+  if (user_merge_operator_) {
+    Log(logger_, "Prev not supported yet if merge_operator is provided");
+    throw std::logic_error("DBIter::Prev backward iteration not supported"
+                           " if merge_operator is provided");
+  }
+
   if (direction_ == kForward) {  // Switch directions?
     // iter_ is pointing at the current entry.  Scan backwards until
     // the key changes so we can use the normal reverse scanning code.
@@ -261,7 +390,7 @@ void DBIter::Seek(const Slice& target) {
       &saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek));
   iter_->Seek(saved_key_);
   if (iter_->Valid()) {
-    FindNextUserEntry(false, &saved_key_ /* temporary storage */);
+    FindNextUserEntry(false /*not skipping */);
   } else {
     valid_ = false;
   }
@@ -272,13 +401,21 @@ void DBIter::SeekToFirst() {
   ClearSavedValue();
   iter_->SeekToFirst();
   if (iter_->Valid()) {
-    FindNextUserEntry(false, &saved_key_ /* temporary storage */);
+    FindNextUserEntry(false /* not skipping */);
   } else {
     valid_ = false;
   }
 }
 
 void DBIter::SeekToLast() {
+  // Throw an exception for now if merge_operator is provided
+  // TODO: support backward iteration
+  if (user_merge_operator_) {
+    Log(logger_, "SeekToLast not supported yet if merge_operator is provided");
+    throw std::logic_error("DBIter::SeekToLast: backward iteration not"
+                           " supported if merge_operator is provided");
+  }
+
   direction_ = kReverse;
   ClearSavedValue();
   iter_->SeekToLast();
@@ -290,10 +427,12 @@ void DBIter::SeekToLast() {
 Iterator* NewDBIterator(
     const std::string* dbname,
     Env* env,
-    const Comparator* user_key_comparator,
+    const Options& options,
+    const Comparator *user_key_comparator,
     Iterator* internal_iter,
     const SequenceNumber& sequence) {
-  return new DBIter(dbname, env, user_key_comparator, internal_iter, sequence);
+  return new DBIter(dbname, env, options, user_key_comparator,
+                    internal_iter, sequence);
 }
 
 }  // namespace leveldb

db/db_iter.h

@@ -17,7 +17,8 @@ namespace leveldb {
 extern Iterator* NewDBIterator(
     const std::string* dbname,
     Env* env,
-    const Comparator* user_key_comparator,
+    const Options& options,
+    const Comparator *user_key_comparator,
     Iterator* internal_iter,
     const SequenceNumber& sequence);
 

db/db_statistics.h

@@ -2,48 +2,65 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#ifndef LEVELDB_STORAGE_DB_DB_STATISTICS_H_
+#define LEVELDB_STORAGE_DB_DB_STATISTICS_H_
+
 #include <cassert>
 #include <stdlib.h>
 #include <vector>
+#include <memory>
 
 #include "leveldb/statistics.h"
+#include "util/histogram.h"
 #include "port/port.h"
 #include "util/mutexlock.h"
+
+
 namespace leveldb {
 
 class DBStatistics: public Statistics {
  public:
-  DBStatistics() : allTickers_(TICKER_ENUM_MAX) { }
+  DBStatistics() : allTickers_(TICKER_ENUM_MAX),
+                   allHistograms_(HISTOGRAM_ENUM_MAX) { }
 
-  void incNumFileOpens() {
-    MutexLock l(&mu_);
-    numFileOpens_++;
-  }
+  virtual ~DBStatistics() {}
 
-  void incNumFileCloses() {
-    MutexLock l(&mu_);
-    numFileCloses_++;
-  }
-
-  void incNumFileErrors() {
-    MutexLock l(&mu_);
-    numFileErrors_++;
-  }
-
-  long getTickerCount(Tickers tickerType) {
+  virtual long getTickerCount(Tickers tickerType) {
     assert(tickerType < TICKER_ENUM_MAX);
     return allTickers_[tickerType].getCount();
   }
 
-  void recordTick(Tickers tickerType) {
+  virtual void setTickerCount(Tickers tickerType, uint64_t count) {
     assert(tickerType < TICKER_ENUM_MAX);
-    allTickers_[tickerType].recordTick();
+    allTickers_[tickerType].setTickerCount(count);
+  }
+
+  virtual void recordTick(Tickers tickerType, uint64_t count) {
+    assert(tickerType < TICKER_ENUM_MAX);
+    allTickers_[tickerType].recordTick(count);
+  }
+
+  virtual void measureTime(Histograms histogramType, uint64_t value) {
+    assert(histogramType < HISTOGRAM_ENUM_MAX);
+    allHistograms_[histogramType].Add(value);
+  }
+
+  virtual void histogramData(Histograms histogramType,
+                             HistogramData * const data) {
+    assert(histogramType < HISTOGRAM_ENUM_MAX);
+    allHistograms_[histogramType].Data(data);
   }
 
- private:
-  port::Mutex mu_;
   std::vector<Ticker> allTickers_;
+  std::vector<HistogramImpl> allHistograms_;
 };
+
+std::shared_ptr<Statistics> CreateDBStatistics() {
+  return std::make_shared<DBStatistics>();
 }
 
+} // namespace leveldb
+
+#endif // LEVELDB_STORAGE_DB_DB_STATISTICS_H_
+
 

db/dbformat.cc

@@ -6,6 +6,7 @@
 #include "db/dbformat.h"
 #include "port/port.h"
 #include "util/coding.h"
+#include "include/leveldb/perf_context.h"
 
 namespace leveldb {
 
@@ -44,7 +45,7 @@ std::string InternalKey::DebugString(bool hex) const {
 }
 
 const char* InternalKeyComparator::Name() const {
-  return "leveldb.InternalKeyComparator";
+  return name_.c_str();
 }
 
 int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
@@ -53,6 +54,7 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
   //    decreasing sequence number
   //    decreasing type (though sequence# should be enough to disambiguate)
   int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
+  perf_context.user_key_comparison_count++;
   if (r == 0) {
     const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
     const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);

db/dbformat.h

@@ -24,7 +24,9 @@ class InternalKey;
 // data structures.
 enum ValueType {
   kTypeDeletion = 0x0,
-  kTypeValue = 0x1
+  kTypeValue = 0x1,
+  kTypeMerge = 0x2,
+  kTypeLogData = 0x3
 };
 // kValueTypeForSeek defines the ValueType that should be passed when
 // constructing a ParsedInternalKey object for seeking to a particular
@@ -32,7 +34,7 @@ enum ValueType {
 // and the value type is embedded as the low 8 bits in the sequence
 // number in internal keys, we need to use the highest-numbered
 // ValueType, not the lowest).
-static const ValueType kValueTypeForSeek = kTypeValue;
+static const ValueType kValueTypeForSeek = kTypeMerge;
 
 // We leave eight bits empty at the bottom so a type and sequence#
 // can be packed together into 64-bits.
@@ -85,8 +87,13 @@ inline ValueType ExtractValueType(const Slice& internal_key) {
 class InternalKeyComparator : public Comparator {
  private:
   const Comparator* user_comparator_;
+  std::string name_;
  public:
-  explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) { }
+  explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c),
+    name_("leveldb.InternalKeyComparator:" +
+          std::string(user_comparator_->Name())) {
+  }
+
   virtual const char* Name() const;
   virtual int Compare(const Slice& a, const Slice& b) const;
   virtual void FindShortestSeparator(
@@ -154,9 +161,28 @@ inline bool ParseInternalKey(const Slice& internal_key,
   result->sequence = num >> 8;
   result->type = static_cast<ValueType>(c);
   result->user_key = Slice(internal_key.data(), n - 8);
-  return (c <= static_cast<unsigned char>(kTypeValue));
+  return (c <= static_cast<unsigned char>(kValueTypeForSeek));
 }
 
+// Update the sequence number in the internal key
+inline void UpdateInternalKey(char* internal_key,
+                              const size_t internal_key_size,
+                              uint64_t seq, ValueType t) {
+  assert(internal_key_size >= 8);
+  char* seqtype = internal_key + internal_key_size - 8;
+  uint64_t newval = (seq << 8) | t;
+  EncodeFixed64(seqtype, newval);
+}
+
+// Get the sequence number from the internal key
+inline uint64_t GetInternalKeySeqno(const Slice& internal_key) {
+  const size_t n = internal_key.size();
+  assert(n >= 8);
+  uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
+  return num >> 8;
+}
+
+
 // A helper class useful for DBImpl::Get()
 class LookupKey {
  public:

db/filename.cc

@@ -46,10 +46,13 @@ extern Status WriteStringToFileSync(Env* env, const Slice& data,
 static std::string MakeFileName(const std::string& name, uint64_t number,
                                 const char* suffix) {
   char buf[100];
-  snprintf(buf, sizeof(buf), "/%06llu.%s",
+  snprintf(buf, sizeof(buf), "%06llu.%s",
            static_cast<unsigned long long>(number),
            suffix);
-  return name + buf;
+  if (name.empty()) {
+    return buf;
+  }
+  return name + "/" + buf;
 }
 
 std::string LogFileName(const std::string& name, uint64_t number) {
@@ -145,7 +148,8 @@ bool ParseFileName(const std::string& fname,
     *type = kInfoLogFile;
   } else if (rest.starts_with("LOG.old.")) {
     uint64_t ts_suffix;
-    rest.remove_prefix(sizeof("LOG.old."));
+    // sizeof also counts the trailing '\0'.
+    rest.remove_prefix(sizeof("LOG.old.") - 1);
     if (!ConsumeDecimalNumber(&rest, &ts_suffix)) {
       return false;
     }

db/log_file.h

@@ -1,48 +0,0 @@
-// Copyright 2008-present Facebook. All Rights Reserved.
-
-#ifndef STORAGE_LEVELDB_DB_LOG_FILE_H_
-#define STORAGE_LEVELDB_DB_LOG_FILE_H_
-
-namespace leveldb {
-
-enum  WalFileType {
-  kArchivedLogFile = 0,
-  kAliveLogFile = 1
-} ;
-
-class LogFile {
-
- public:
-  uint64_t logNumber;
-  WalFileType type;
-
-  LogFile(uint64_t logNum,WalFileType logType) :
-    logNumber(logNum),
-    type(logType) {}
-
-  LogFile(const LogFile& that) {
-   logNumber = that.logNumber;
-   type = that.type;
-  }
-
-  bool operator < (const LogFile& that) const {
-    return logNumber < that.logNumber;
-  }
-
-  std::string ToString() const {
-    char response[100];
-    const char* typeOfLog;
-    if (type == kAliveLogFile) {
-      typeOfLog = "Alive Log";
-    } else {
-      typeOfLog = "Archived Log";
-    }
-    sprintf(response,
-            "LogNumber  : %ld LogType : %s",
-            logNumber,
-            typeOfLog);
-    return std::string(response);
-  }
-};
-}  //  namespace leveldb
-#endif  // STORAGE_LEVELDB_DB_LOG_FILE_H_

db/log_format.h

@@ -24,7 +24,7 @@ enum RecordType {
 };
 static const int kMaxRecordType = kLastType;
 
-static const int kBlockSize = 32768;
+static const unsigned int kBlockSize = 32768;
 
 // Header is checksum (4 bytes), type (1 byte), length (2 bytes).
 static const int kHeaderSize = 4 + 1 + 2;

db/log_reader.cc

@@ -15,9 +15,9 @@ namespace log {
 Reader::Reporter::~Reporter() {
 }
 
-Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
-               uint64_t initial_offset)
-    : file_(file),
+Reader::Reader(unique_ptr<SequentialFile>&& file, Reporter* reporter,
+               bool checksum, uint64_t initial_offset)
+    : file_(std::move(file)),
       reporter_(reporter),
       checksum_(checksum),
       backing_store_(new char[kBlockSize]),
@@ -170,7 +170,7 @@ void Reader::ReportCorruption(size_t bytes, const char* reason) {
 }
 
 void Reader::ReportDrop(size_t bytes, const Status& reason) {
-  if (reporter_ != NULL &&
+  if (reporter_ != nullptr &&
       end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) {
     reporter_->Corruption(bytes, reason);
   }

db/log_reader.h

@@ -5,6 +5,7 @@
 #ifndef STORAGE_LEVELDB_DB_LOG_READER_H_
 #define STORAGE_LEVELDB_DB_LOG_READER_H_
 
+#include <memory>
 #include <stdint.h>
 
 #include "db/log_format.h"
@@ -14,6 +15,7 @@
 namespace leveldb {
 
 class SequentialFile;
+using std::unique_ptr;
 
 namespace log {
 
@@ -32,7 +34,7 @@ class Reader {
   // Create a reader that will return log records from "*file".
   // "*file" must remain live while this Reader is in use.
   //
-  // If "reporter" is non-NULL, it is notified whenever some data is
+  // If "reporter" is non-nullptr, it is notified whenever some data is
   // dropped due to a detected corruption.  "*reporter" must remain
   // live while this Reader is in use.
   //
@@ -40,8 +42,8 @@ class Reader {
   //
   // The Reader will start reading at the first record located at physical
   // position >= initial_offset within the file.
-  Reader(SequentialFile* file, Reporter* reporter, bool checksum,
-         uint64_t initial_offset);
+  Reader(unique_ptr<SequentialFile>&& file, Reporter* reporter,
+         bool checksum, uint64_t initial_offset);
 
   ~Reader();
 
@@ -57,8 +59,21 @@ class Reader {
   // Undefined before the first call to ReadRecord.
   uint64_t LastRecordOffset();
 
+  // returns true if the reader has encountered an eof condition.
+  bool IsEOF() {
+    return eof_;
+  }
+
+  // when we know more data has been written to the file. we can use this
+  // function to force the reader to look again in the file.
+  void UnmarkEOF() {
+    eof_ = false;
+  }
+
+  SequentialFile* file() { return file_.get(); }
+
  private:
-  SequentialFile* const file_;
+  const unique_ptr<SequentialFile> file_;
   Reporter* const reporter_;
   bool const checksum_;
   char* const backing_store_;

db/log_test.cc

@@ -100,8 +100,26 @@ class LogTest {
     }
   };
 
-  StringDest dest_;
-  StringSource source_;
+  std::string& dest_contents() {
+    auto dest = dynamic_cast<StringDest*>(writer_.file());
+    assert(dest);
+    return dest->contents_;
+  }
+
+  const std::string& dest_contents() const {
+    auto dest = dynamic_cast<const StringDest*>(writer_.file());
+    assert(dest);
+    return dest->contents_;
+  }
+
+  void reset_source_contents() {
+    auto src = dynamic_cast<StringSource*>(reader_.file());
+    assert(src);
+    src->contents_ = dest_contents();
+  }
+
+  unique_ptr<StringDest> dest_holder_;
+  unique_ptr<StringSource> source_holder_;
   ReportCollector report_;
   bool reading_;
   Writer writer_;
@@ -112,9 +130,11 @@ class LogTest {
   static uint64_t initial_offset_last_record_offsets_[];
 
  public:
-  LogTest() : reading_(false),
-              writer_(&dest_),
-              reader_(&source_, &report_, true/*checksum*/,
+  LogTest() : dest_holder_(new StringDest),
+              source_holder_(new StringSource),
+              reading_(false),
+              writer_(std::move(dest_holder_)),
+              reader_(std::move(source_holder_), &report_, true/*checksum*/,
                       0/*initial_offset*/) {
   }
 
@@ -124,13 +144,13 @@ class LogTest {
   }
 
   size_t WrittenBytes() const {
-    return dest_.contents_.size();
+    return dest_contents().size();
   }
 
   std::string Read() {
     if (!reading_) {
       reading_ = true;
-      source_.contents_ = Slice(dest_.contents_);
+      reset_source_contents();
     }
     std::string scratch;
     Slice record;
@@ -142,26 +162,27 @@ class LogTest {
   }
 
   void IncrementByte(int offset, int delta) {
-    dest_.contents_[offset] += delta;
+    dest_contents()[offset] += delta;
   }
 
   void SetByte(int offset, char new_byte) {
-    dest_.contents_[offset] = new_byte;
+    dest_contents()[offset] = new_byte;
   }
 
   void ShrinkSize(int bytes) {
-    dest_.contents_.resize(dest_.contents_.size() - bytes);
+    dest_contents().resize(dest_contents().size() - bytes);
   }
 
   void FixChecksum(int header_offset, int len) {
     // Compute crc of type/len/data
-    uint32_t crc = crc32c::Value(&dest_.contents_[header_offset+6], 1 + len);
+    uint32_t crc = crc32c::Value(&dest_contents()[header_offset+6], 1 + len);
     crc = crc32c::Mask(crc);
-    EncodeFixed32(&dest_.contents_[header_offset], crc);
+    EncodeFixed32(&dest_contents()[header_offset], crc);
   }
 
   void ForceError() {
-    source_.force_error_ = true;
+    auto src = dynamic_cast<StringSource*>(reader_.file());
+    src->force_error_ = true;
   }
 
   size_t DroppedBytes() const {
@@ -192,22 +213,25 @@ class LogTest {
   void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) {
     WriteInitialOffsetLog();
     reading_ = true;
-    source_.contents_ = Slice(dest_.contents_);
-    Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
-                                       WrittenBytes() + offset_past_end);
+    unique_ptr<StringSource> source(new StringSource);
+    source->contents_ = dest_contents();
+    unique_ptr<Reader> offset_reader(
+      new Reader(std::move(source), &report_, true/*checksum*/,
+                 WrittenBytes() + offset_past_end));
     Slice record;
     std::string scratch;
     ASSERT_TRUE(!offset_reader->ReadRecord(&record, &scratch));
-    delete offset_reader;
   }
 
   void CheckInitialOffsetRecord(uint64_t initial_offset,
                                 int expected_record_offset) {
     WriteInitialOffsetLog();
     reading_ = true;
-    source_.contents_ = Slice(dest_.contents_);
-    Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
-                                       initial_offset);
+    unique_ptr<StringSource> source(new StringSource);
+    source->contents_ = dest_contents();
+    unique_ptr<Reader> offset_reader(
+      new Reader(std::move(source), &report_, true/*checksum*/,
+                 initial_offset));
     Slice record;
     std::string scratch;
     ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
@@ -216,7 +240,6 @@ class LogTest {
     ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
               offset_reader->LastRecordOffset());
     ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
-    delete offset_reader;
   }
 
 };
@@ -427,13 +450,13 @@ TEST(LogTest, ErrorJoinsRecords) {
   Write("correct");
 
   // Wipe the middle block
-  for (int offset = kBlockSize; offset < 2*kBlockSize; offset++) {
+  for (unsigned int offset = kBlockSize; offset < 2*kBlockSize; offset++) {
     SetByte(offset, 'x');
   }
 
   ASSERT_EQ("correct", Read());
   ASSERT_EQ("EOF", Read());
-  const int dropped = DroppedBytes();
+  const unsigned int dropped = DroppedBytes();
   ASSERT_LE(dropped, 2*kBlockSize + 100);
   ASSERT_GE(dropped, 2*kBlockSize);
 }

db/log_writer.cc

@@ -12,8 +12,8 @@
 namespace leveldb {
 namespace log {
 
-Writer::Writer(WritableFile* dest)
-    : dest_(dest),
+Writer::Writer(unique_ptr<WritableFile>&& dest)
+    : dest_(std::move(dest)),
       block_offset_(0) {
   for (int i = 0; i <= kMaxRecordType; i++) {
     char t = static_cast<char>(i);

db/log_writer.h

@@ -5,6 +5,7 @@
 #ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_
 #define STORAGE_LEVELDB_DB_LOG_WRITER_H_
 
+#include <memory>
 #include <stdint.h>
 #include "db/log_format.h"
 #include "leveldb/slice.h"
@@ -14,6 +15,8 @@ namespace leveldb {
 
 class WritableFile;
 
+using std::unique_ptr;
+
 namespace log {
 
 class Writer {
@@ -21,13 +24,16 @@ class Writer {
   // Create a writer that will append data to "*dest".
   // "*dest" must be initially empty.
   // "*dest" must remain live while this Writer is in use.
-  explicit Writer(WritableFile* dest);
+  explicit Writer(unique_ptr<WritableFile>&& dest);
   ~Writer();
 
   Status AddRecord(const Slice& slice);
 
+  WritableFile* file() { return dest_.get(); }
+  const WritableFile* file() const { return dest_.get(); }
+
  private:
-  WritableFile* dest_;
+  unique_ptr<WritableFile> dest_;
   int block_offset_;       // Current offset in block
 
   // crc32c values for all supported record types.  These are

db/memtable.cc

@@ -3,10 +3,14 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include "db/memtable.h"
+
+#include <memory>
+
 #include "db/dbformat.h"
 #include "leveldb/comparator.h"
 #include "leveldb/env.h"
 #include "leveldb/iterator.h"
+#include "leveldb/merge_operator.h"
 #include "util/coding.h"
 
 namespace leveldb {
@@ -18,21 +22,28 @@ static Slice GetLengthPrefixedSlice(const char* data) {
   return Slice(p, len);
 }
 
-MemTable::MemTable(const InternalKeyComparator& cmp, int numlevel)
+MemTable::MemTable(const InternalKeyComparator& cmp,
+                   std::shared_ptr<MemTableRepFactory> table_factory,
+                   int numlevel,
+                   const Options& options)
     : comparator_(cmp),
       refs_(0),
-      table_(comparator_, &arena_),
+      arena_impl_(options.arena_block_size),
+      table_(table_factory->CreateMemTableRep(comparator_, &arena_impl_)),
       flush_in_progress_(false),
       flush_completed_(false),
       file_number_(0),
-      edit_(numlevel) {
-}
+      edit_(numlevel),
+      first_seqno_(0),
+      mem_logfile_number_(0) { }
 
 MemTable::~MemTable() {
   assert(refs_ == 0);
 }
 
-size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); }
+size_t MemTable::ApproximateMemoryUsage() {
+  return arena_impl_.ApproximateMemoryUsage();
+}
 
 int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr)
     const {
@@ -54,24 +65,27 @@ static const char* EncodeKey(std::string* scratch, const Slice& target) {
 
 class MemTableIterator: public Iterator {
  public:
-  explicit MemTableIterator(MemTable::Table* table) : iter_(table) { }
+  explicit MemTableIterator(MemTableRep* table)
+    : iter_(table->GetIterator()) { }
 
-  virtual bool Valid() const { return iter_.Valid(); }
-  virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); }
-  virtual void SeekToFirst() { iter_.SeekToFirst(); }
-  virtual void SeekToLast() { iter_.SeekToLast(); }
-  virtual void Next() { iter_.Next(); }
-  virtual void Prev() { iter_.Prev(); }
-  virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); }
+  virtual bool Valid() const { return iter_->Valid(); }
+  virtual void Seek(const Slice& k) { iter_->Seek(EncodeKey(&tmp_, k)); }
+  virtual void SeekToFirst() { iter_->SeekToFirst(); }
+  virtual void SeekToLast() { iter_->SeekToLast(); }
+  virtual void Next() { iter_->Next(); }
+  virtual void Prev() { iter_->Prev(); }
+  virtual Slice key() const {
+    return GetLengthPrefixedSlice(iter_->key());
+  }
   virtual Slice value() const {
-    Slice key_slice = GetLengthPrefixedSlice(iter_.key());
+    Slice key_slice = GetLengthPrefixedSlice(iter_->key());
     return GetLengthPrefixedSlice(key_slice.data() + key_slice.size());
   }
 
   virtual Status status() const { return Status::OK(); }
 
  private:
-  MemTable::Table::Iterator iter_;
+  std::shared_ptr<MemTableRep::Iterator> iter_;
   std::string tmp_;       // For passing to EncodeKey
 
   // No copying allowed
@@ -80,7 +94,7 @@ class MemTableIterator: public Iterator {
 };
 
 Iterator* MemTable::NewIterator() {
-  return new MemTableIterator(&table_);
+  return new MemTableIterator(table_.get());
 }
 
 void MemTable::Add(SequenceNumber s, ValueType type,
@@ -97,7 +111,7 @@ void MemTable::Add(SequenceNumber s, ValueType type,
   const size_t encoded_len =
       VarintLength(internal_key_size) + internal_key_size +
       VarintLength(val_size) + val_size;
-  char* buf = arena_.Allocate(encoded_len);
+  char* buf = arena_impl_.Allocate(encoded_len);
   char* p = EncodeVarint32(buf, internal_key_size);
   memcpy(p, key.data(), key_size);
   p += key_size;
@@ -106,14 +120,30 @@ void MemTable::Add(SequenceNumber s, ValueType type,
   p = EncodeVarint32(p, val_size);
   memcpy(p, value.data(), val_size);
   assert((p + val_size) - buf == (unsigned)encoded_len);
-  table_.Insert(buf);
+  table_->Insert(buf);
+
+  // The first sequence number inserted into the memtable
+  assert(first_seqno_ == 0 || s > first_seqno_);
+  if (first_seqno_ == 0) {
+    first_seqno_ = s;
+  }
 }
 
-bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
+bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
+                   std::deque<std::string>* operands, const Options& options) {
   Slice memkey = key.memtable_key();
-  Table::Iterator iter(&table_);
-  iter.Seek(memkey.data());
-  if (iter.Valid()) {
+  std::shared_ptr<MemTableRep::Iterator> iter(table_.get()->GetIterator());
+  iter->Seek(memkey.data());
+
+  // It is the caller's responsibility to allocate/delete operands list
+  assert(operands != nullptr);
+
+  bool merge_in_progress = s->IsMergeInProgress();
+  auto merge_operator = options.merge_operator.get();
+  auto logger = options.info_log;
+  std::string merge_result;
+
+  for (; iter->Valid(); iter->Next()) {
     // entry format is:
     //    klength  varint32
     //    userkey  char[klength-8]
@@ -123,7 +153,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
     // Check that it belongs to same user key.  We do not check the
     // sequence number since the Seek() call above should have skipped
     // all entries with overly large sequence numbers.
-    const char* entry = iter.key();
+    const char* entry = iter->key();
     uint32_t key_length;
     const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length);
     if (comparator_.comparator.user_comparator()->Compare(
@@ -134,15 +164,68 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
       switch (static_cast<ValueType>(tag & 0xff)) {
         case kTypeValue: {
           Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
-          value->assign(v.data(), v.size());
+          *s = Status::OK();
+          if (merge_in_progress) {
+            assert(merge_operator);
+            if (!merge_operator->FullMerge(key.user_key(), &v, *operands,
+                                           value, logger.get())) {
+              RecordTick(options.statistics, NUMBER_MERGE_FAILURES);
+              *s = Status::Corruption("Error: Could not perform merge.");
+            }
+          } else {
+            value->assign(v.data(), v.size());
+          }
           return true;
         }
-        case kTypeDeletion:
-          *s = Status::NotFound(Slice());
+        case kTypeDeletion: {
+          if (merge_in_progress) {
+            assert(merge_operator);
+            *s = Status::OK();
+            if (!merge_operator->FullMerge(key.user_key(), nullptr, *operands,
+                                           value, logger.get())) {
+              RecordTick(options.statistics, NUMBER_MERGE_FAILURES);
+              *s = Status::Corruption("Error: Could not perform merge.");
+            }
+          } else {
+            *s = Status::NotFound(Slice());
+          }
           return true;
+        }
+        case kTypeMerge: {
+          Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
+          merge_in_progress = true;
+          operands->push_front(v.ToString());
+          while(operands->size() >= 2) {
+            // Attempt to associative merge. (Returns true if successful)
+            if (merge_operator->PartialMerge(key.user_key(),
+                                             Slice((*operands)[0]),
+                                             Slice((*operands)[1]),
+                                             &merge_result,
+                                             logger.get())) {
+              operands->pop_front();
+              swap(operands->front(), merge_result);
+            } else {
+              // Stack them because user can't associative merge
+              break;
+            }
+          }
+          break;
+        }
+        case kTypeLogData:
+          assert(false);
+          break;
       }
+    } else {
+      // exit loop if user key does not match
+      break;
     }
   }
+
+  // No change to value, since we have not yet found a Put/Delete
+
+  if (merge_in_progress) {
+    *s = Status::MergeInProgress("");
+  }
   return false;
 }
 

db/memtable.h

@@ -6,24 +6,35 @@
 #define STORAGE_LEVELDB_DB_MEMTABLE_H_
 
 #include <string>
-#include "leveldb/db.h"
+#include <memory>
+#include <deque>
 #include "db/dbformat.h"
 #include "db/skiplist.h"
 #include "db/version_set.h"
-#include "util/arena.h"
+#include "leveldb/db.h"
+#include "leveldb/memtablerep.h"
+#include "util/arena_impl.h"
 
 namespace leveldb {
 
-class InternalKeyComparator;
 class Mutex;
 class MemTableIterator;
 
 class MemTable {
  public:
+  struct KeyComparator : public MemTableRep::KeyComparator {
+    const InternalKeyComparator comparator;
+    explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { }
+    virtual int operator()(const char* a, const char* b) const;
+  };
+
   // MemTables are reference counted.  The initial reference count
   // is zero and the caller must call Ref() at least once.
-  explicit MemTable(const InternalKeyComparator& comparator,
-                    int numlevel = 7);
+  explicit MemTable(
+    const InternalKeyComparator& comparator,
+    std::shared_ptr<MemTableRepFactory> table_factory,
+    int numlevel = 7,
+    const Options& options = Options());
 
   // Increase reference count.
   void Ref() { ++refs_; }
@@ -62,30 +73,39 @@ class MemTable {
   // If memtable contains a value for key, store it in *value and return true.
   // If memtable contains a deletion for key, store a NotFound() error
   // in *status and return true.
+  // If memtable contains Merge operation as the most recent entry for a key,
+  //   and the merge process does not stop (not reaching a value or delete),
+  //   prepend the current merge operand to *operands.
+  //   store MergeInProgress in s, and return false.
   // Else, return false.
-  bool Get(const LookupKey& key, std::string* value, Status* s);
+  bool Get(const LookupKey& key, std::string* value, Status* s,
+           std::deque<std::string>* operands, const Options& options);
 
   // Returns the edits area that is needed for flushing the memtable
   VersionEdit* GetEdits() { return &edit_; }
 
+  // Returns the sequence number of the first element that was inserted
+  // into the memtable
+  SequenceNumber GetFirstSequenceNumber() { return first_seqno_; }
+
+  // Returns the logfile number that can be safely deleted when this
+  // memstore is flushed to storage
+  uint64_t GetLogNumber() { return mem_logfile_number_; }
+
+  // Sets the logfile number that can be safely deleted when this
+  // memstore is flushed to storage
+  void SetLogNumber(uint64_t num) { mem_logfile_number_ = num; }
+
  private:
   ~MemTable();  // Private since only Unref() should be used to delete it
-
-  struct KeyComparator {
-    const InternalKeyComparator comparator;
-    explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { }
-    int operator()(const char* a, const char* b) const;
-  };
   friend class MemTableIterator;
   friend class MemTableBackwardIterator;
   friend class MemTableList;
 
-  typedef SkipList<const char*, KeyComparator> Table;
-
   KeyComparator comparator_;
   int refs_;
-  Arena arena_;
-  Table table_;
+  ArenaImpl arena_impl_;
+  shared_ptr<MemTableRep> table_;
 
   // These are used to manage memtable flushes to storage
   bool flush_in_progress_; // started the flush
@@ -96,6 +116,12 @@ class MemTable {
   // memtable is flushed to storage.
   VersionEdit edit_;
 
+  // The sequence number of the kv that was inserted first
+  SequenceNumber first_seqno_;
+
+  // The log files earlier than this number can be deleted.
+  uint64_t mem_logfile_number_;
+
   // No copying allowed
   MemTable(const MemTable&);
   void operator=(const MemTable&);

db/memtablelist.cc

@@ -21,17 +21,15 @@ using std::list;
 
 // Increase reference count on all underling memtables
 void MemTableList::RefAll() {
-  for (list<MemTable*>::iterator it = memlist_.begin();
-       it != memlist_.end() ; ++it) {
-    (*it)->Ref();
+  for (auto &memtable : memlist_) {
+    memtable->Ref();
   }
 }
 
 // Drop reference count on all underling memtables
 void MemTableList::UnrefAll() {
-  for (list<MemTable*>::iterator it = memlist_.begin();
-       it != memlist_.end() ; ++it) {
-    (*it)->Unref();
+  for (auto &memtable : memlist_) {
+    memtable->Unref();
   }
 }
 
@@ -43,58 +41,67 @@ int MemTableList::size() {
 
 // Returns true if there is at least one memtable on which flush has
 // not yet started.
-bool MemTableList::IsFlushPending() {
-  if (num_flush_not_started_ > 0) {
-    assert(imm_flush_needed.NoBarrier_Load() != NULL);
+bool MemTableList::IsFlushPending(int min_write_buffer_number_to_merge) {
+  if (num_flush_not_started_ >= min_write_buffer_number_to_merge) {
+    assert(imm_flush_needed.NoBarrier_Load() != nullptr);
     return true;
   }
   return false;
 }
 
-// Returns the earliest memtable that needs to be flushed.
-// Returns null, if no such memtable exist.
-MemTable* MemTableList::PickMemtableToFlush() {
-  for (list<MemTable*>::reverse_iterator it = memlist_.rbegin();
-       it != memlist_.rend(); it++) {
+// Returns the memtables that need to be flushed.
+void MemTableList::PickMemtablesToFlush(std::vector<MemTable*>* ret) {
+  for (auto it = memlist_.rbegin(); it != memlist_.rend(); it++) {
     MemTable* m = *it;
     if (!m->flush_in_progress_) {
       assert(!m->flush_completed_);
       num_flush_not_started_--;
       if (num_flush_not_started_ == 0) {
-        imm_flush_needed.Release_Store(NULL);
+        imm_flush_needed.Release_Store(nullptr);
       }
       m->flush_in_progress_ = true; // flushing will start very soon
-      return m;
+      ret->push_back(m);
     }
   }
-  return NULL;
 }
 
 // Record a successful flush in the manifest file
-Status MemTableList::InstallMemtableFlushResults(MemTable* m,
+Status MemTableList::InstallMemtableFlushResults(
+                      const std::vector<MemTable*> &mems,
                       VersionSet* vset, Status flushStatus,
                       port::Mutex* mu, Logger* info_log,
                       uint64_t file_number,
                       std::set<uint64_t>& pending_outputs) {
   mu->AssertHeld();
-  assert(m->flush_in_progress_);
-  assert(m->file_number_ == 0);
 
   // If the flush was not successful, then just reset state.
   // Maybe a suceeding attempt to flush will be successful.
   if (!flushStatus.ok()) {
-    m->flush_in_progress_ = false;
-    m->flush_completed_ = false;
-    m->edit_.Clear();
-    num_flush_not_started_++;
-    imm_flush_needed.Release_Store((void *)1);
-    pending_outputs.erase(file_number);
+    for (MemTable* m : mems) {
+      assert(m->flush_in_progress_);
+      assert(m->file_number_ == 0);
+
+      m->flush_in_progress_ = false;
+      m->flush_completed_ = false;
+      m->edit_.Clear();
+      num_flush_not_started_++;
+      imm_flush_needed.Release_Store((void *)1);
+      pending_outputs.erase(file_number);
+    }
     return flushStatus;
   }
 
   // flush was sucessful
-  m->flush_completed_ = true;
-  m->file_number_ = file_number;
+  bool first = true;
+  for (MemTable* m : mems) {
+
+    // All the edits are associated with the first memtable of this batch.
+    assert(first || m->GetEdits()->NumEntries() == 0);
+    first = false;
+
+    m->flush_completed_ = true;
+    m->file_number_ = file_number;
+  }
 
   // if some other thread is already commiting, then return
   Status s;
@@ -106,12 +113,15 @@ Status MemTableList::InstallMemtableFlushResults(MemTable* m,
   commit_in_progress_ = true;
 
   // scan all memtables from the earliest, and commit those
-  // (in that order) that have finished flushing.
-  while (!memlist_.empty()) {
-    m = memlist_.back(); // get the last element
+  // (in that order) that have finished flushing. Memetables
+  // are always committed in the order that they were created.
+  while (!memlist_.empty() && s.ok()) {
+    MemTable* m = memlist_.back(); // get the last element
     if (!m->flush_completed_) {
       break;
     }
+    first = true;
+
     Log(info_log,
         "Level-0 commit table #%llu: started",
         (unsigned long long)m->file_number_);
@@ -119,33 +129,39 @@ Status MemTableList::InstallMemtableFlushResults(MemTable* m,
     // this can release and reacquire the mutex.
     s = vset->LogAndApply(&m->edit_, mu);
 
-    if (s.ok()) { // commit new state
-      Log(info_log, "Level-0 commit table #%llu: done",
-                     (unsigned long long)m->file_number_);
-      memlist_.remove(m);
-      assert(m->file_number_ > 0);
+    // All the later memtables that have the same filenum
+    // are part of the same batch. They can be committed now.
+    do {
+      if (s.ok()) { // commit new state
+        Log(info_log, "Level-0 commit table #%llu: done %s",
+                       (unsigned long long)m->file_number_,
+                       first ? "": "bulk");
+        memlist_.remove(m);
+        assert(m->file_number_ > 0);
 
-      // pending_outputs can be cleared only after the newly created file
-      // has been written to a committed version so that other concurrently
-      // executing compaction threads do not mistakenly assume that this
-      // file is not live.
-      pending_outputs.erase(m->file_number_);
-      m->Unref();
-      size_--;
-    } else {
-      //commit failed. setup state so that we can flush again.
-      Log(info_log, "Level-0 commit table #%llu: failed",
-                     (unsigned long long)m->file_number_);
-      m->flush_completed_ = false;
-      m->flush_in_progress_ = false;
-      m->edit_.Clear();
-      num_flush_not_started_++;
-      pending_outputs.erase(m->file_number_);
-      m->file_number_ = 0;
-      imm_flush_needed.Release_Store((void *)1);
-      s = Status::IOError("Unable to commit flushed memtable");
-      break;
-    }
+        // pending_outputs can be cleared only after the newly created file
+        // has been written to a committed version so that other concurrently
+        // executing compaction threads do not mistakenly assume that this
+        // file is not live.
+        pending_outputs.erase(m->file_number_);
+        m->Unref();
+        size_--;
+      } else {
+        //commit failed. setup state so that we can flush again.
+        Log(info_log, "Level-0 commit table #%llu: failed",
+                       (unsigned long long)m->file_number_);
+        m->flush_completed_ = false;
+        m->flush_in_progress_ = false;
+        m->edit_.Clear();
+        num_flush_not_started_++;
+        pending_outputs.erase(m->file_number_);
+        m->file_number_ = 0;
+        imm_flush_needed.Release_Store((void *)1);
+        s = Status::IOError("Unable to commit flushed memtable");
+      }
+      first = false;
+    } while (!memlist_.empty() && (m = memlist_.back()) &&
+             m->file_number_ == file_number);
   }
   commit_in_progress_ = false;
   return s;
@@ -165,19 +181,20 @@ void MemTableList::Add(MemTable* m) {
 // Returns an estimate of the number of bytes of data in use.
 size_t MemTableList::ApproximateMemoryUsage() {
   size_t size = 0;
-  for (list<MemTable*>::iterator it = memlist_.begin();
-       it != memlist_.end(); ++it) {
-    size += (*it)->ApproximateMemoryUsage();
+  for (auto &memtable : memlist_) {
+    size += memtable->ApproximateMemoryUsage();
   }
   return size;
 }
 
 // Search all the memtables starting from the most recent one.
 // Return the most recent value found, if any.
-bool MemTableList::Get(const LookupKey& key, std::string* value, Status* s) {
-  for (list<MemTable*>::iterator it = memlist_.begin();
-       it != memlist_.end(); ++it) {
-    if ((*it)->Get(key, value, s)) {
+// Operands stores the list of merge operations to apply, so far.
+bool MemTableList::Get(const LookupKey& key, std::string* value, Status* s,
+                       std::deque<std::string>* operands,
+                       const Options& options) {
+  for (auto &memtable : memlist_) {
+    if (memtable->Get(key, value, s, operands, options)) {
       return true;
     }
   }
@@ -185,9 +202,8 @@ bool MemTableList::Get(const LookupKey& key, std::string* value, Status* s) {
 }
 
 void MemTableList::GetMemTables(std::vector<MemTable*>* output) {
-  for (list<MemTable*>::iterator it = memlist_.begin();
-       it != memlist_.end(); ++it) {
-    output->push_back(*it);
+  for (auto &memtable : memlist_) {
+    output->push_back(memtable);
   }
 }
 

db/memtablelist.h

@@ -5,10 +5,10 @@
 
 #include <string>
 #include <list>
+#include <deque>
 #include "leveldb/db.h"
 #include "db/dbformat.h"
 #include "db/skiplist.h"
-#include "util/arena.h"
 #include "memtable.h"
 
 namespace leveldb {
@@ -18,7 +18,7 @@ class Mutex;
 class MemTableListIterator;
 
 //
-// This class stores refeernces to all the immutable memtables.
+// This class stores references to all the immutable memtables.
 // The memtables are flushed to L0 as soon as possible and in
 // any order. If there are more than one immutable memtable, their
 // flushes can occur concurrently.  However, they are 'committed'
@@ -30,11 +30,11 @@ class MemTableList {
   // A list of memtables.
   MemTableList() : size_(0), num_flush_not_started_(0),
     commit_in_progress_(false) {
-    imm_flush_needed.Release_Store(NULL);
+    imm_flush_needed.Release_Store(nullptr);
   }
   ~MemTableList() {};
 
-  // so that backgrund threads can detect non-NULL pointer to
+  // so that backgrund threads can detect non-nullptr pointer to
   // determine whether this is anything more to start flushing.
   port::AtomicPointer imm_flush_needed;
 
@@ -49,14 +49,13 @@ class MemTableList {
 
   // Returns true if there is at least one memtable on which flush has
   // not yet started.
-  bool IsFlushPending();
+  bool IsFlushPending(int min_write_buffer_number_to_merge);
 
-  // Returns the earliest memtable that needs to be flushed.
-  // Returns null, if no such memtable exist.
-  MemTable* PickMemtableToFlush();
+  // Returns the earliest memtables that needs to be flushed.
+  void PickMemtablesToFlush(std::vector<MemTable*>* mems);
 
   // Commit a successful flush in the manifest file
-  Status InstallMemtableFlushResults(MemTable* m,
+  Status InstallMemtableFlushResults(const std::vector<MemTable*> &m,
                       VersionSet* vset, Status flushStatus,
                       port::Mutex* mu, Logger* info_log,
                       uint64_t file_number,
@@ -71,7 +70,8 @@ class MemTableList {
 
   // Search all the memtables starting from the most recent one.
   // Return the most recent value found, if any.
-  bool Get(const LookupKey& key, std::string* value, Status* s);
+  bool Get(const LookupKey& key, std::string* value, Status* s,
+           std::deque<std::string>* operands, const Options& options);
 
   // Returns the list of underlying memtables.
   void GetMemTables(std::vector<MemTable*>* list);

db/merge_helper.cc

@@ -0,0 +1,192 @@
+#include "merge_helper.h"
+#include "db/dbformat.h"
+#include "leveldb/comparator.h"
+#include "leveldb/db.h"
+#include "leveldb/merge_operator.h"
+#include <string>
+#include <stdio.h>
+
+namespace leveldb {
+
+// PRE:  iter points to the first merge type entry
+// POST: iter points to the first entry beyond the merge process (or the end)
+//       keys_, operands_ are updated to reflect the merge result.
+//       keys_ stores the list of keys encountered while merging.
+//       operands_ stores the list of merge operands encountered while merging.
+//       keys_[i] corresponds to operands_[i] for each i.
+void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
+                             bool at_bottom, shared_ptr<Statistics> stats) {
+  // Get a copy of the internal key, before it's invalidated by iter->Next()
+  // Also maintain the list of merge operands seen.
+  keys_.clear();
+  operands_.clear();
+  keys_.push_front(iter->key().ToString());
+  operands_.push_front(iter->value().ToString());
+
+  success_ = false;   // Will become true if we hit Put/Delete or bottom
+
+  // We need to parse the internal key again as the parsed key is
+  // backed by the internal key!
+  // Assume no internal key corruption as it has been successfully parsed
+  // by the caller.
+  // Invariant: keys_.back() will not change. Hence, orig_ikey is always valid.
+  ParsedInternalKey orig_ikey;
+  ParseInternalKey(keys_.back(), &orig_ikey);
+
+  bool hit_the_next_user_key = false;
+  ParsedInternalKey ikey;
+  std::string merge_result;  // Temporary value for merge results
+  for (iter->Next(); iter->Valid(); iter->Next()) {
+    assert(operands_.size() >= 1);        // Should be invariants!
+    assert(keys_.size() == operands_.size());
+
+    if (!ParseInternalKey(iter->key(), &ikey)) {
+      // stop at corrupted key
+      if (assert_valid_internal_key_) {
+        assert(!"corrupted internal key is not expected");
+      }
+      break;
+    }
+
+    if (user_comparator_->Compare(ikey.user_key, orig_ikey.user_key) != 0) {
+      // hit a different user key, stop right here
+      hit_the_next_user_key = true;
+      break;
+    }
+
+    if (stop_before && ikey.sequence <= stop_before) {
+      // hit an entry that's visible by the previous snapshot, can't touch that
+      break;
+    }
+
+    // At this point we are guaranteed that we need to process this key.
+
+    if (kTypeDeletion == ikey.type) {
+      // hit a delete
+      //   => merge nullptr with operands_
+      //   => store result in operands_.back() (and update keys_.back())
+      //   => change the entry type to kTypeValue for keys_.back()
+      // We are done! Return a success if the merge passes.
+      success_ = user_merge_operator_->FullMerge(ikey.user_key, nullptr,
+                                                 operands_, &merge_result,
+                                                 logger_);
+
+      // We store the result in keys_.back() and operands_.back()
+      // if nothing went wrong (i.e.: no operand corruption on disk)
+      if (success_) {
+        std::string& key = keys_.back();  // The original key encountered
+        orig_ikey.type = kTypeValue;
+        UpdateInternalKey(&key[0], key.size(),
+                          orig_ikey.sequence, orig_ikey.type);
+        swap(operands_.back(), merge_result);
+      } else {
+        RecordTick(stats, NUMBER_MERGE_FAILURES);
+      }
+
+      // move iter to the next entry (before doing anything else)
+      iter->Next();
+      return;
+    }
+
+    if (kTypeValue == ikey.type) {
+      // hit a put
+      //   => merge the put value with operands_
+      //   => store result in operands_.back() (and update keys_.back())
+      //   => change the entry type to kTypeValue for keys_.back()
+      // We are done! Success!
+      const Slice value = iter->value();
+      success_ = user_merge_operator_->FullMerge(ikey.user_key, &value,
+                                                 operands_, &merge_result,
+                                                 logger_);
+
+      // We store the result in keys_.back() and operands_.back()
+      // if nothing went wrong (i.e.: no operand corruption on disk)
+      if (success_) {
+        std::string& key = keys_.back();  // The original key encountered
+        orig_ikey.type = kTypeValue;
+        UpdateInternalKey(&key[0], key.size(),
+                          orig_ikey.sequence, orig_ikey.type);
+        swap(operands_.back(), merge_result);
+      } else {
+        RecordTick(stats, NUMBER_MERGE_FAILURES);
+      }
+
+      // move iter to the next entry
+      iter->Next();
+      return;
+    }
+
+    if (kTypeMerge == ikey.type) {
+      // hit a merge
+      //   => merge the operand into the front of the operands_ list
+      //   => use the user's associative merge function to determine how.
+      //   => then continue because we haven't yet seen a Put/Delete.
+      assert(!operands_.empty()); // Should have at least one element in it
+
+      keys_.push_front(iter->key().ToString());
+      operands_.push_front(iter->value().ToString());
+      while (operands_.size() >= 2) {
+        // Returns false when the merge_operator can no longer process it
+        if (user_merge_operator_->PartialMerge(ikey.user_key,
+                                               Slice(operands_[0]),
+                                               Slice(operands_[1]),
+                                               &merge_result,
+                                               logger_)) {
+          // Merging of operands (associative merge) was successful.
+          // Replace these frontmost two operands with the merge result
+          keys_.pop_front();
+          operands_.pop_front();
+          swap(operands_.front(), merge_result);
+        } else {
+          // Merging of operands (associative merge) returned false.
+          // The user merge_operator does not know how to merge these operands.
+          // So we just stack them up until we find a Put/Delete or end of key.
+          break;
+        }
+      }
+      continue;
+    }
+  }
+
+  // We are sure we have seen this key's entire history if we are at the
+  // last level and exhausted all internal keys of this user key.
+  // NOTE: !iter->Valid() does not necessarily mean we hit the
+  // beginning of a user key, as versions of a user key might be
+  // split into multiple files (even files on the same level)
+  // and some files might not be included in the compaction/merge.
+  //
+  // There are also cases where we have seen the root of history of this
+  // key without being sure of it. Then, we simply miss the opportunity
+  // to combine the keys. Since VersionSet::SetupOtherInputs() always makes
+  // sure that all merge-operands on the same level get compacted together,
+  // this will simply lead to these merge operands moving to the next level.
+  //
+  // So, we only perform the following logic (to merge all operands together
+  // without a Put/Delete) if we are certain that we have seen the end of key.
+  bool surely_seen_the_beginning = hit_the_next_user_key && at_bottom;
+  if (surely_seen_the_beginning) {
+    // do a final merge with nullptr as the existing value and say
+    // bye to the merge type (it's now converted to a Put)
+    assert(kTypeMerge == orig_ikey.type);
+    assert(operands_.size() >= 1);
+    assert(operands_.size() == keys_.size());
+    success_ = user_merge_operator_->FullMerge(ikey.user_key, nullptr,
+                                               operands_, &merge_result,
+                                               logger_);
+
+    if (success_) {
+      std::string& key = keys_.back();  // The original key encountered
+      orig_ikey.type = kTypeValue;
+      UpdateInternalKey(&key[0], key.size(),
+                        orig_ikey.sequence, orig_ikey.type);
+
+      // The final value() is always stored in operands_.back()
+      swap(operands_.back(),merge_result);
+    } else {
+      RecordTick(stats, NUMBER_MERGE_FAILURES);
+      // Do nothing if not success_. Leave keys() and operands() as they are.
+    }
+  }
+}
+
+} // namespace leveldb

db/merge_helper.h

@@ -0,0 +1,98 @@
+#ifndef MERGE_HELPER_H
+#define MERGE_HELPER_H
+
+#include "db/dbformat.h"
+#include "leveldb/slice.h"
+#include "leveldb/statistics.h"
+#include <string>
+#include <deque>
+
+namespace leveldb {
+
+class Comparator;
+class Iterator;
+class Logger;
+class MergeOperator;
+
+class MergeHelper {
+ public:
+  MergeHelper(const Comparator* user_comparator,
+              const MergeOperator* user_merge_operator,
+              Logger* logger,
+              bool assert_valid_internal_key)
+      : user_comparator_(user_comparator),
+        user_merge_operator_(user_merge_operator),
+        logger_(logger),
+        assert_valid_internal_key_(assert_valid_internal_key),
+        keys_(),
+        operands_(),
+        success_(false) {}
+
+  // Merge entries until we hit
+  //     - a corrupted key
+  //     - a Put/Delete,
+  //     - a different user key,
+  //     - a specific sequence number (snapshot boundary),
+  //  or - the end of iteration
+  // iter: (IN)  points to the first merge type entry
+  //       (OUT) points to the first entry not included in the merge process
+  // stop_before: (IN) a sequence number that merge should not cross.
+  //                   0 means no restriction
+  // at_bottom:   (IN) true if the iterator covers the bottem level, which means
+  //                   we could reach the start of the history of this user key.
+  void MergeUntil(Iterator* iter, SequenceNumber stop_before = 0,
+                  bool at_bottom = false, shared_ptr<Statistics> stats=nullptr);
+
+  // Query the merge result
+  // These are valid until the next MergeUntil call
+  // If the merging was successful:
+  //   - IsSuccess() will be true
+  //   - key() will have the latest sequence number of the merges.
+  //           The type will be Put or Merge. See IMPORTANT 1 note, below.
+  //   - value() will be the result of merging all the operands together
+  //   - The user should ignore keys() and values().
+  //
+  //   IMPORTANT 1: the key type could change after the MergeUntil call.
+  //        Put/Delete + Merge + ... + Merge => Put
+  //        Merge + ... + Merge => Merge
+  //
+  // If the merge operator is not associative, and if a Put/Delete is not found
+  // then the merging will be unsuccessful. In this case:
+  //   - IsSuccess() will be false
+  //   - keys() contains the list of internal keys seen in order of iteration.
+  //   - values() contains the list of values (merges) seen in the same order.
+  //              values() is parallel to keys() so that the first entry in
+  //              keys() is the key associated with the first entry in values()
+  //              and so on. These lists will be the same length.
+  //              All of these pairs will be merges over the same user key.
+  //              See IMPORTANT 2 note below.
+  //   - The user should ignore key() and value().
+  //
+  //   IMPORTANT 2: The entries were traversed in order from BACK to FRONT.
+  //                So keys().back() was the first key seen by iterator.
+  // TODO: Re-style this comment to be like the first one
+  bool IsSuccess() { return success_; }
+  Slice key() { assert(success_); return Slice(keys_.back()); }
+  Slice value() { assert(success_); return Slice(operands_.back()); }
+  const std::deque<std::string>& keys() { assert(!success_); return keys_; }
+  const std::deque<std::string>& values() {
+    assert(!success_); return operands_;
+  }
+
+ private:
+  const Comparator* user_comparator_;
+  const MergeOperator* user_merge_operator_;
+  Logger* logger_;
+  Iterator* iter_; // in: the internal iterator, positioned at the first merge entry
+  bool assert_valid_internal_key_; // enforce no internal key corruption?
+
+  // the scratch area that holds the result of MergeUntil
+  // valid up to the next MergeUntil call
+  std::deque<std::string> keys_;    // Keeps track of the sequence of keys seen
+  std::deque<std::string> operands_;  // Parallel with keys_; stores the values
+  bool success_;
+};
+
+} // namespace leveldb
+
+#endif

db/merge_operator.cc

@@ -0,0 +1,51 @@
+/**
+ * Back-end implementation details specific to the Merge Operator.
+ *
+ * @author Deon Nicholas (dnicholas@fb.com)
+ * Copyright 2013 Facebook
+ */
+
+#include "leveldb/merge_operator.h"
+
+namespace leveldb {
+
+// Given a "real" merge from the library, call the user's
+// associative merge function one-by-one on each of the operands.
+// NOTE: It is assumed that the client's merge-operator will handle any errors.
+bool AssociativeMergeOperator::FullMerge(
+    const Slice& key,
+    const Slice* existing_value,
+    const std::deque<std::string>& operand_list,
+    std::string* new_value,
+    Logger* logger) const {
+
+  // Simply loop through the operands
+  Slice temp_existing;
+  std::string temp_value;
+  for (const auto& operand : operand_list) {
+    Slice value(operand);
+    if (!Merge(key, existing_value, value, &temp_value, logger)) {
+      return false;
+    }
+    swap(temp_value, *new_value);
+    temp_existing = Slice(*new_value);
+    existing_value = &temp_existing;
+  }
+
+  // The result will be in *new_value. All merges succeeded.
+  return true;
+}
+
+// Call the user defined simple merge on the operands;
+// NOTE: It is assumed that the client's merge-operator will handle any errors.
+bool AssociativeMergeOperator::PartialMerge(
+    const Slice& key,
+    const Slice& left_operand,
+    const Slice& right_operand,
+    std::string* new_value,
+    Logger* logger) const {
+
+  return Merge(key, &left_operand, right_operand, new_value, logger);
+}
+
+} // namespace leveldb

db/merge_test.cc

@@ -0,0 +1,270 @@
+#include <assert.h>
+#include <memory>
+#include <iostream>
+
+#include "leveldb/cache.h"
+#include "leveldb/comparator.h"
+#include "leveldb/db.h"
+#include "leveldb/env.h"
+#include "leveldb/merge_operator.h"
+#include "db/dbformat.h"
+#include "db/db_impl.h"
+#include "utilities/merge_operators.h"
+#include "util/testharness.h"
+#include "utilities/utility_db.h"
+
+using namespace std;
+using namespace leveldb;
+
+
+std::shared_ptr<DB> OpenDb(const string& dbname, const bool ttl = false) {
+  DB* db;
+  StackableDB* sdb;
+  Options options;
+  options.create_if_missing = true;
+  options.merge_operator = MergeOperators::CreateUInt64AddOperator();
+  Status s;
+  DestroyDB(dbname, Options());
+  if (ttl) {
+    cout << "Opening database with TTL\n";
+    s = UtilityDB::OpenTtlDB(options, dbname, &sdb);
+    db = sdb;
+  } else {
+    s = DB::Open(options, dbname, &db);
+  }
+  if (!s.ok()) {
+    cerr << s.ToString() << endl;
+    assert(false);
+  }
+  return std::shared_ptr<DB>(db);
+}
+
+// Imagine we are maintaining a set of uint64 counters.
+// Each counter has a distinct name. And we would like
+// to support four high level operations:
+// set, add, get and remove
+// This is a quick implementation without a Merge operation.
+class Counters {
+
+ protected:
+  std::shared_ptr<DB> db_;
+
+  WriteOptions put_option_;
+  ReadOptions get_option_;
+  WriteOptions delete_option_;
+
+  uint64_t default_;
+
+ public:
+  explicit Counters(std::shared_ptr<DB> db, uint64_t defaultCount = 0)
+      : db_(db),
+        put_option_(),
+        get_option_(),
+        delete_option_(),
+        default_(defaultCount) {
+    assert(db_);
+  }
+
+  virtual ~Counters() {}
+
+  // public interface of Counters.
+  // All four functions return false
+  // if the underlying level db operation failed.
+
+  // mapped to a levedb Put
+  bool set(const string& key, uint64_t value) {
+    // just treat the internal rep of int64 as the string
+    Slice slice((char *)&value, sizeof(value));
+    auto s = db_->Put(put_option_, key, slice);
+
+    if (s.ok()) {
+      return true;
+    } else {
+      cerr << s.ToString() << endl;
+      return false;
+    }
+  }
+
+  // mapped to a leveldb Delete
+  bool remove(const string& key) {
+    auto s = db_->Delete(delete_option_, key);
+
+    if (s.ok()) {
+      return true;
+    } else {
+      cerr << s.ToString() << std::endl;
+      return false;
+    }
+  }
+
+  // mapped to a leveldb Get
+  bool get(const string& key, uint64_t *value) {
+    string str;
+    auto s = db_->Get(get_option_, key, &str);
+
+    if (s.IsNotFound()) {
+      // return default value if not found;
+      *value = default_;
+      return true;
+    } else if (s.ok()) {
+      // deserialization
+      if (str.size() != sizeof(uint64_t)) {
+        cerr << "value corruption\n";
+        return false;
+      }
+      *value = DecodeFixed64(&str[0]);
+      return true;
+    } else {
+      cerr << s.ToString() << std::endl;
+      return false;
+    }
+  }
+
+  // 'add' is implemented as get -> modify -> set
+  // An alternative is a single merge operation, see MergeBasedCounters
+  virtual bool add(const string& key, uint64_t value) {
+    uint64_t base = default_;
+    return get(key, &base) && set(key, base + value);
+  }
+
+
+  // convenience functions for testing
+  void assert_set(const string& key, uint64_t value) {
+    assert(set(key, value));
+  }
+
+  void assert_remove(const string& key) {
+    assert(remove(key));
+  }
+
+  uint64_t assert_get(const string& key) {
+    uint64_t value = default_;
+    assert(get(key, &value));
+    return value;
+  }
+
+  void assert_add(const string& key, uint64_t value) {
+    assert(add(key, value));
+  }
+};
+
+// Implement 'add' directly with the new Merge operation
+class MergeBasedCounters : public Counters {
+ private:
+  WriteOptions merge_option_; // for merge
+
+ public:
+  explicit MergeBasedCounters(std::shared_ptr<DB> db, uint64_t defaultCount = 0)
+      : Counters(db, defaultCount),
+        merge_option_() {
+  }
+
+  // mapped to a leveldb Merge operation
+  virtual bool add(const string& key, uint64_t value) override {
+    char encoded[sizeof(uint64_t)];
+    EncodeFixed64(encoded, value);
+    Slice slice(encoded, sizeof(uint64_t));
+    auto s = db_->Merge(merge_option_, key, slice);
+
+    if (s.ok()) {
+      return true;
+    } else {
+      cerr << s.ToString() << endl;
+      return false;
+    }
+  }
+};
+
+void dumpDb(DB* db) {
+  auto it = unique_ptr<Iterator>(db->NewIterator(ReadOptions()));
+  for (it->SeekToFirst(); it->Valid(); it->Next()) {
+    uint64_t value = DecodeFixed64(it->value().data());
+    cout << it->key().ToString() << ": "  << value << endl;
+  }
+  assert(it->status().ok());  // Check for any errors found during the scan
+}
+
+void testCounters(Counters& counters, DB* db, bool test_compaction) {
+
+  FlushOptions o;
+  o.wait = true;
+
+  counters.assert_set("a", 1);
+
+  if (test_compaction) db->Flush(o);
+
+  assert(counters.assert_get("a") == 1);
+
+  counters.assert_remove("b");
+
+  // defaut value is 0 if non-existent
+  assert(counters.assert_get("b") == 0);
+
+  counters.assert_add("a", 2);
+
+  if (test_compaction) db->Flush(o);
+
+  // 1+2 = 3
+  assert(counters.assert_get("a")== 3);
+
+  dumpDb(db);
+
+  std::cout << "1\n";
+
+  // 1+...+49 = ?
+  uint64_t sum = 0;
+  for (int i = 1; i < 50; i++) {
+    counters.assert_add("b", i);
+    sum += i;
+  }
+  assert(counters.assert_get("b") == sum);
+
+  std::cout << "2\n";
+  dumpDb(db);
+
+  std::cout << "3\n";
+
+  if (test_compaction) {
+    db->Flush(o);
+
+    cout << "Compaction started ...\n";
+    db->CompactRange(nullptr, nullptr);
+    cout << "Compaction ended\n";
+
+    dumpDb(db);
+
+    assert(counters.assert_get("a")== 3);
+    assert(counters.assert_get("b") == sum);
+  }
+}
+
+void runTest(int argc, const string& dbname, const bool use_ttl = false) {
+  auto db = OpenDb(dbname, use_ttl);
+
+  {
+    cout << "Test read-modify-write counters... \n";
+    Counters counters(db, 0);
+    testCounters(counters, db.get(), true);
+  }
+
+  bool compact = false;
+  if (argc > 1) {
+    compact = true;
+    cout << "Turn on Compaction\n";
+  }
+
+  {
+    cout << "Test merge-based counters... \n";
+    MergeBasedCounters counters(db, 0);
+    testCounters(counters, db.get(), compact);
+  }
+
+  DestroyDB(dbname, Options());
+}
+
+int main(int argc, char *argv[]) {
+  //TODO: Make this test like a general rocksdb unit-test
+  runTest(argc, test::TmpDir() + "/merge_testdb");
+  runTest(argc, test::TmpDir() + "/merge_testdbttl", true); // Run test on TTL database
+  return 0;
+}

db/perf_context.cc

@@ -0,0 +1,12 @@
+#include "include/leveldb/perf_context.h"
+
+
+namespace leveldb {
+
+void PerfContext::Reset() {
+  user_key_comparison_count = 0;
+}
+
+__thread PerfContext perf_context;
+
+}

db/perf_context_test.cc

@@ -0,0 +1,81 @@
+#include <iostream>
+
+#include "leveldb/db.h"
+#include "leveldb/perf_context.h"
+#include "util/testharness.h"
+
+
+namespace leveldb {
+
+// Path to the database on file system
+const std::string kDbName = test::TmpDir() + "/perf_context_test";
+
+std::shared_ptr<DB> OpenDb() {
+    DB* db;
+    Options options;
+    options.create_if_missing = true;
+    options.write_buffer_size = 1000000000;     // give it a big memtable
+    Status s = DB::Open(options, kDbName,  &db);
+    ASSERT_OK(s);
+    return std::shared_ptr<DB>(db);
+}
+
+class PerfContextTest { };
+
+int kTotalKeys = 100;
+
+TEST(PerfContextTest, KeyComparisonCount) {
+
+  DestroyDB(kDbName, Options());    // Start this test with a fresh DB
+
+  auto db = OpenDb();
+
+  WriteOptions write_options;
+  ReadOptions read_options;
+
+  uint64_t total_user_key_comparison_get = 0;
+  uint64_t total_user_key_comparison_put = 0;
+  uint64_t max_user_key_comparison_get = 0;
+
+  std::cout << "Inserting " << kTotalKeys << " key/value pairs\n...\n";
+
+  for (int i = 0; i < kTotalKeys; ++i) {
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
+
+    perf_context.Reset();
+    db->Put(write_options, key, value);
+    total_user_key_comparison_put += perf_context.user_key_comparison_count;
+
+    perf_context.Reset();
+    db->Get(read_options, key, &value);
+    total_user_key_comparison_get += perf_context.user_key_comparison_count;
+    max_user_key_comparison_get =
+      std::max(max_user_key_comparison_get,
+               perf_context.user_key_comparison_count);
+  }
+
+  std::cout << "total user key comparison get: "
+            << total_user_key_comparison_get << "\n"
+            << "total user key comparison put: "
+            << total_user_key_comparison_put << "\n"
+            << "max user key comparison get: "
+            << max_user_key_comparison_get << "\n"
+            << "avg user key comparison get:"
+            << total_user_key_comparison_get/kTotalKeys << "\n";
+
+}
+
+
+}
+
+
+int main(int argc, char** argv) {
+
+  if (argc > 1) {
+    leveldb::kTotalKeys = std::stoi(argv[1]);
+  }
+
+  leveldb::test::RunAllTests();
+  return 0;
+}

db/prefix_filter_iterator.h

@@ -0,0 +1,76 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// Wrap an underlying iterator, but exclude any results not starting
+// with a given prefix.  Seeking to keys not beginning with the prefix
+// is invalid, and SeekToLast is not implemented (that would be
+// non-trivial), but otherwise this iterator will behave just like the
+// underlying iterator would if there happened to be no non-matching
+// keys in the dataset.
+
+#ifndef STORAGE_LEVELDB_DB_PREFIX_FILTER_ITERATOR_H_
+#define STORAGE_LEVELDB_DB_PREFIX_FILTER_ITERATOR_H_
+
+#include "leveldb/iterator.h"
+
+namespace leveldb {
+
+class PrefixFilterIterator : public Iterator {
+ private:
+  Iterator* iter_;
+  const Slice &prefix_;
+  const SliceTransform *prefix_extractor_;
+  Status status_;
+
+ public:
+  PrefixFilterIterator(Iterator* iter,
+                       const Slice &prefix,
+                       const SliceTransform* prefix_extractor)
+                             : iter_(iter), prefix_(prefix),
+                               prefix_extractor_(prefix_extractor),
+                               status_(Status::OK()) {
+    if (prefix_extractor == nullptr) {
+      status_ = Status::InvalidArgument("A prefix filter may not be used "
+                                        "unless a function is also defined "
+                                        "for extracting prefixes");
+    } else if (!prefix_extractor_->InRange(prefix)) {
+      status_ = Status::InvalidArgument("Must provide a slice for prefix which"
+                                        "is a prefix for some key");
+    }
+  }
+  ~PrefixFilterIterator() {
+    delete iter_;
+  }
+  Slice key() const { return iter_->key(); }
+  Slice value() const { return iter_->value(); }
+  Status status() const {
+    if (!status_.ok()) {
+      return status_;
+    }
+    return iter_->status();
+  }
+  void Next() { iter_->Next(); }
+  void Prev() { iter_->Prev(); }
+  void Seek(const Slice& k) {
+    if (prefix_extractor_->Transform(k) == prefix_) {
+      iter_->Seek(k);
+    } else {
+      status_ = Status::InvalidArgument("Seek must begin with target prefix");
+    }
+  }
+  void SeekToFirst() {
+    Seek(prefix_);
+  }
+  void SeekToLast() {
+    status_ = Status::NotSupported("SeekToLast is incompatible with prefixes");
+  }
+  bool Valid() const {
+    return (status_.ok() && iter_->Valid() &&
+            prefix_extractor_->Transform(iter_->key()) == prefix_);
+  }
+};
+
+}  // namespace leveldb
+
+#endif

db/repair.cc

@@ -50,23 +50,15 @@ class Repairer {
         icmp_(options.comparator),
         ipolicy_(options.filter_policy),
         options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
-        owns_info_log_(options_.info_log != options.info_log),
-        owns_cache_(options_.block_cache != options.block_cache),
         next_file_number_(1) {
     // TableCache can be small since we expect each table to be opened once.
-    table_cache_ = new TableCache(dbname_, &options_, 10);
+    table_cache_ = new TableCache(dbname_, &options_, storage_options_, 10);
     edit_ = new VersionEdit(options.num_levels);
   }
 
   ~Repairer() {
     delete table_cache_;
     delete edit_;
-    if (owns_info_log_) {
-      delete options_.info_log;
-    }
-    if (owns_cache_) {
-      delete options_.block_cache;
-    }
   }
 
   Status Run() {
@@ -96,6 +88,7 @@ class Repairer {
  private:
   struct TableInfo {
     FileMetaData meta;
+    SequenceNumber min_sequence;
     SequenceNumber max_sequence;
   };
 
@@ -104,7 +97,6 @@ class Repairer {
   InternalKeyComparator const icmp_;
   InternalFilterPolicy const ipolicy_;
   Options const options_;
-  bool owns_info_log_;
   bool owns_cache_;
   TableCache* table_cache_;
   VersionEdit* edit_;
@@ -114,6 +106,7 @@ class Repairer {
   std::vector<uint64_t> logs_;
   std::vector<TableInfo> tables_;
   uint64_t next_file_number_;
+  const EnvOptions storage_options_;
 
   Status FindFiles() {
     std::vector<std::string> filenames;
@@ -164,7 +157,7 @@ class Repairer {
   Status ConvertLogToTable(uint64_t log) {
     struct LogReporter : public log::Reader::Reporter {
       Env* env;
-      Logger* info_log;
+      std::shared_ptr<Logger> info_log;
       uint64_t lognum;
       virtual void Corruption(size_t bytes, const Status& s) {
         // We print error messages for corruption, but continue repairing.
@@ -177,8 +170,8 @@ class Repairer {
 
     // Open the log file
     std::string logname = LogFileName(dbname_, log);
-    SequentialFile* lfile;
-    Status status = env_->NewSequentialFile(logname, &lfile);
+    unique_ptr<SequentialFile> lfile;
+    Status status = env_->NewSequentialFile(logname, &lfile, storage_options_);
     if (!status.ok()) {
       return status;
     }
@@ -192,14 +185,15 @@ class Repairer {
     // corruptions cause entire commits to be skipped instead of
     // propagating bad information (like overly large sequence
     // numbers).
-    log::Reader reader(lfile, &reporter, false/*do not checksum*/,
+    log::Reader reader(std::move(lfile), &reporter, false/*do not checksum*/,
                        0/*initial_offset*/);
 
     // Read all the records and add to a memtable
     std::string scratch;
     Slice record;
     WriteBatch batch;
-    MemTable* mem = new MemTable(icmp_, options_.num_levels);
+    MemTable* mem = new MemTable(icmp_, options_.memtable_factory,
+      options_.num_levels);
     mem->Ref();
     int counter = 0;
     while (reader.ReadRecord(&record, &scratch)) {
@@ -219,17 +213,18 @@ class Repairer {
         status = Status::OK();  // Keep going with rest of file
       }
     }
-    delete lfile;
 
     // Do not record a version edit for this conversion to a Table
     // since ExtractMetaData() will also generate edits.
     FileMetaData meta;
     meta.number = next_file_number_++;
     Iterator* iter = mem->NewIterator();
-    status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta);
+    status = BuildTable(dbname_, env_, options_, storage_options_,
+                        table_cache_, iter, &meta,
+                        icmp_.user_comparator(), 0, 0);
     delete iter;
     mem->Unref();
-    mem = NULL;
+    mem = nullptr;
     if (status.ok()) {
       if (meta.file_size > 0) {
         table_numbers_.push_back(meta.number);
@@ -267,9 +262,10 @@ class Repairer {
     Status status = env_->GetFileSize(fname, &t->meta.file_size);
     if (status.ok()) {
       Iterator* iter = table_cache_->NewIterator(
-          ReadOptions(), t->meta.number, t->meta.file_size);
+          ReadOptions(), storage_options_, t->meta.number, t->meta.file_size);
       bool empty = true;
       ParsedInternalKey parsed;
+      t->min_sequence = 0;
       t->max_sequence = 0;
       for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
         Slice key = iter->key();
@@ -286,6 +282,9 @@ class Repairer {
           t->meta.smallest.DecodeFrom(key);
         }
         t->meta.largest.DecodeFrom(key);
+        if (parsed.sequence < t->min_sequence) {
+          t->min_sequence = parsed.sequence;
+        }
         if (parsed.sequence > t->max_sequence) {
           t->max_sequence = parsed.sequence;
         }
@@ -304,8 +303,8 @@ class Repairer {
 
   Status WriteDescriptor() {
     std::string tmp = TempFileName(dbname_, 1);
-    WritableFile* file;
-    Status status = env_->NewWritableFile(tmp, &file);
+    unique_ptr<WritableFile> file;
+    Status status = env_->NewWritableFile(tmp, &file, storage_options_);
     if (!status.ok()) {
       return status;
     }
@@ -326,21 +325,17 @@ class Repairer {
       // TODO(opt): separate out into multiple levels
       const TableInfo& t = tables_[i];
       edit_->AddFile(0, t.meta.number, t.meta.file_size,
-                    t.meta.smallest, t.meta.largest);
+                    t.meta.smallest, t.meta.largest,
+                    t.min_sequence, t.max_sequence);
     }
 
     //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
     {
-      log::Writer log(file);
+      log::Writer log(std::move(file));
       std::string record;
       edit_->EncodeTo(&record);
       status = log.AddRecord(record);
     }
-    if (status.ok()) {
-      status = file->Close();
-    }
-    delete file;
-    file = NULL;
 
     if (!status.ok()) {
       env_->DeleteFile(tmp);
@@ -368,14 +363,14 @@ class Repairer {
     //    dir/lost/foo
     const char* slash = strrchr(fname.c_str(), '/');
     std::string new_dir;
-    if (slash != NULL) {
+    if (slash != nullptr) {
       new_dir.assign(fname.data(), slash - fname.data());
     }
     new_dir.append("/lost");
     env_->CreateDir(new_dir);  // Ignore error
     std::string new_file = new_dir;
     new_file.append("/");
-    new_file.append((slash == NULL) ? fname.c_str() : slash + 1);
+    new_file.append((slash == nullptr) ? fname.c_str() : slash + 1);
     Status s = env_->RenameFile(fname, new_file);
     Log(options_.info_log, "Archiving %s: %s\n",
         fname.c_str(), s.ToString().c_str());

db/skiplist.h

@@ -31,13 +31,10 @@
 #include <assert.h>
 #include <stdlib.h>
 #include "port/port.h"
-#include "util/arena.h"
 #include "util/random.h"
 
 namespace leveldb {
 
-class Arena;
-
 template<typename Key, class Comparator>
 class SkipList {
  private:
@@ -127,9 +124,9 @@ class SkipList {
   bool KeyIsAfterNode(const Key& key, Node* n) const;
 
   // Return the earliest node that comes at or after key.
-  // Return NULL if there is no such node.
+  // Return nullptr if there is no such node.
   //
-  // If prev is non-NULL, fills prev[level] with pointer to previous
+  // If prev is non-nullptr, fills prev[level] with pointer to previous
   // node at "level" for every level in [0..max_height_-1].
   Node* FindGreaterOrEqual(const Key& key, Node** prev) const;
 
@@ -194,12 +191,12 @@ SkipList<Key,Comparator>::NewNode(const Key& key, int height) {
 template<typename Key, class Comparator>
 inline SkipList<Key,Comparator>::Iterator::Iterator(const SkipList* list) {
   list_ = list;
-  node_ = NULL;
+  node_ = nullptr;
 }
 
 template<typename Key, class Comparator>
 inline bool SkipList<Key,Comparator>::Iterator::Valid() const {
-  return node_ != NULL;
+  return node_ != nullptr;
 }
 
 template<typename Key, class Comparator>
@@ -221,13 +218,13 @@ inline void SkipList<Key,Comparator>::Iterator::Prev() {
   assert(Valid());
   node_ = list_->FindLessThan(node_->key);
   if (node_ == list_->head_) {
-    node_ = NULL;
+    node_ = nullptr;
   }
 }
 
 template<typename Key, class Comparator>
 inline void SkipList<Key,Comparator>::Iterator::Seek(const Key& target) {
-  node_ = list_->FindGreaterOrEqual(target, NULL);
+  node_ = list_->FindGreaterOrEqual(target, nullptr);
 }
 
 template<typename Key, class Comparator>
@@ -239,7 +236,7 @@ template<typename Key, class Comparator>
 inline void SkipList<Key,Comparator>::Iterator::SeekToLast() {
   node_ = list_->FindLast();
   if (node_ == list_->head_) {
-    node_ = NULL;
+    node_ = nullptr;
   }
 }
 
@@ -258,8 +255,8 @@ int SkipList<Key,Comparator>::RandomHeight() {
 
 template<typename Key, class Comparator>
 bool SkipList<Key,Comparator>::KeyIsAfterNode(const Key& key, Node* n) const {
-  // NULL n is considered infinite
-  return (n != NULL) && (compare_(n->key, key) < 0);
+  // nullptr n is considered infinite
+  return (n != nullptr) && (compare_(n->key, key) < 0);
 }
 
 template<typename Key, class Comparator>
@@ -282,7 +279,7 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr
       // Keep searching in this list
       x = next;
     } else {
-      if (prev != NULL) prev[level] = x;
+      if (prev != nullptr) prev[level] = x;
       if (level == 0) {
         return next;
       } else {
@@ -301,7 +298,7 @@ SkipList<Key,Comparator>::FindLessThan(const Key& key) const {
   while (true) {
     assert(x == head_ || compare_(x->key, key) < 0);
     Node* next = x->Next(level);
-    if (next == NULL || compare_(next->key, key) >= 0) {
+    if (next == nullptr || compare_(next->key, key) >= 0) {
       if (level == 0) {
         return x;
       } else {
@@ -321,7 +318,7 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindLast()
   int level = GetMaxHeight() - 1;
   while (true) {
     Node* next = x->Next(level);
-    if (next == NULL) {
+    if (next == nullptr) {
       if (level == 0) {
         return x;
       } else {
@@ -342,7 +339,7 @@ SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena)
       max_height_(reinterpret_cast<void*>(1)),
       rnd_(0xdeadbeef) {
   for (int i = 0; i < kMaxHeight; i++) {
-    head_->SetNext(i, NULL);
+    head_->SetNext(i, nullptr);
     prev_[i] = head_;
   }
 }
@@ -354,7 +351,7 @@ void SkipList<Key,Comparator>::Insert(const Key& key) {
   Node* x = FindGreaterOrEqual(key, prev_);
 
   // Our data structure does not allow duplicate insertion
-  assert(x == NULL || !Equal(key, x->key));
+  assert(x == nullptr || !Equal(key, x->key));
 
   int height = RandomHeight();
   if (height > GetMaxHeight()) {
@@ -366,9 +363,9 @@ void SkipList<Key,Comparator>::Insert(const Key& key) {
     // It is ok to mutate max_height_ without any synchronization
     // with concurrent readers.  A concurrent reader that observes
     // the new value of max_height_ will see either the old value of
-    // new level pointers from head_ (NULL), or a new value set in
+    // new level pointers from head_ (nullptr), or a new value set in
     // the loop below.  In the former case the reader will
-    // immediately drop to the next level since NULL sorts after all
+    // immediately drop to the next level since nullptr sorts after all
     // keys.  In the latter case the reader will use the new node.
     max_height_.NoBarrier_Store(reinterpret_cast<void*>(height));
   }
@@ -385,8 +382,8 @@ void SkipList<Key,Comparator>::Insert(const Key& key) {
 
 template<typename Key, class Comparator>
 bool SkipList<Key,Comparator>::Contains(const Key& key) const {
-  Node* x = FindGreaterOrEqual(key, NULL);
-  if (x != NULL && Equal(key, x->key)) {
+  Node* x = FindGreaterOrEqual(key, nullptr);
+  if (x != nullptr && Equal(key, x->key)) {
     return true;
   } else {
     return false;

db/skiplist_test.cc

@@ -5,7 +5,7 @@
 #include "db/skiplist.h"
 #include <set>
 #include "leveldb/env.h"
-#include "util/arena.h"
+#include "util/arena_impl.h"
 #include "util/hash.h"
 #include "util/random.h"
 #include "util/testharness.h"
@@ -14,7 +14,7 @@ namespace leveldb {
 
 typedef uint64_t Key;
 
-struct Comparator {
+struct TestComparator {
   int operator()(const Key& a, const Key& b) const {
     if (a < b) {
       return -1;
@@ -29,12 +29,12 @@ struct Comparator {
 class SkipTest { };
 
 TEST(SkipTest, Empty) {
-  Arena arena;
-  Comparator cmp;
-  SkipList<Key, Comparator> list(cmp, &arena);
+  ArenaImpl arena_impl;
+  TestComparator cmp;
+  SkipList<Key, TestComparator> list(cmp, &arena_impl);
   ASSERT_TRUE(!list.Contains(10));
 
-  SkipList<Key, Comparator>::Iterator iter(&list);
+  SkipList<Key, TestComparator>::Iterator iter(&list);
   ASSERT_TRUE(!iter.Valid());
   iter.SeekToFirst();
   ASSERT_TRUE(!iter.Valid());
@@ -49,9 +49,9 @@ TEST(SkipTest, InsertAndLookup) {
   const int R = 5000;
   Random rnd(1000);
   std::set<Key> keys;
-  Arena arena;
-  Comparator cmp;
-  SkipList<Key, Comparator> list(cmp, &arena);
+  ArenaImpl arena_impl;
+  TestComparator cmp;
+  SkipList<Key, TestComparator> list(cmp, &arena_impl);
   for (int i = 0; i < N; i++) {
     Key key = rnd.Next() % R;
     if (keys.insert(key).second) {
@@ -69,7 +69,7 @@ TEST(SkipTest, InsertAndLookup) {
 
   // Simple iterator tests
   {
-    SkipList<Key, Comparator>::Iterator iter(&list);
+    SkipList<Key, TestComparator>::Iterator iter(&list);
     ASSERT_TRUE(!iter.Valid());
 
     iter.Seek(0);
@@ -87,7 +87,7 @@ TEST(SkipTest, InsertAndLookup) {
 
   // Forward iteration test
   for (int i = 0; i < R; i++) {
-    SkipList<Key, Comparator>::Iterator iter(&list);
+    SkipList<Key, TestComparator>::Iterator iter(&list);
     iter.Seek(i);
 
     // Compare against model iterator
@@ -107,7 +107,7 @@ TEST(SkipTest, InsertAndLookup) {
 
   // Backward iteration test
   {
-    SkipList<Key, Comparator>::Iterator iter(&list);
+    SkipList<Key, TestComparator>::Iterator iter(&list);
     iter.SeekToLast();
 
     // Compare against model iterator
@@ -204,14 +204,14 @@ class ConcurrentTest {
   // Current state of the test
   State current_;
 
-  Arena arena_;
+  ArenaImpl arena_impl_;
 
   // SkipList is not protected by mu_.  We just use a single writer
   // thread to modify it.
-  SkipList<Key, Comparator> list_;
+  SkipList<Key, TestComparator> list_;
 
  public:
-  ConcurrentTest() : list_(Comparator(), &arena_) { }
+  ConcurrentTest() : list_(TestComparator(), &arena_impl_) { }
 
   // REQUIRES: External synchronization
   void WriteStep(Random* rnd) {
@@ -230,7 +230,7 @@ class ConcurrentTest {
     }
 
     Key pos = RandomTarget(rnd);
-    SkipList<Key, Comparator>::Iterator iter(&list_);
+    SkipList<Key, TestComparator>::Iterator iter(&list_);
     iter.Seek(pos);
     while (true) {
       Key current;
@@ -308,7 +308,7 @@ class TestState {
 
   explicit TestState(int s)
       : seed_(s),
-        quit_flag_(NULL),
+        quit_flag_(nullptr),
         state_(STARTING),
         state_cv_(&mu_) {}
 
@@ -360,7 +360,7 @@ static void RunConcurrent(int run) {
     for (int i = 0; i < kSize; i++) {
       state.t_.WriteStep(&rnd);
     }
-    state.quit_flag_.Release_Store(&state);  // Any non-NULL arg will do
+    state.quit_flag_.Release_Store(&state);  // Any non-nullptr arg will do
     state.Wait(TestState::DONE);
   }
 }

db/skiplistrep.h

@@ -0,0 +1,102 @@
+#ifndef STORAGE_LEVELDB_DB_SKIPLISTREP_H_
+#define STORAGE_LEVELDB_DB_SKIPLISTREP_H_
+
+#include "leveldb/memtablerep.h"
+#include "db/memtable.h"
+#include "db/skiplist.h"
+
+namespace leveldb {
+
+class Arena;
+
+class SkipListRep : public MemTableRep {
+  SkipList<const char*, MemTableRep::KeyComparator&> skip_list_;
+public:
+  explicit SkipListRep(MemTableRep::KeyComparator& compare, Arena* arena)
+    : skip_list_(compare, arena) {
+}
+
+  // Insert key into the list.
+  // REQUIRES: nothing that compares equal to key is currently in the list.
+  virtual void Insert(const char* key) {
+    skip_list_.Insert(key);
+  }
+
+  // Returns true iff an entry that compares equal to key is in the list.
+  virtual bool Contains(const char* key) const {
+    return skip_list_.Contains(key);
+  }
+
+  virtual ~SkipListRep() { }
+
+  // Iteration over the contents of a skip list
+  class Iterator : public MemTableRep::Iterator {
+    SkipList<const char*, MemTableRep::KeyComparator&>::Iterator iter_;
+   public:
+    // Initialize an iterator over the specified list.
+    // The returned iterator is not valid.
+    explicit Iterator(
+      const SkipList<const char*, MemTableRep::KeyComparator&>* list
+    ) : iter_(list) { }
+
+    virtual ~Iterator() { }
+
+    // Returns true iff the iterator is positioned at a valid node.
+    virtual bool Valid() const {
+      return iter_.Valid();
+    }
+
+    // Returns the key at the current position.
+    // REQUIRES: Valid()
+    virtual const char* key() const {
+      return iter_.key();
+    }
+
+    // Advances to the next position.
+    // REQUIRES: Valid()
+    virtual void Next() {
+      iter_.Next();
+    }
+
+    // Advances to the previous position.
+    // REQUIRES: Valid()
+    virtual void Prev() {
+      iter_.Prev();
+    }
+
+    // Advance to the first entry with a key >= target
+    virtual void Seek(const char* target) {
+      iter_.Seek(target);
+    }
+
+    // Position at the first entry in list.
+    // Final state of iterator is Valid() iff list is not empty.
+    virtual void SeekToFirst() {
+      iter_.SeekToFirst();
+    }
+
+    // Position at the last entry in list.
+    // Final state of iterator is Valid() iff list is not empty.
+    virtual void SeekToLast() {
+      iter_.SeekToLast();
+    }
+  };
+
+  virtual std::shared_ptr<MemTableRep::Iterator> GetIterator() {
+    return std::shared_ptr<MemTableRep::Iterator>(
+      new SkipListRep::Iterator(&skip_list_)
+    );
+  }
+};
+
+class SkipListFactory : public MemTableRepFactory {
+public:
+  virtual std::shared_ptr<MemTableRep> CreateMemTableRep (
+    MemTableRep::KeyComparator& compare, Arena* arena) {
+      return std::shared_ptr<MemTableRep>(new SkipListRep(compare, arena));
+  }
+};
+
+}
+
+#endif // STORAGE_LEVELDB_DB_SKIPLISTREP_H_

db/snapshot.h

@@ -59,18 +59,22 @@ class SnapshotList {
 
   // retrieve all snapshot numbers. They are sorted in ascending order.
   void getAll(std::vector<SequenceNumber>& ret) {
-    SnapshotImpl* s = &list_;
-    SequenceNumber prev;
-    prev = 0;
     if (empty()) return;
+    SnapshotImpl* s = &list_;
     while (s->next_ != &list_) {
-      assert(prev <= s->next_->number_);
-      assert(prev = s->next_->number_); // assignment
       ret.push_back(s->next_->number_);
       s = s ->next_;
     }
   }
 
+  // get the sequence number of the most recent snapshot
+  const SequenceNumber GetNewest() {
+    if (empty()) {
+      return 0;
+    }
+    return newest()->number_;
+  }
+
  private:
   // Dummy head of doubly-linked list of snapshots
   SnapshotImpl list_;

db/table_cache.cc

@@ -5,26 +5,17 @@
 #include "db/table_cache.h"
 
 #include "db/filename.h"
-#include "db/db_statistics.h"
-#include "leveldb/env.h"
-#include "leveldb/table.h"
+
+#include "leveldb/statistics.h"
+#include "table/table.h"
 #include "util/coding.h"
+#include "util/stop_watch.h"
 
 namespace leveldb {
 
-struct TableAndFile {
-  RandomAccessFile* file;
-  Table* table;
-};
-
-static class DBStatistics* dbstatistics;
-
 static void DeleteEntry(const Slice& key, void* value) {
-  TableAndFile* tf = reinterpret_cast<TableAndFile*>(value);
-  delete tf->table;
-  delete tf->file;
-  dbstatistics ? dbstatistics->incNumFileCloses() : (void)0;
-  delete tf;
+  Table* table = reinterpret_cast<Table*>(value);
+  delete table;
 }
 
 static void UnrefEntry(void* arg1, void* arg2) {
@@ -35,75 +26,87 @@ static void UnrefEntry(void* arg1, void* arg2) {
 
 TableCache::TableCache(const std::string& dbname,
                        const Options* options,
+                       const EnvOptions& storage_options,
                        int entries)
     : env_(options->env),
       dbname_(dbname),
       options_(options),
-      cache_(NewLRUCache(entries, options->table_cache_numshardbits)) {
-  dbstatistics = (DBStatistics*)options->statistics;
-}
+      storage_options_(storage_options),
+      cache_(NewLRUCache(entries, options->table_cache_numshardbits)) {}
 
 TableCache::~TableCache() {
-  delete cache_;
 }
 
-Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
-                             Cache::Handle** handle, bool* tableIO) {
+Status TableCache::FindTable(const EnvOptions& toptions,
+                             uint64_t file_number, uint64_t file_size,
+                             Cache::Handle** handle, bool* table_io,
+                             const bool no_io) {
   Status s;
   char buf[sizeof(file_number)];
   EncodeFixed64(buf, file_number);
   Slice key(buf, sizeof(buf));
-  DBStatistics* stats = (DBStatistics*) options_->statistics;
   *handle = cache_->Lookup(key);
-  if (*handle == NULL) {
-    if (tableIO != NULL) {
-      *tableIO = true;    // we had to do IO from storage
+  if (*handle == nullptr) {
+    if (no_io) { // Dont do IO and return a not-found status
+      return Status::NotFound("Table not found in table_cache, no_io is set");
+    }
+    if (table_io != nullptr) {
+      *table_io = true;    // we had to do IO from storage
     }
     std::string fname = TableFileName(dbname_, file_number);
-    RandomAccessFile* file = NULL;
-    Table* table = NULL;
-    s = env_->NewRandomAccessFile(fname, &file);
-    stats ? stats->incNumFileOpens() : (void)0;
+    unique_ptr<RandomAccessFile> file;
+    unique_ptr<Table> table;
+    s = env_->NewRandomAccessFile(fname, &file, toptions);
+    RecordTick(options_->statistics, NO_FILE_OPENS);
     if (s.ok()) {
-      s = Table::Open(*options_, file, file_size, &table);
+      if (options_->advise_random_on_open) {
+        file->Hint(RandomAccessFile::RANDOM);
+      }
+      StopWatch sw(env_, options_->statistics, TABLE_OPEN_IO_MICROS);
+      s = Table::Open(*options_, toptions, std::move(file), file_size, &table);
     }
 
     if (!s.ok()) {
-      assert(table == NULL);
-      delete file;
-      stats ? stats->incNumFileErrors() : (void)0;
+      assert(table == nullptr);
+      RecordTick(options_->statistics, NO_FILE_ERRORS);
       // We do not cache error results so that if the error is transient,
       // or somebody repairs the file, we recover automatically.
     } else {
-      TableAndFile* tf = new TableAndFile;
-      tf->file = file;
-      tf->table = table;
-      *handle = cache_->Insert(key, tf, 1, &DeleteEntry);
+      assert(file.get() == nullptr);
+      *handle = cache_->Insert(key, table.release(), 1, &DeleteEntry);
     }
   }
   return s;
 }
 
 Iterator* TableCache::NewIterator(const ReadOptions& options,
+                                  const EnvOptions& toptions,
                                   uint64_t file_number,
                                   uint64_t file_size,
-                                  Table** tableptr) {
-  if (tableptr != NULL) {
-    *tableptr = NULL;
+                                  Table** tableptr,
+                                  bool for_compaction) {
+  if (tableptr != nullptr) {
+    *tableptr = nullptr;
   }
 
-  Cache::Handle* handle = NULL;
-  Status s = FindTable(file_number, file_size, &handle);
+  Cache::Handle* handle = nullptr;
+  Status s = FindTable(toptions, file_number, file_size, &handle);
   if (!s.ok()) {
     return NewErrorIterator(s);
   }
 
-  Table* table = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
+  Table* table =
+    reinterpret_cast<Table*>(cache_->Value(handle));
   Iterator* result = table->NewIterator(options);
-  result->RegisterCleanup(&UnrefEntry, cache_, handle);
-  if (tableptr != NULL) {
+  result->RegisterCleanup(&UnrefEntry, cache_.get(), handle);
+  if (tableptr != nullptr) {
     *tableptr = table;
   }
+
+  if (for_compaction) {
+    table->SetupForCompaction();
+  }
+
   return result;
 }
 
@@ -112,14 +115,22 @@ Status TableCache::Get(const ReadOptions& options,
                        uint64_t file_size,
                        const Slice& k,
                        void* arg,
-                       void (*saver)(void*, const Slice&, const Slice&, bool),
-                       bool* tableIO) {
-  Cache::Handle* handle = NULL;
-  Status s = FindTable(file_number, file_size, &handle, tableIO);
+                       bool (*saver)(void*, const Slice&, const Slice&, bool),
+                       bool* table_io,
+                       void (*mark_key_may_exist)(void*),
+                       const bool no_io) {
+  Cache::Handle* handle = nullptr;
+  Status s = FindTable(storage_options_, file_number, file_size,
+                       &handle, table_io, no_io);
   if (s.ok()) {
-    Table* t = reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table;
-    s = t->InternalGet(options, k, arg, saver);
+    Table* t =
+      reinterpret_cast<Table*>(cache_->Value(handle));
+    s = t->InternalGet(options, k, arg, saver, mark_key_may_exist, no_io);
     cache_->Release(handle);
+  } else if (no_io && s.IsNotFound()) {
+    // Couldnt find Table in cache but treat as kFound if no_io set
+    (*mark_key_may_exist)(arg);
+    return Status::OK();
   }
   return s;
 }

db/table_cache.h

@@ -10,9 +10,10 @@
 #include <string>
 #include <stdint.h>
 #include "db/dbformat.h"
+#include "leveldb/env.h"
 #include "leveldb/cache.h"
-#include "leveldb/table.h"
 #include "port/port.h"
+#include "table/table.h"
 
 namespace leveldb {
 
@@ -20,30 +21,36 @@ class Env;
 
 class TableCache {
  public:
-  TableCache(const std::string& dbname, const Options* options, int entries);
+  TableCache(const std::string& dbname, const Options* options,
+             const EnvOptions& storage_options, int entries);
   ~TableCache();
 
   // Return an iterator for the specified file number (the corresponding
   // file length must be exactly "file_size" bytes).  If "tableptr" is
-  // non-NULL, also sets "*tableptr" to point to the Table object
-  // underlying the returned iterator, or NULL if no Table object underlies
+  // non-nullptr, also sets "*tableptr" to point to the Table object
+  // underlying the returned iterator, or nullptr if no Table object underlies
   // the returned iterator.  The returned "*tableptr" object is owned by
   // the cache and should not be deleted, and is valid for as long as the
   // returned iterator is live.
   Iterator* NewIterator(const ReadOptions& options,
+                        const EnvOptions& toptions,
                         uint64_t file_number,
                         uint64_t file_size,
-                        Table** tableptr = NULL);
+                        Table** tableptr = nullptr,
+                        bool for_compaction = false);
 
   // If a seek to internal key "k" in specified file finds an entry,
-  // call (*handle_result)(arg, found_key, found_value).
+  // call (*handle_result)(arg, found_key, found_value) repeatedly until
+  // it returns false.
   Status Get(const ReadOptions& options,
              uint64_t file_number,
              uint64_t file_size,
              const Slice& k,
              void* arg,
-             void (*handle_result)(void*, const Slice&, const Slice&, bool),
-             bool* tableIO);
+             bool (*handle_result)(void*, const Slice&, const Slice&, bool),
+             bool* table_io,
+             void (*mark_key_may_exist)(void*) = nullptr,
+             const bool no_io = false);
 
   // Evict any entry for the specified file number
   void Evict(uint64_t file_number);
@@ -52,10 +59,12 @@ class TableCache {
   Env* const env_;
   const std::string dbname_;
   const Options* options_;
-  Cache* cache_;
+  const EnvOptions& storage_options_;
+  std::shared_ptr<Cache> cache_;
 
-  Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**,
-                   bool* tableIO = NULL);
+  Status FindTable(const EnvOptions& toptions, uint64_t file_number,
+                   uint64_t file_size, Cache::Handle**, bool* table_io=nullptr,
+                   const bool no_io = false);
 };
 
 }  // namespace leveldb

db/transaction_log_impl.cc

@@ -0,0 +1,171 @@
+#include "db/transaction_log_impl.h"
+#include "db/write_batch_internal.h"
+
+namespace leveldb {
+
+TransactionLogIteratorImpl::TransactionLogIteratorImpl(
+                           const std::string& dbname,
+                           const Options* options,
+                           const EnvOptions& soptions,
+                           const SequenceNumber seq,
+                           std::unique_ptr<VectorLogPtr> files,
+                           SequenceNumber const * const lastFlushedSequence) :
+    dbname_(dbname),
+    options_(options),
+    soptions_(soptions),
+    startingSequenceNumber_(seq),
+    files_(std::move(files)),
+    started_(false),
+    isValid_(false),
+    currentFileIndex_(0),
+    lastFlushedSequence_(lastFlushedSequence) {
+  assert(startingSequenceNumber_ <= *lastFlushedSequence_);
+  assert(files_.get() != nullptr);
+
+  reporter_.env = options_->env;
+  reporter_.info_log = options_->info_log.get();
+}
+
+Status TransactionLogIteratorImpl::OpenLogFile(
+  const LogFile* logFile,
+  unique_ptr<SequentialFile>* file) {
+  Env* env = options_->env;
+  if (logFile->Type() == kArchivedLogFile) {
+    std::string fname = ArchivedLogFileName(dbname_, logFile->LogNumber());
+    return env->NewSequentialFile(fname, file, soptions_);
+  } else {
+    std::string fname = LogFileName(dbname_, logFile->LogNumber());
+    Status status = env->NewSequentialFile(fname, file, soptions_);
+    if (!status.ok()) {
+      //  If cannot open file in DB directory.
+      //  Try the archive dir, as it could have moved in the meanwhile.
+      fname = ArchivedLogFileName(dbname_, logFile->LogNumber());
+      status = env->NewSequentialFile(fname, file, soptions_);
+      if (!status.ok()) {
+        return Status::IOError(" Requested file not present in the dir");
+      }
+    }
+    return status;
+  }
+}
+
+BatchResult TransactionLogIteratorImpl::GetBatch()  {
+  assert(isValid_);  //  cannot call in a non valid state.
+  BatchResult result;
+  result.sequence = currentSequence_;
+  result.writeBatchPtr = std::move(currentBatch_);
+  return result;
+}
+
+Status TransactionLogIteratorImpl::status() {
+  return currentStatus_;
+}
+
+bool TransactionLogIteratorImpl::Valid() {
+  return started_ && isValid_;
+}
+
+void TransactionLogIteratorImpl::Next() {
+  LogFile* currentLogFile = files_.get()->at(currentFileIndex_).get();
+
+//  First seek to the given seqNo. in the current file.
+  std::string scratch;
+  Slice record;
+  if (!started_) {
+    started_ = true;  // this piece only runs onced.
+    isValid_ = false;
+    if (startingSequenceNumber_ > *lastFlushedSequence_) {
+      currentStatus_ = Status::IOError("Looking for a sequence, "
+                                        "which is not flushed yet.");
+      return;
+    }
+    Status s = OpenLogReader(currentLogFile);
+    if (!s.ok()) {
+      currentStatus_ = s;
+      isValid_ = false;
+      return;
+    }
+    while (currentLogReader_->ReadRecord(&record, &scratch)) {
+      if (record.size() < 12) {
+        reporter_.Corruption(
+          record.size(), Status::Corruption("log record too small"));
+        continue;
+      }
+      UpdateCurrentWriteBatch(record);
+      if (currentSequence_ >= startingSequenceNumber_) {
+        assert(currentSequence_ <= *lastFlushedSequence_);
+        isValid_ = true;
+        break;
+      } else {
+        isValid_ = false;
+      }
+    }
+    if (isValid_) {
+      // Done for this iteration
+      return;
+    }
+  }
+  bool openNextFile = true;
+  while(openNextFile) {
+    assert(currentLogReader_);
+    if (currentSequence_ < *lastFlushedSequence_) {
+      if (currentLogReader_->IsEOF()) {
+        currentLogReader_->UnmarkEOF();
+      }
+      while (currentLogReader_->ReadRecord(&record, &scratch)) {
+        if (record.size() < 12) {
+          reporter_.Corruption(
+            record.size(), Status::Corruption("log record too small"));
+          continue;
+        } else {
+          UpdateCurrentWriteBatch(record);
+          openNextFile = false;
+          break;
+        }
+      }
+    }
+
+    if (openNextFile) {
+      if (currentFileIndex_ < files_.get()->size() - 1) {
+        ++currentFileIndex_;
+        Status status =OpenLogReader(files_.get()->at(currentFileIndex_).get());
+        if (!status.ok()) {
+          isValid_ = false;
+          currentStatus_ = status;
+          return;
+        }
+      } else {
+        isValid_ = false;
+        openNextFile = false;
+        if (currentSequence_ == *lastFlushedSequence_) {
+          currentStatus_ = Status::OK();
+        } else {
+          currentStatus_ = Status::IOError(" NO MORE DATA LEFT");
+        }
+      }
+    }
+  }
+}
+
+void TransactionLogIteratorImpl::UpdateCurrentWriteBatch(const Slice& record) {
+  WriteBatch* batch = new WriteBatch();
+  WriteBatchInternal::SetContents(batch, record);
+  currentSequence_ = WriteBatchInternal::Sequence(batch);
+  currentBatch_.reset(batch);
+  isValid_ = true;
+  currentStatus_ = Status::OK();
+}
+
+Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* logFile) {
+  unique_ptr<SequentialFile> file;
+  Status status = OpenLogFile(logFile, &file);
+  if (!status.ok()) {
+    return status;
+  }
+  assert(file);
+  currentLogReader_.reset(
+    new log::Reader(std::move(file), &reporter_, true, 0)
+  );
+  return Status::OK();
+}
+}  //  namespace leveldb

db/transaction_log_impl.h

@@ -0,0 +1,98 @@
+// Copyright 2008-present Facebook. All Rights Reserved.
+#ifndef STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
+#define STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
+
+#include <vector>
+
+#include "leveldb/env.h"
+#include "leveldb/options.h"
+#include "leveldb/types.h"
+#include "leveldb/transaction_log.h"
+#include "db/log_reader.h"
+#include "db/filename.h"
+
+namespace leveldb {
+
+struct LogReporter : public log::Reader::Reporter {
+  Env* env;
+  Logger* info_log;
+  virtual void Corruption(size_t bytes, const Status& s) {
+    Log(info_log, "dropping %zu bytes; %s", bytes, s.ToString().c_str());
+  }
+};
+
+class LogFileImpl : public LogFile {
+ public:
+  LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
+              uint64_t sizeBytes) :
+    logNumber_(logNum),
+    type_(logType),
+    startSequence_(startSeq),
+    sizeFileBytes_(sizeBytes) {
+  }
+
+  std::string Filename() const { return LogFileName("", logNumber_); }
+
+  uint64_t LogNumber() const { return logNumber_; }
+
+  WalFileType Type() const { return type_; }
+
+  SequenceNumber StartSequence() const { return startSequence_; }
+
+  uint64_t SizeFileBytes() const { return sizeFileBytes_; }
+
+  bool operator < (const LogFile& that) const {
+    return LogNumber() < that.LogNumber();
+  }
+
+ private:
+  uint64_t logNumber_;
+  WalFileType type_;
+  SequenceNumber startSequence_;
+  uint64_t sizeFileBytes_;
+
+};
+
+class TransactionLogIteratorImpl : public TransactionLogIterator {
+ public:
+  TransactionLogIteratorImpl(const std::string& dbname,
+                             const Options* options,
+                             const EnvOptions& soptions,
+                             const SequenceNumber seqNum,
+                             std::unique_ptr<VectorLogPtr> files,
+                             SequenceNumber const * const lastFlushedSequence);
+
+  virtual bool Valid();
+
+  virtual void Next();
+
+  virtual Status status();
+
+  virtual BatchResult GetBatch();
+
+ private:
+  const std::string& dbname_;
+  const Options* options_;
+  const EnvOptions& soptions_;
+  const SequenceNumber startingSequenceNumber_;
+  std::unique_ptr<VectorLogPtr> files_;
+  bool started_;
+  bool isValid_;  // not valid when it starts of.
+  Status currentStatus_;
+  size_t currentFileIndex_;
+  std::unique_ptr<WriteBatch> currentBatch_;
+  unique_ptr<log::Reader> currentLogReader_;
+  Status OpenLogFile(const LogFile* logFile, unique_ptr<SequentialFile>* file);
+  LogReporter reporter_;
+  SequenceNumber const * const lastFlushedSequence_;
+  // represents the sequence number being read currently.
+  SequenceNumber currentSequence_;
+
+  void UpdateCurrentWriteBatch(const Slice& record);
+  Status OpenLogReader(const LogFile* file);
+};
+
+
+
+}  //  namespace leveldb
+#endif  //  STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_

db/transaction_log_iterator_impl.cc

@@ -1,148 +0,0 @@
-#include "db/transaction_log_iterator_impl.h"
-#include "db/write_batch_internal.h"
-#include "db/filename.h"
-namespace leveldb {
-
-TransactionLogIteratorImpl::TransactionLogIteratorImpl(
-                                       const std::string& dbname,
-                                       const Options* options,
-                                       SequenceNumber& seq,
-                                       std::vector<LogFile>* files) :
-  dbname_(dbname),
-  options_(options),
-  sequenceNumber_(seq),
-  files_(files),
-  started_(false),
-  isValid_(true),
-  currentFileIndex_(0),
-  currentLogReader_(NULL) {
-  assert( files_ != NULL);
-}
-
-LogReporter
-TransactionLogIteratorImpl::NewLogReporter(const uint64_t logNumber) {
-  LogReporter reporter;
-  reporter.env = options_->env;
-  reporter.info_log = options_->info_log;
-  reporter.log_number = logNumber;
-  return reporter;
-}
-
-Status TransactionLogIteratorImpl::OpenLogFile(const LogFile& logFile,
-                                       SequentialFile** file) {
-  Env* env = options_->env;
-  if (logFile.type == kArchivedLogFile) {
-    std::string fname = ArchivedLogFileName(dbname_, logFile.logNumber);
-    return env->NewSequentialFile(fname, file);
-  } else {
-    std::string fname = LogFileName(dbname_, logFile.logNumber);
-    Status status = env->NewSequentialFile(fname, file);
-    if (!status.ok()) {
-      //  If cannot open file in DB directory.
-      //  Try the archive dir, as it could have moved in the meanwhile.
-      fname = ArchivedLogFileName(dbname_, logFile.logNumber);
-      status = env->NewSequentialFile(fname, file);
-      if (!status.ok()) {
-        //  TODO stringprintf
-        return Status::IOError(" Requested file not present in the dir");
-      }
-    }
-    return status;
-  }
-}
-
-void TransactionLogIteratorImpl::GetBatch(WriteBatch* batch,
-                                          SequenceNumber* seq)  {
-  assert(isValid_);  //  cannot call in a non valid state.
-  WriteBatchInternal::SetContents(batch, currentRecord_);
-  *seq = WriteBatchInternal::Sequence(batch);
-}
-
-Status TransactionLogIteratorImpl::status() {
-  return currentStatus_;
-}
-
-bool TransactionLogIteratorImpl::Valid() {
-  return started_ && isValid_;
-}
-
-void TransactionLogIteratorImpl::Next() {
-//  First seek to the given seqNo. in the current file.
-  LogFile currentLogFile = files_->at(currentFileIndex_);
-  LogReporter reporter = NewLogReporter(currentLogFile.logNumber);
-  std::string scratch;
-  Slice record;
-  if (!started_) {
-    SequentialFile* file = NULL;
-    Status status = OpenLogFile(currentLogFile, &file);
-    if (!status.ok()) {
-      isValid_ = false;
-      currentStatus_ = status;
-      return;
-    }
-    assert(file != NULL);
-    WriteBatch batch;
-    log::Reader* reader = new log::Reader(file, &reporter, true, 0);
-    assert(reader != NULL);
-    while (reader->ReadRecord(&record, &scratch)) {
-      if (record.size() < 12) {
-        reporter.Corruption(
-          record.size(), Status::Corruption("log record too small"));
-        continue;
-      }
-      WriteBatchInternal::SetContents(&batch, record);
-      SequenceNumber currentNum = WriteBatchInternal::Sequence(&batch);
-      if (currentNum >= sequenceNumber_) {
-        isValid_ = true;
-        currentRecord_ = record;
-        currentLogReader_ = reader;
-        break;
-      }
-    }
-    if (!isValid_) {
-      //  TODO read the entire first file. and did not find the seq number.
-      //  Error out.
-      currentStatus_ =
-        Status::NotFound("Did not find the Seq no. in first file");
-    }
-    started_ = true;
-  } else {
-LOOK_NEXT_FILE:
-    assert(currentLogReader_ != NULL);
-    bool openNextFile = true;
-    while (currentLogReader_->ReadRecord(&record, &scratch)) {
-      if (record.size() < 12) {
-        reporter.Corruption(
-          record.size(), Status::Corruption("log record too small"));
-        continue;
-      } else {
-        currentRecord_ = record;
-        openNextFile = false;
-        break;
-      }
-    }
-
-    if (openNextFile) {
-      if (currentFileIndex_ < files_->size() - 1) {
-        ++currentFileIndex_;
-        delete currentLogReader_;
-        SequentialFile *file;
-        Status status = OpenLogFile(files_->at(currentFileIndex_), &file);
-        if (!status.ok()) {
-          isValid_ = false;
-          currentStatus_ = status;
-          return;
-        }
-        currentLogReader_ = new log::Reader(file, &reporter, true, 0);
-        goto LOOK_NEXT_FILE;
-      } else {
-        //  LOOKED AT FILES. WE ARE DONE HERE.
-        isValid_ = false;
-        currentStatus_ = Status::IOError(" NO MORE DATA LEFT");
-      }
-    }
-
-  }
-}
-
-}  //  namespace leveldb

db/transaction_log_iterator_impl.h

@@ -1,66 +0,0 @@
-// Copyright 2008-present Facebook. All Rights Reserved.
-#ifndef STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
-#define STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
-
-#include <vector>
-
-#include "leveldb/env.h"
-#include "leveldb/options.h"
-#include "leveldb/types.h"
-#include "leveldb/transaction_log_iterator.h"
-#include "db/log_file.h"
-#include "db/log_reader.h"
-
-namespace leveldb {
-
-struct LogReporter : public log::Reader::Reporter {
-  Env* env;
-  Logger* info_log;
-  uint64_t log_number;
-  virtual void Corruption(size_t bytes, const Status& s) {
-    Log(info_log, "%ld: dropping %d bytes; %s",
-        log_number, static_cast<int>(bytes), s.ToString().c_str());
-  }
-};
-
-class TransactionLogIteratorImpl : public TransactionLogIterator {
- public:
-  TransactionLogIteratorImpl(const std::string& dbname,
-                     const Options* options,
-                     SequenceNumber& seqNum,
-                     std::vector<LogFile>* files);
-  virtual ~TransactionLogIteratorImpl() {
-    //  TODO move to cc file.
-    if (currentLogReader_ != NULL) {
-      delete currentLogReader_;
-    }
-    delete files_;
-  }
-
-  virtual bool Valid();
-
-  virtual void Next();
-
-  virtual Status status();
-
-  virtual void GetBatch(WriteBatch* batch, SequenceNumber* seq);
-
- private:
-  const std::string& dbname_;
-  const Options* options_;
-  const uint64_t sequenceNumber_;
-  const std::vector<LogFile>* files_;
-  bool started_;
-  bool isValid_;  // not valid when it starts of.
-  Status currentStatus_;
-  size_t currentFileIndex_;
-  Slice currentRecord_;
-  log::Reader* currentLogReader_;
-  Status OpenLogFile(const LogFile& logFile, SequentialFile** file);
-  LogReporter NewLogReporter(uint64_t logNumber);
-};
-
-
-
-}  //  namespace leveldb
-#endif  //  STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_

db/version_edit.cc

@@ -20,7 +20,10 @@ enum Tag {
   kDeletedFile          = 6,
   kNewFile              = 7,
   // 8 was used for large value refs
-  kPrevLogNumber        = 9
+  kPrevLogNumber        = 9,
+
+  // these are new formats divergent from open source leveldb
+  kNewFile2             = 100  // store smallest & largest seqno
 };
 
 void VersionEdit::Clear() {
@@ -76,12 +79,14 @@ void VersionEdit::EncodeTo(std::string* dst) const {
 
   for (size_t i = 0; i < new_files_.size(); i++) {
     const FileMetaData& f = new_files_[i].second;
-    PutVarint32(dst, kNewFile);
+    PutVarint32(dst, kNewFile2);
     PutVarint32(dst, new_files_[i].first);  // level
     PutVarint64(dst, f.number);
     PutVarint64(dst, f.file_size);
     PutLengthPrefixedSlice(dst, f.smallest.Encode());
     PutLengthPrefixedSlice(dst, f.largest.Encode());
+    PutVarint64(dst, f.smallest_seqno);
+    PutVarint64(dst, f.largest_seqno);
   }
 }
 
@@ -95,13 +100,16 @@ static bool GetInternalKey(Slice* input, InternalKey* dst) {
   }
 }
 
-bool VersionEdit::GetLevel(Slice* input, int* level) {
+bool VersionEdit::GetLevel(Slice* input, int* level, const char** msg) {
   uint32_t v;
   if (GetVarint32(input, &v) &&
       (int)v < number_levels_) {
     *level = v;
     return true;
   } else {
+    if ((int)v >= number_levels_) {
+      *msg = "db already has more levels than options.num_levels";
+    }
     return false;
   }
 }
@@ -109,7 +117,7 @@ bool VersionEdit::GetLevel(Slice* input, int* level) {
 Status VersionEdit::DecodeFrom(const Slice& src) {
   Clear();
   Slice input = src;
-  const char* msg = NULL;
+  const char* msg = nullptr;
   uint32_t tag;
 
   // Temporary storage for parsing
@@ -119,7 +127,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
   Slice str;
   InternalKey key;
 
-  while (msg == NULL && GetVarint32(&input, &tag)) {
+  while (msg == nullptr && GetVarint32(&input, &tag)) {
     switch (tag) {
       case kComparator:
         if (GetLengthPrefixedSlice(&input, &str)) {
@@ -163,32 +171,54 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
         break;
 
       case kCompactPointer:
-        if (GetLevel(&input, &level) &&
+        if (GetLevel(&input, &level, &msg) &&
             GetInternalKey(&input, &key)) {
           compact_pointers_.push_back(std::make_pair(level, key));
         } else {
-          msg = "compaction pointer";
+          if (!msg) {
+            msg = "compaction pointer";
+          }
         }
         break;
 
       case kDeletedFile:
-        if (GetLevel(&input, &level) &&
+        if (GetLevel(&input, &level, &msg) &&
             GetVarint64(&input, &number)) {
           deleted_files_.insert(std::make_pair(level, number));
         } else {
-          msg = "deleted file";
+          if (!msg) {
+            msg = "deleted file";
+          }
         }
         break;
 
       case kNewFile:
-        if (GetLevel(&input, &level) &&
+        if (GetLevel(&input, &level, &msg) &&
             GetVarint64(&input, &f.number) &&
             GetVarint64(&input, &f.file_size) &&
             GetInternalKey(&input, &f.smallest) &&
             GetInternalKey(&input, &f.largest)) {
           new_files_.push_back(std::make_pair(level, f));
         } else {
-          msg = "new-file entry";
+          if (!msg) {
+            msg = "new-file entry";
+          }
+        }
+        break;
+
+      case kNewFile2:
+        if (GetLevel(&input, &level, &msg) &&
+            GetVarint64(&input, &f.number) &&
+            GetVarint64(&input, &f.file_size) &&
+            GetInternalKey(&input, &f.smallest) &&
+            GetInternalKey(&input, &f.largest) &&
+            GetVarint64(&input, &f.smallest_seqno) &&
+            GetVarint64(&input, &f.largest_seqno) ) {
+          new_files_.push_back(std::make_pair(level, f));
+        } else {
+          if (!msg) {
+            msg = "new-file2 entry";
+          }
         }
         break;
 
@@ -198,18 +228,18 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
     }
   }
 
-  if (msg == NULL && !input.empty()) {
+  if (msg == nullptr && !input.empty()) {
     msg = "invalid tag";
   }
 
   Status result;
-  if (msg != NULL) {
+  if (msg != nullptr) {
     result = Status::Corruption("VersionEdit", msg);
   }
   return result;
 }
 
-std::string VersionEdit::DebugString() const {
+std::string VersionEdit::DebugString(bool hex_key) const {
   std::string r;
   r.append("VersionEdit {");
   if (has_comparator_) {
@@ -236,7 +266,7 @@ std::string VersionEdit::DebugString() const {
     r.append("\n  CompactPointer: ");
     AppendNumberTo(&r, compact_pointers_[i].first);
     r.append(" ");
-    r.append(compact_pointers_[i].second.DebugString());
+    r.append(compact_pointers_[i].second.DebugString(hex_key));
   }
   for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
        iter != deleted_files_.end();
@@ -255,9 +285,9 @@ std::string VersionEdit::DebugString() const {
     r.append(" ");
     AppendNumberTo(&r, f.file_size);
     r.append(" ");
-    r.append(f.smallest.DebugString());
+    r.append(f.smallest.DebugString(hex_key));
     r.append(" .. ");
-    r.append(f.largest.DebugString());
+    r.append(f.largest.DebugString(hex_key));
   }
   r.append("\n}\n");
   return r;

db/version_edit.h

@@ -22,6 +22,8 @@ struct FileMetaData {
   InternalKey smallest;       // Smallest internal key served by table
   InternalKey largest;        // Largest internal key served by table
   bool being_compacted;       // Is this file undergoing compaction?
+  SequenceNumber smallest_seqno;// The smallest seqno in this file
+  SequenceNumber largest_seqno; // The largest seqno in this file
 
   FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0),
                    being_compacted(false) { }
@@ -67,12 +69,17 @@ class VersionEdit {
   void AddFile(int level, uint64_t file,
                uint64_t file_size,
                const InternalKey& smallest,
-               const InternalKey& largest) {
+               const InternalKey& largest,
+               const SequenceNumber& smallest_seqno,
+               const SequenceNumber& largest_seqno) {
     FileMetaData f;
     f.number = file;
     f.file_size = file_size;
     f.smallest = smallest;
     f.largest = largest;
+    f.smallest_seqno = smallest_seqno;
+    f.largest_seqno = largest_seqno;
+    assert(smallest_seqno <= largest_seqno);
     new_files_.push_back(std::make_pair(level, f));
   }
 
@@ -81,17 +88,22 @@ class VersionEdit {
     deleted_files_.insert(std::make_pair(level, file));
   }
 
+  // Number of edits
+  int NumEntries() {
+    return new_files_.size() + deleted_files_.size();
+  }
+
   void EncodeTo(std::string* dst) const;
   Status DecodeFrom(const Slice& src);
 
-  std::string DebugString() const;
+  std::string DebugString(bool hex_key = false) const;
 
  private:
   friend class VersionSet;
 
   typedef std::set< std::pair<int, uint64_t> > DeletedFileSet;
 
-  bool GetLevel(Slice* input, int* level);
+  bool GetLevel(Slice* input, int* level, const char** msg);
 
   int number_levels_;
   std::string comparator_;

db/version_edit_test.cc

@@ -27,7 +27,9 @@ TEST(VersionEditTest, EncodeDecode) {
     TestEncodeDecode(edit);
     edit.AddFile(3, kBig + 300 + i, kBig + 400 + i,
                  InternalKey("foo", kBig + 500 + i, kTypeValue),
-                 InternalKey("zoo", kBig + 600 + i, kTypeDeletion));
+                 InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
+                 kBig + 500 + i,
+                 kBig + 600 + i);
     edit.DeleteFile(4, kBig + 700 + i);
     edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue));
   }

db/version_set.h

@@ -16,6 +16,7 @@
 #define STORAGE_LEVELDB_DB_VERSION_SET_H_
 
 #include <map>
+#include <memory>
 #include <set>
 #include <vector>
 #include <deque>
@@ -46,8 +47,8 @@ extern int FindFile(const InternalKeyComparator& icmp,
 
 // Returns true iff some file in "files" overlaps the user key range
 // [*smallest,*largest].
-// smallest==NULL represents a key smaller than all keys in the DB.
-// largest==NULL represents a key largest than all keys in the DB.
+// smallest==nullptr represents a key smaller than all keys in the DB.
+// largest==nullptr represents a key largest than all keys in the DB.
 // REQUIRES: If disjoint_sorted_files, files[] contains disjoint ranges
 //           in sorted order.
 extern bool SomeFileOverlapsRange(
@@ -62,17 +63,21 @@ class Version {
   // Append to *iters a sequence of iterators that will
   // yield the contents of this Version when merged together.
   // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters);
+  void AddIterators(const ReadOptions&, const EnvOptions& soptions,
+                    std::vector<Iterator*>* iters);
 
   // Lookup the value for key.  If found, store it in *val and
   // return OK.  Else return a non-OK status.  Fills *stats.
+  // Uses *operands to store merge_operator operations to apply later
   // REQUIRES: lock is not held
   struct GetStats {
     FileMetaData* seek_file;
     int seek_file_level;
   };
-  Status Get(const ReadOptions&, const LookupKey& key, std::string* val,
-             GetStats* stats);
+  void Get(const ReadOptions&, const LookupKey& key, std::string* val,
+           Status* status, std::deque<std::string>* operands, GetStats* stats,
+           const Options& db_option, const bool no_io = false,
+           bool* value_found = nullptr);
 
   // Adds "stats" into the current state.  Returns true if a new
   // compaction may need to be triggered, false otherwise.
@@ -86,26 +91,26 @@ class Version {
 
   void GetOverlappingInputs(
       int level,
-      const InternalKey* begin,         // NULL means before all keys
-      const InternalKey* end,           // NULL means after all keys
+      const InternalKey* begin,         // nullptr means before all keys
+      const InternalKey* end,           // nullptr means after all keys
       std::vector<FileMetaData*>* inputs,
       int hint_index = -1,              // index of overlap file
-      int* file_index = NULL);          // return index of overlap file
+      int* file_index = nullptr);          // return index of overlap file
 
   void GetOverlappingInputsBinarySearch(
       int level,
-      const Slice& begin,         // NULL means before all keys
-      const Slice& end,           // NULL means after all keys
+      const Slice& begin,         // nullptr means before all keys
+      const Slice& end,           // nullptr means after all keys
       std::vector<FileMetaData*>* inputs,
       int hint_index,             // index of overlap file
       int* file_index);           // return index of overlap file
 
   void ExtendOverlappingInputs(
       int level,
-      const Slice& begin,         // NULL means before all keys
-      const Slice& end,           // NULL means after all keys
+      const Slice& begin,         // nullptr means before all keys
+      const Slice& end,           // nullptr means after all keys
       std::vector<FileMetaData*>* inputs,
-      int index);                 // start extending from this index
+      unsigned int index);                 // start extending from this index
 
   // Returns true iff some file in the specified level overlaps
   // some part of [*smallest_user_key,*largest_user_key].
@@ -115,6 +120,14 @@ class Version {
                       const Slice* smallest_user_key,
                       const Slice* largest_user_key);
 
+  // Returns true iff the first or last file in inputs contains
+  // an overlapping user key to the file "just outside" of it (i.e.
+  // just after the last file, or just before the first file)
+  // REQUIRES: "*inputs" is a sorted list of non-overlapping files
+  bool HasOverlappingUserKey(const std::vector<FileMetaData*>* inputs,
+                             int level);
+
+
   // Return the level at which we should place a new memtable compaction
   // result that covers the range [smallest_user_key,largest_user_key].
   int PickLevelForMemTableOutput(const Slice& smallest_user_key,
@@ -133,9 +146,12 @@ class Version {
  private:
   friend class Compaction;
   friend class VersionSet;
+  friend class DBImpl;
 
   class LevelFileNumIterator;
-  Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const;
+  Iterator* NewConcatenatingIterator(const ReadOptions&,
+                                     const EnvOptions& soptions,
+                                     int level) const;
 
   VersionSet* vset_;            // VersionSet to which this Version belongs
   Version* next_;               // Next version in linked list
@@ -175,6 +191,7 @@ class Version {
   std::vector<double> compaction_score_;
   std::vector<int> compaction_level_;
   double max_compaction_score_; // max score in l1 to ln-1
+  int max_compaction_score_level_; // level on which max score occurs
 
   // The offset in the manifest file where this version is stored.
   uint64_t offset_manifest_file_;
@@ -202,6 +219,7 @@ class VersionSet {
  public:
   VersionSet(const std::string& dbname,
              const Options* options,
+             const EnvOptions& storage_options,
              TableCache* table_cache,
              const InternalKeyComparator*);
   ~VersionSet();
@@ -271,13 +289,13 @@ class VersionSet {
   int NumberLevels() const { return num_levels_; }
 
   // Pick level and inputs for a new compaction.
-  // Returns NULL if there is no compaction to be done.
+  // Returns nullptr if there is no compaction to be done.
   // Otherwise returns a pointer to a heap-allocated object that
   // describes the compaction.  Caller should delete the result.
   Compaction* PickCompaction();
 
   // Return a compaction object for compacting the range [begin,end] in
-  // the specified level.  Returns NULL if there is nothing in that
+  // the specified level.  Returns nullptr if there is nothing in that
   // level that overlaps the specified range.  Caller should delete
   // the result.
   Compaction* CompactRange(
@@ -305,7 +323,7 @@ class VersionSet {
   }
   // Returns true iff some level needs a compaction.
   bool NeedsCompaction() const {
-    return ((current_->file_to_compact_ != NULL) ||
+    return ((current_->file_to_compact_ != nullptr) ||
             NeedsSizeCompaction());
   }
 
@@ -314,9 +332,13 @@ class VersionSet {
     return current_->max_compaction_score_;
   }
 
+  // See field declaration
+  int MaxCompactionScoreLevel() const {
+    return current_->max_compaction_score_level_;
+  }
+
   // Add all files listed in any live version to *live.
-  // May also mutate some internal state.
-  void AddLiveFiles(std::set<uint64_t>* live);
+  void AddLiveFiles(std::vector<uint64_t>* live_list);
 
   // Add all files listed in the current version to *live.
   void AddLiveFilesCurrentVersion(std::set<uint64_t>* live);
@@ -330,6 +352,9 @@ class VersionSet {
   struct LevelSummaryStorage {
     char buffer[100];
   };
+  struct FileSummaryStorage {
+    char buffer[1000];
+  };
   const char* LevelSummary(LevelSummaryStorage* scratch) const;
 
   // printf contents (for debugging)
@@ -340,13 +365,22 @@ class VersionSet {
   // of files per level.  Uses *scratch as backing store.
   const char* LevelDataSizeSummary(LevelSummaryStorage* scratch) const;
 
+  // Return a human-readable short (single-line) summary of files
+  // in a specified level.  Uses *scratch as backing store.
+  const char* LevelFileSummary(FileSummaryStorage* scratch, int level) const;
+
   // Return the size of the current manifest file
   const uint64_t ManifestFileSize() { return current_->offset_manifest_file_; }
 
   // For the specfied level, pick a compaction.
-  // Returns NULL if there is no compaction to be done.
+  // Returns nullptr if there is no compaction to be done.
+  // If level is 0 and there is already a compaction on that level, this
+  // function will return nullptr.
   Compaction* PickCompactionBySize(int level, double score);
 
+  // Pick files to compact in Universal mode
+  Compaction* PickCompactionUniversal(int level, double score);
+
   // Free up the files that were participated in a compaction
   void ReleaseCompactionFiles(Compaction* c, Status status);
 
@@ -366,6 +400,11 @@ class VersionSet {
   // record results in files_by_size_. The largest files are listed first.
   void UpdateFilesBySize(Version *v);
 
+  // Get the max file size in a given level.
+  uint64_t MaxFileSizeForLevel(int level);
+
+  double MaxBytesForLevel(int level);
+
  private:
   class Builder;
   struct ManifestWriter;
@@ -375,7 +414,7 @@ class VersionSet {
 
   void Init(int num_levels);
 
-  void Finalize(Version* v);
+  void Finalize(Version* v, std::vector<uint64_t>&);
 
   void GetRange(const std::vector<FileMetaData*>& inputs,
                 InternalKey* smallest,
@@ -386,6 +425,8 @@ class VersionSet {
                  InternalKey* smallest,
                  InternalKey* largest);
 
+  void ExpandWhileOverlapping(Compaction* c);
+
   void SetupOtherInputs(Compaction* c);
 
   // Save current contents to *log
@@ -393,13 +434,11 @@ class VersionSet {
 
   void AppendVersion(Version* v);
 
-  double MaxBytesForLevel(int level);
+  bool ManifestContains(const std::string& record) const;
 
-  uint64_t MaxFileSizeForLevel(int level);
+  uint64_t ExpandedCompactionByteSizeLimit(int level);
 
-  int64_t ExpandedCompactionByteSizeLimit(int level);
-
-  int64_t MaxGrandParentOverlapBytes(int level);
+  uint64_t MaxGrandParentOverlapBytes(int level);
 
   Env* const env_;
   const std::string dbname_;
@@ -415,8 +454,7 @@ class VersionSet {
   int num_levels_;
 
   // Opened lazily
-  WritableFile* descriptor_file_;
-  log::Writer* descriptor_log_;
+  unique_ptr<log::Writer> descriptor_log_;
   Version dummy_versions_;  // Head of circular doubly-linked list of versions.
   Version* current_;        // == dummy_versions_.prev_
 
@@ -439,13 +477,24 @@ class VersionSet {
   // Queue of writers to the manifest file
   std::deque<ManifestWriter*> manifest_writers_;
 
+  // Store the manifest file size when it is checked.
+  // Save us the cost of checking file size twice in LogAndApply
+  uint64_t last_observed_manifest_size_;
+
+  // storage options for all reads and writes except compactions
+  const EnvOptions& storage_options_;
+
+  // storage options used for compactions. This is a copy of
+  // storage_options_ but with readaheads set to readahead_compactions_.
+  const EnvOptions storage_options_compactions_;
+
   // No copying allowed
   VersionSet(const VersionSet&);
   void operator=(const VersionSet&);
 
   // Return the total amount of data that is undergoing
-  // compactions at this level
-  uint64_t SizeBeingCompacted(int level);
+  // compactions per level
+  void SizeBeingCompacted(std::vector<uint64_t>&);
 
   // Returns true if any one of the parent files are being compacted
   bool ParentRangeInCompaction(const InternalKey* smallest,
@@ -464,9 +513,12 @@ class Compaction {
   ~Compaction();
 
   // Return the level that is being compacted.  Inputs from "level"
-  // and "level+1" will be merged to produce a set of "level+1" files.
+  // will be merged.
   int level() const { return level_; }
 
+  // Outputs will go to this level
+  int output_level() const { return out_level_; }
+
   // Return the object that holds the edits to the descriptor done
   // by this compaction.
   VersionEdit* edit() { return edit_; }
@@ -505,17 +557,21 @@ class Compaction {
   // Return the score that was used to pick this compaction run.
   double score() const { return score_; }
 
+  // Is this compaction creating a file in the bottom most level?
+  bool BottomMostLevel() { return bottommost_level_; }
+
  private:
   friend class Version;
   friend class VersionSet;
 
-  explicit Compaction(int level, uint64_t target_file_size,
+  explicit Compaction(int level, int out_level, uint64_t target_file_size,
     uint64_t max_grandparent_overlap_bytes, int number_levels,
     bool seek_compaction = false);
 
   int level_;
+  int out_level_; // levels to which output files are stored
   uint64_t max_output_file_size_;
-  int64_t maxGrandParentOverlapBytes_;
+  uint64_t maxGrandParentOverlapBytes_;
   Version* input_version_;
   VersionEdit* edit_;
   int number_levels_;
@@ -530,23 +586,27 @@ class Compaction {
   std::vector<FileMetaData*> grandparents_;
   size_t grandparent_index_;  // Index in grandparent_starts_
   bool seen_key_;             // Some output key has been seen
-  int64_t overlapped_bytes_;  // Bytes of overlap between current output
+  uint64_t overlapped_bytes_;  // Bytes of overlap between current output
                               // and grandparent files
   int base_index_;   // index of the file in files_[level_]
   int parent_index_; // index of some file with same range in files_[level_+1]
   double score_;     // score that was used to pick this compaction.
 
-  // State for implementing IsBaseLevelForKey
+  // Is this compaction creating a file in the bottom most level?
+  bool bottommost_level_;
 
   // level_ptrs_ holds indices into input_version_->levels_: our state
   // is that we are positioned at one of the file ranges for each
   // higher level than the ones involved in this compaction (i.e. for
   // all L >= level_ + 2).
-  size_t* level_ptrs_;
+  std::vector<size_t> level_ptrs_;
 
   // mark (or clear) all files that are being compacted
   void MarkFilesBeingCompacted(bool);
 
+  // Initialize whether compaction producing files at the bottommost level
+  void SetupBottomMostLevel(bool isManual);
+
   // In case of compaction error, reset the nextIndex that is used
   // to pick up the next file to be compacted from files_by_size_
   void ResetNextCompactionIndex();

db/version_set_reduce_num_levels.cc

@@ -67,7 +67,8 @@ Status VersionSet::ReduceNumberOfLevels(int new_levels, port::Mutex* mu) {
   num_levels_ = new_levels;
   compact_pointer_ = new std::string[new_levels];
   Init(new_levels);
-  st = LogAndApply(new VersionEdit(new_levels), mu, true);
+  VersionEdit ve(new_levels);
+  st = LogAndApply(&ve , mu, true);
   return st;
 }
 

db/version_set_test.cc

@@ -40,20 +40,20 @@ class FindFileTest {
 
   bool Overlaps(const char* smallest, const char* largest) {
     InternalKeyComparator cmp(BytewiseComparator());
-    Slice s(smallest != NULL ? smallest : "");
-    Slice l(largest != NULL ? largest : "");
+    Slice s(smallest != nullptr ? smallest : "");
+    Slice l(largest != nullptr ? largest : "");
     return SomeFileOverlapsRange(cmp, disjoint_sorted_files_, files_,
-                                 (smallest != NULL ? &s : NULL),
-                                 (largest != NULL ? &l : NULL));
+                                 (smallest != nullptr ? &s : nullptr),
+                                 (largest != nullptr ? &l : nullptr));
   }
 };
 
 TEST(FindFileTest, Empty) {
   ASSERT_EQ(0, Find("foo"));
   ASSERT_TRUE(! Overlaps("a", "z"));
-  ASSERT_TRUE(! Overlaps(NULL, "z"));
-  ASSERT_TRUE(! Overlaps("a", NULL));
-  ASSERT_TRUE(! Overlaps(NULL, NULL));
+  ASSERT_TRUE(! Overlaps(nullptr, "z"));
+  ASSERT_TRUE(! Overlaps("a", nullptr));
+  ASSERT_TRUE(! Overlaps(nullptr, nullptr));
 }
 
 TEST(FindFileTest, Single) {
@@ -78,12 +78,12 @@ TEST(FindFileTest, Single) {
   ASSERT_TRUE(Overlaps("q", "q"));
   ASSERT_TRUE(Overlaps("q", "q1"));
 
-  ASSERT_TRUE(! Overlaps(NULL, "j"));
-  ASSERT_TRUE(! Overlaps("r", NULL));
-  ASSERT_TRUE(Overlaps(NULL, "p"));
-  ASSERT_TRUE(Overlaps(NULL, "p1"));
-  ASSERT_TRUE(Overlaps("q", NULL));
-  ASSERT_TRUE(Overlaps(NULL, NULL));
+  ASSERT_TRUE(! Overlaps(nullptr, "j"));
+  ASSERT_TRUE(! Overlaps("r", nullptr));
+  ASSERT_TRUE(Overlaps(nullptr, "p"));
+  ASSERT_TRUE(Overlaps(nullptr, "p1"));
+  ASSERT_TRUE(Overlaps("q", nullptr));
+  ASSERT_TRUE(Overlaps(nullptr, nullptr));
 }
 
 
@@ -130,19 +130,19 @@ TEST(FindFileTest, MultipleNullBoundaries) {
   Add("200", "250");
   Add("300", "350");
   Add("400", "450");
-  ASSERT_TRUE(! Overlaps(NULL, "149"));
-  ASSERT_TRUE(! Overlaps("451", NULL));
-  ASSERT_TRUE(Overlaps(NULL, NULL));
-  ASSERT_TRUE(Overlaps(NULL, "150"));
-  ASSERT_TRUE(Overlaps(NULL, "199"));
-  ASSERT_TRUE(Overlaps(NULL, "200"));
-  ASSERT_TRUE(Overlaps(NULL, "201"));
-  ASSERT_TRUE(Overlaps(NULL, "400"));
-  ASSERT_TRUE(Overlaps(NULL, "800"));
-  ASSERT_TRUE(Overlaps("100", NULL));
-  ASSERT_TRUE(Overlaps("200", NULL));
-  ASSERT_TRUE(Overlaps("449", NULL));
-  ASSERT_TRUE(Overlaps("450", NULL));
+  ASSERT_TRUE(! Overlaps(nullptr, "149"));
+  ASSERT_TRUE(! Overlaps("451", nullptr));
+  ASSERT_TRUE(Overlaps(nullptr, nullptr));
+  ASSERT_TRUE(Overlaps(nullptr, "150"));
+  ASSERT_TRUE(Overlaps(nullptr, "199"));
+  ASSERT_TRUE(Overlaps(nullptr, "200"));
+  ASSERT_TRUE(Overlaps(nullptr, "201"));
+  ASSERT_TRUE(Overlaps(nullptr, "400"));
+  ASSERT_TRUE(Overlaps(nullptr, "800"));
+  ASSERT_TRUE(Overlaps("100", nullptr));
+  ASSERT_TRUE(Overlaps("200", nullptr));
+  ASSERT_TRUE(Overlaps("449", nullptr));
+  ASSERT_TRUE(Overlaps("450", nullptr));
 }
 
 TEST(FindFileTest, OverlapSequenceChecks) {

db/write_batch.cc

@@ -7,7 +7,8 @@
 //    count: fixed32
 //    data: record[count]
 // record :=
-//    kTypeValue varstring varstring         |
+//    kTypeValue varstring varstring
+//    kTypeMerge varstring varstring
 //    kTypeDeletion varstring
 // varstring :=
 //    len: varint32
@@ -15,11 +16,15 @@
 
 #include "leveldb/write_batch.h"
 
-#include "leveldb/db.h"
+#include "leveldb/options.h"
+#include "leveldb/statistics.h"
 #include "db/dbformat.h"
+#include "db/db_impl.h"
 #include "db/memtable.h"
+#include "db/snapshot.h"
 #include "db/write_batch_internal.h"
 #include "util/coding.h"
+#include <stdexcept>
 
 namespace leveldb {
 
@@ -34,11 +39,24 @@ WriteBatch::~WriteBatch() { }
 
 WriteBatch::Handler::~Handler() { }
 
+void WriteBatch::Handler::Merge(const Slice& key, const Slice& value) {
+  throw std::runtime_error("Handler::Merge not implemented!");
+}
+
+void WriteBatch::Handler::LogData(const Slice& blob) {
+  // If the user has not specified something to do with blobs, then we ignore
+  // them.
+}
+
 void WriteBatch::Clear() {
   rep_.clear();
   rep_.resize(kHeader);
 }
 
+int WriteBatch::Count() const {
+  return WriteBatchInternal::Count(this);
+}
+
 Status WriteBatch::Iterate(Handler* handler) const {
   Slice input(rep_);
   if (input.size() < kHeader) {
@@ -46,10 +64,9 @@ Status WriteBatch::Iterate(Handler* handler) const {
   }
 
   input.remove_prefix(kHeader);
-  Slice key, value;
+  Slice key, value, blob;
   int found = 0;
   while (!input.empty()) {
-    found++;
     char tag = input[0];
     input.remove_prefix(1);
     switch (tag) {
@@ -57,6 +74,7 @@ Status WriteBatch::Iterate(Handler* handler) const {
         if (GetLengthPrefixedSlice(&input, &key) &&
             GetLengthPrefixedSlice(&input, &value)) {
           handler->Put(key, value);
+          found++;
         } else {
           return Status::Corruption("bad WriteBatch Put");
         }
@@ -64,10 +82,27 @@ Status WriteBatch::Iterate(Handler* handler) const {
       case kTypeDeletion:
         if (GetLengthPrefixedSlice(&input, &key)) {
           handler->Delete(key);
+          found++;
         } else {
           return Status::Corruption("bad WriteBatch Delete");
         }
         break;
+      case kTypeMerge:
+        if (GetLengthPrefixedSlice(&input, &key) &&
+            GetLengthPrefixedSlice(&input, &value)) {
+          handler->Merge(key, value);
+          found++;
+        } else {
+          return Status::Corruption("bad WriteBatch Merge");
+        }
+        break;
+      case kTypeLogData:
+        if (GetLengthPrefixedSlice(&input, &blob)) {
+          handler->LogData(blob);
+        } else {
+          return Status::Corruption("bad WriteBatch Blob");
+        }
+        break;
       default:
         return Status::Corruption("unknown WriteBatch tag");
     }
@@ -108,28 +143,72 @@ void WriteBatch::Delete(const Slice& key) {
   PutLengthPrefixedSlice(&rep_, key);
 }
 
+void WriteBatch::Merge(const Slice& key, const Slice& value) {
+  WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
+  rep_.push_back(static_cast<char>(kTypeMerge));
+  PutLengthPrefixedSlice(&rep_, key);
+  PutLengthPrefixedSlice(&rep_, value);
+}
+
+void WriteBatch::PutLogData(const Slice& blob) {
+  rep_.push_back(static_cast<char>(kTypeLogData));
+  PutLengthPrefixedSlice(&rep_, blob);
+}
+
 namespace {
 class MemTableInserter : public WriteBatch::Handler {
  public:
   SequenceNumber sequence_;
   MemTable* mem_;
+  const Options* options_;
+  DBImpl* db_;
+  const bool filter_deletes_;
+
+  MemTableInserter(SequenceNumber sequence, MemTable* mem, const Options* opts,
+                   DB* db, const bool filter_deletes)
+    : sequence_(sequence),
+      mem_(mem),
+      options_(opts),
+      db_(reinterpret_cast<DBImpl*>(db)),
+      filter_deletes_(filter_deletes) {
+    assert(mem_);
+    if (filter_deletes_) {
+      assert(options_);
+      assert(db_);
+    }
+  }
 
   virtual void Put(const Slice& key, const Slice& value) {
     mem_->Add(sequence_, kTypeValue, key, value);
     sequence_++;
   }
+  virtual void Merge(const Slice& key, const Slice& value) {
+    mem_->Add(sequence_, kTypeMerge, key, value);
+    sequence_++;
+  }
   virtual void Delete(const Slice& key) {
+    if (filter_deletes_) {
+      SnapshotImpl read_from_snapshot;
+      read_from_snapshot.number_ = sequence_;
+      ReadOptions ropts;
+      ropts.snapshot = &read_from_snapshot;
+      std::string value;
+      if (!db_->KeyMayExist(ropts, key, &value)) {
+        RecordTick(options_->statistics, NUMBER_FILTERED_DELETES);
+        return;
+      }
+    }
     mem_->Add(sequence_, kTypeDeletion, key, Slice());
     sequence_++;
   }
 };
 }  // namespace
 
-Status WriteBatchInternal::InsertInto(const WriteBatch* b,
-                                      MemTable* memtable) {
-  MemTableInserter inserter;
-  inserter.sequence_ = WriteBatchInternal::Sequence(b);
-  inserter.mem_ = memtable;
+Status WriteBatchInternal::InsertInto(const WriteBatch* b, MemTable* mem,
+                                      const Options* opts, DB* db,
+                                      const bool filter_deletes) {
+  MemTableInserter inserter(WriteBatchInternal::Sequence(b), mem, opts, db,
+                            filter_deletes);
   return b->Iterate(&inserter);
 }
 

db/write_batch_internal.h

@@ -7,6 +7,8 @@
 
 #include "leveldb/types.h"
 #include "leveldb/write_batch.h"
+#include "leveldb/db.h"
+#include "leveldb/options.h"
 
 namespace leveldb {
 
@@ -39,7 +41,12 @@ class WriteBatchInternal {
 
   static void SetContents(WriteBatch* batch, const Slice& contents);
 
-  static Status InsertInto(const WriteBatch* batch, MemTable* memtable);
+  // Inserts batch entries into memtable
+  // Drops deletes in batch if filter_del is set to true and
+  // db->KeyMayExist returns false
+  static Status InsertInto(const WriteBatch* batch, MemTable* memtable,
+                           const Options* opts = nullptr, DB* db = nullptr,
+                           const bool filter_del = false);
 
   static void Append(WriteBatch* dst, const WriteBatch* src);
 };

db/write_batch_test.cc

@@ -4,6 +4,8 @@
 
 #include "leveldb/db.h"
 
+#include <memory>
+#include "db/skiplistrep.h"
 #include "db/memtable.h"
 #include "db/write_batch_internal.h"
 #include "leveldb/env.h"
@@ -14,7 +16,8 @@ namespace leveldb {
 
 static std::string PrintContents(WriteBatch* b) {
   InternalKeyComparator cmp(BytewiseComparator());
-  MemTable* mem = new MemTable(cmp);
+  auto factory = std::make_shared<SkipListFactory>();
+  MemTable* mem = new MemTable(cmp, factory);
   mem->Ref();
   std::string state;
   Status s = WriteBatchInternal::InsertInto(b, mem);
@@ -33,19 +36,30 @@ static std::string PrintContents(WriteBatch* b) {
         state.append(")");
         count++;
         break;
+      case kTypeMerge:
+        state.append("Merge(");
+        state.append(ikey.user_key.ToString());
+        state.append(", ");
+        state.append(iter->value().ToString());
+        state.append(")");
+        count++;
+        break;
       case kTypeDeletion:
         state.append("Delete(");
         state.append(ikey.user_key.ToString());
         state.append(")");
         count++;
         break;
+      case kTypeLogData:
+        assert(false);
+        break;
     }
     state.append("@");
     state.append(NumberToString(ikey.sequence));
   }
   delete iter;
   if (!s.ok()) {
-    state.append("ParseError()");
+    state.append(s.ToString());
   } else if (count != WriteBatchInternal::Count(b)) {
     state.append("CountMismatch()");
   }
@@ -59,6 +73,7 @@ TEST(WriteBatchTest, Empty) {
   WriteBatch batch;
   ASSERT_EQ("", PrintContents(&batch));
   ASSERT_EQ(0, WriteBatchInternal::Count(&batch));
+  ASSERT_EQ(0, batch.Count());
 }
 
 TEST(WriteBatchTest, Multiple) {
@@ -73,6 +88,7 @@ TEST(WriteBatchTest, Multiple) {
             "Delete(box)@101"
             "Put(foo, bar)@100",
             PrintContents(&batch));
+  ASSERT_EQ(3, batch.Count());
 }
 
 TEST(WriteBatchTest, Corruption) {
@@ -84,7 +100,7 @@ TEST(WriteBatchTest, Corruption) {
   WriteBatchInternal::SetContents(&batch,
                                   Slice(contents.data(),contents.size()-1));
   ASSERT_EQ("Put(foo, bar)@200"
-            "ParseError()",
+            "Corruption: bad WriteBatch Delete",
             PrintContents(&batch));
 }
 
@@ -95,16 +111,19 @@ TEST(WriteBatchTest, Append) {
   WriteBatchInternal::Append(&b1, &b2);
   ASSERT_EQ("",
             PrintContents(&b1));
+  ASSERT_EQ(0, b1.Count());
   b2.Put("a", "va");
   WriteBatchInternal::Append(&b1, &b2);
   ASSERT_EQ("Put(a, va)@200",
             PrintContents(&b1));
+  ASSERT_EQ(1, b1.Count());
   b2.Clear();
   b2.Put("b", "vb");
   WriteBatchInternal::Append(&b1, &b2);
   ASSERT_EQ("Put(a, va)@200"
             "Put(b, vb)@201",
             PrintContents(&b1));
+  ASSERT_EQ(2, b1.Count());
   b2.Delete("foo");
   WriteBatchInternal::Append(&b1, &b2);
   ASSERT_EQ("Put(a, va)@200"
@@ -112,6 +131,51 @@ TEST(WriteBatchTest, Append) {
             "Put(b, vb)@201"
             "Delete(foo)@203",
             PrintContents(&b1));
+  ASSERT_EQ(4, b1.Count());
+}
+
+TEST(WriteBatchTest, Blob) {
+  WriteBatch batch;
+  batch.Put(Slice("k1"), Slice("v1"));
+  batch.Put(Slice("k2"), Slice("v2"));
+  batch.Put(Slice("k3"), Slice("v3"));
+  batch.PutLogData(Slice("blob1"));
+  batch.Delete(Slice("k2"));
+  batch.PutLogData(Slice("blob2"));
+  batch.Merge(Slice("foo"), Slice("bar"));
+  ASSERT_EQ(5, batch.Count());
+  ASSERT_EQ("Merge(foo, bar)@4"
+            "Put(k1, v1)@0"
+            "Delete(k2)@3"
+            "Put(k2, v2)@1"
+            "Put(k3, v3)@2",
+            PrintContents(&batch));
+
+  struct Handler : public WriteBatch::Handler {
+    std::string seen;
+    virtual void Put(const Slice& key, const Slice& value) {
+      seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
+    }
+    virtual void Merge(const Slice& key, const Slice& value) {
+      seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
+    }
+    virtual void LogData(const Slice& blob) {
+      seen += "LogData(" + blob.ToString() + ")";
+    }
+    virtual void Delete(const Slice& key) {
+      seen += "Delete(" + key.ToString() + ")";
+    }
+  } handler;
+  batch.Iterate(&handler);
+  ASSERT_EQ(
+            "Put(k1, v1)"
+            "Put(k2, v2)"
+            "Put(k3, v3)"
+            "LogData(blob1)"
+            "Delete(k2)"
+            "LogData(blob2)"
+            "Merge(foo, bar)",
+            handler.seen);
 }
 
 }  // namespace leveldb

hdfs/env_hdfs.h

@@ -235,13 +235,20 @@ class HdfsEnv : public Env {
   }
 
   virtual Status NewSequentialFile(const std::string& fname,
-                                   SequentialFile** result);
+                                   unique_ptr<SequentialFile>* result,
+                                   const EnvOptions& options);
 
   virtual Status NewRandomAccessFile(const std::string& fname,
-                                     RandomAccessFile** result){ return notsup;}
+                                     unique_ptr<RandomAccessFile>* result,
+                                     const EnvOptions& options) {
+    return notsup;
+  }
 
   virtual Status NewWritableFile(const std::string& fname,
-                                 WritableFile** result){return notsup;}
+                                 unique_ptr<WritableFile>* result,
+                                 const EnvOptions& options) {
+    return notsup;
+  }
 
   virtual bool FileExists(const std::string& fname){return false;}
 
@@ -269,7 +276,8 @@ class HdfsEnv : public Env {
 
   virtual Status UnlockFile(FileLock* lock){return notsup;}
 
-  virtual Status NewLogger(const std::string& fname, Logger** result){return notsup;}
+  virtual Status NewLogger(const std::string& fname,
+                           shared_ptr<Logger>* result){return notsup;}
 
   virtual void Schedule( void (*function)(void* arg), void* arg) {}
 

helpers/memenv/memenv.cc

@@ -233,31 +233,34 @@ class InMemoryEnv : public EnvWrapper {
 
   // Partial implementation of the Env interface.
   virtual Status NewSequentialFile(const std::string& fname,
-                                   SequentialFile** result) {
+                                   unique_ptr<SequentialFile>* result,
+                                   const EnvOptions& soptions) {
     MutexLock lock(&mutex_);
     if (file_map_.find(fname) == file_map_.end()) {
       *result = NULL;
       return Status::IOError(fname, "File not found");
     }
 
-    *result = new SequentialFileImpl(file_map_[fname]);
+    result->reset(new SequentialFileImpl(file_map_[fname]));
     return Status::OK();
   }
 
   virtual Status NewRandomAccessFile(const std::string& fname,
-                                     RandomAccessFile** result) {
+                                     unique_ptr<RandomAccessFile>* result,
+                                     const EnvOptions& soptions) {
     MutexLock lock(&mutex_);
     if (file_map_.find(fname) == file_map_.end()) {
       *result = NULL;
       return Status::IOError(fname, "File not found");
     }
 
-    *result = new RandomAccessFileImpl(file_map_[fname]);
+    result->reset(new RandomAccessFileImpl(file_map_[fname]));
     return Status::OK();
   }
 
   virtual Status NewWritableFile(const std::string& fname,
-                                 WritableFile** result) {
+                                 unique_ptr<WritableFile>* result,
+                                 const EnvOptions& soptions) {
     MutexLock lock(&mutex_);
     if (file_map_.find(fname) != file_map_.end()) {
       DeleteFileInternal(fname);
@@ -267,7 +270,7 @@ class InMemoryEnv : public EnvWrapper {
     file->Ref();
     file_map_[fname] = file;
 
-    *result = new WritableFileImpl(file);
+    result->reset(new WritableFileImpl(file));
     return Status::OK();
   }
 

helpers/memenv/memenv_test.cc

@@ -8,6 +8,7 @@
 #include "leveldb/db.h"
 #include "leveldb/env.h"
 #include "util/testharness.h"
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -16,6 +17,7 @@ namespace leveldb {
 class MemEnvTest {
  public:
   Env* env_;
+  const EnvOptions soptions_;
 
   MemEnvTest()
       : env_(NewMemEnv(Env::Default())) {
@@ -27,7 +29,7 @@ class MemEnvTest {
 
 TEST(MemEnvTest, Basics) {
   uint64_t file_size;
-  WritableFile* writable_file;
+  unique_ptr<WritableFile> writable_file;
   std::vector<std::string> children;
 
   ASSERT_OK(env_->CreateDir("/dir"));
@@ -39,8 +41,8 @@ TEST(MemEnvTest, Basics) {
   ASSERT_EQ(0U, children.size());
 
   // Create a file.
-  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
-  delete writable_file;
+  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_));
+  writable_file.reset();
 
   // Check that the file exists.
   ASSERT_TRUE(env_->FileExists("/dir/f"));
@@ -51,9 +53,9 @@ TEST(MemEnvTest, Basics) {
   ASSERT_EQ("f", children[0]);
 
   // Write to the file.
-  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
+  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_));
   ASSERT_OK(writable_file->Append("abc"));
-  delete writable_file;
+  writable_file.reset();
 
   // Check for expected size.
   ASSERT_OK(env_->GetFileSize("/dir/f", &file_size));
@@ -68,11 +70,13 @@ TEST(MemEnvTest, Basics) {
   ASSERT_EQ(3U, file_size);
 
   // Check that opening non-existent file fails.
-  SequentialFile* seq_file;
-  RandomAccessFile* rand_file;
-  ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file).ok());
+  unique_ptr<SequentialFile> seq_file;
+  unique_ptr<RandomAccessFile> rand_file;
+  ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file,
+                                       soptions_).ok());
   ASSERT_TRUE(!seq_file);
-  ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file).ok());
+  ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file,
+                                         soptions_).ok());
   ASSERT_TRUE(!rand_file);
 
   // Check that deleting works.
@@ -85,21 +89,21 @@ TEST(MemEnvTest, Basics) {
 }
 
 TEST(MemEnvTest, ReadWrite) {
-  WritableFile* writable_file;
-  SequentialFile* seq_file;
-  RandomAccessFile* rand_file;
+  unique_ptr<WritableFile> writable_file;
+  unique_ptr<SequentialFile> seq_file;
+  unique_ptr<RandomAccessFile> rand_file;
   Slice result;
   char scratch[100];
 
   ASSERT_OK(env_->CreateDir("/dir"));
 
-  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
+  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_));
   ASSERT_OK(writable_file->Append("hello "));
   ASSERT_OK(writable_file->Append("world"));
-  delete writable_file;
+  writable_file.reset();
 
   // Read sequentially.
-  ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
+  ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file, soptions_));
   ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
   ASSERT_EQ(0, result.compare("hello"));
   ASSERT_OK(seq_file->Skip(1));
@@ -110,10 +114,9 @@ TEST(MemEnvTest, ReadWrite) {
   ASSERT_OK(seq_file->Skip(100)); // Try to skip past end of file.
   ASSERT_OK(seq_file->Read(1000, &result, scratch));
   ASSERT_EQ(0U, result.size());
-  delete seq_file;
 
   // Random reads.
-  ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file));
+  ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file, soptions_));
   ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
   ASSERT_EQ(0, result.compare("world"));
   ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
@@ -123,7 +126,6 @@ TEST(MemEnvTest, ReadWrite) {
 
   // Too high offset.
   ASSERT_TRUE(!rand_file->Read(1000, 5, &result, scratch).ok());
-  delete rand_file;
 }
 
 TEST(MemEnvTest, Locks) {
@@ -139,14 +141,14 @@ TEST(MemEnvTest, Misc) {
   ASSERT_OK(env_->GetTestDirectory(&test_dir));
   ASSERT_TRUE(!test_dir.empty());
 
-  WritableFile* writable_file;
-  ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file));
+  unique_ptr<WritableFile> writable_file;
+  ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file, soptions_));
 
   // These are no-ops, but we test they return success.
   ASSERT_OK(writable_file->Sync());
   ASSERT_OK(writable_file->Flush());
   ASSERT_OK(writable_file->Close());
-  delete writable_file;
+  writable_file.reset();
 }
 
 TEST(MemEnvTest, LargeWrite) {
@@ -158,15 +160,15 @@ TEST(MemEnvTest, LargeWrite) {
     write_data.append(1, static_cast<char>(i));
   }
 
-  WritableFile* writable_file;
-  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
+  unique_ptr<WritableFile> writable_file;
+  ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_));
   ASSERT_OK(writable_file->Append("foo"));
   ASSERT_OK(writable_file->Append(write_data));
-  delete writable_file;
+  writable_file.reset();
 
-  SequentialFile* seq_file;
+  unique_ptr<SequentialFile> seq_file;
   Slice result;
-  ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
+  ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file, soptions_));
   ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
   ASSERT_EQ(0, result.compare("foo"));
 
@@ -178,7 +180,6 @@ TEST(MemEnvTest, LargeWrite) {
     read += result.size();
   }
   ASSERT_TRUE(write_data == read_data);
-  delete seq_file;
   delete [] scratch;
 }
 

include/leveldb/arena.h

@@ -0,0 +1,38 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// Arena class defines memory allocation methods. It's used by memtable and
+// skiplist.
+
+#ifndef STORAGE_LEVELDB_INCLUDE_ARENA_H_
+#define STORAGE_LEVELDB_INCLUDE_ARENA_H_
+
+namespace leveldb {
+
+class Arena {
+ public:
+  Arena() {};
+  virtual ~Arena() {};
+
+  // Return a pointer to a newly allocated memory block of "bytes" bytes.
+  virtual char* Allocate(size_t bytes) = 0;
+
+  // Allocate memory with the normal alignment guarantees provided by malloc.
+  virtual char* AllocateAligned(size_t bytes) = 0;
+
+  // Returns an estimate of the total memory used by arena.
+  virtual const size_t ApproximateMemoryUsage() = 0;
+
+  // Returns the total number of bytes in all blocks allocated so far.
+  virtual const size_t MemoryAllocatedBytes() = 0;
+
+ private:
+  // No copying allowed
+  Arena(const Arena&);
+  void operator=(const Arena&);
+};
+
+}  // namespace leveldb
+
+#endif  // STORAGE_LEVELDB_INCLUDE_ARENA_H_

include/leveldb/c.h

@@ -183,6 +183,10 @@ extern void leveldb_options_destroy(leveldb_options_t*);
 extern void leveldb_options_set_comparator(
     leveldb_options_t*,
     leveldb_comparator_t*);
+extern void leveldb_options_set_compression_per_level(
+  leveldb_options_t* opt,
+  int* level_values,
+  size_t num_levels);
 extern void leveldb_options_set_filter_policy(
     leveldb_options_t*,
     leveldb_filterpolicy_t*);
@@ -199,6 +203,8 @@ extern void leveldb_options_set_max_open_files(leveldb_options_t*, int);
 extern void leveldb_options_set_cache(leveldb_options_t*, leveldb_cache_t*);
 extern void leveldb_options_set_block_size(leveldb_options_t*, size_t);
 extern void leveldb_options_set_block_restart_interval(leveldb_options_t*, int);
+extern void leveldb_options_set_compression_options(
+    leveldb_options_t* opt, int w_bits, int level, int strategy);
 
 enum {
   leveldb_no_compression = 0,

include/leveldb/cache.h

@@ -18,17 +18,20 @@
 #ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_
 #define STORAGE_LEVELDB_INCLUDE_CACHE_H_
 
+#include <memory>
 #include <stdint.h>
 #include "leveldb/slice.h"
 
 namespace leveldb {
 
+using std::shared_ptr;
+
 class Cache;
 
 // Create a new cache with a fixed size capacity.  This implementation
 // of Cache uses a least-recently-used eviction policy.
-extern Cache* NewLRUCache(size_t capacity);
-extern Cache* NewLRUCache(size_t capacity, int numShardBits);
+extern shared_ptr<Cache> NewLRUCache(size_t capacity);
+extern shared_ptr<Cache> NewLRUCache(size_t capacity, int numShardBits);
 
 class Cache {
  public:
@@ -53,7 +56,7 @@ class Cache {
   virtual Handle* Insert(const Slice& key, void* value, size_t charge,
                          void (*deleter)(const Slice& key, void* value)) = 0;
 
-  // If the cache has no mapping for "key", returns NULL.
+  // If the cache has no mapping for "key", returns nullptr.
   //
   // Else return a handle that corresponds to the mapping.  The caller
   // must call this->Release(handle) when the returned mapping is no

include/leveldb/compaction_filter.h

@@ -0,0 +1,71 @@
+// Copyright (c) 2013 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef STORAGE_LEVELDB_INCLUDE_COMPACTION_FILTER_H_
+#define STORAGE_LEVELDB_INCLUDE_COMPACTION_FILTER_H_
+
+#include <string>
+
+namespace leveldb {
+
+class Slice;
+
+// CompactionFilter allows an application to modify/delete a key-value at
+// the time of compaction.
+
+class CompactionFilter {
+ public:
+  virtual ~CompactionFilter() {}
+
+
+  // The compaction process invokes this
+  // method for kv that is being compacted. A return value
+  // of false indicates that the kv should be preserved in the
+  // output of this compaction run and a return value of true
+  // indicates that this key-value should be removed from the
+  // output of the compaction.  The application can inspect
+  // the existing value of the key and make decision based on it.
+  //
+  // When the value is to be preserved, the application has the option
+  // to modify the existing_value and pass it back through new_value.
+  // value_changed needs to be set to true in this case.
+  virtual bool Filter(int level,
+                      const Slice& key,
+                      const Slice& existing_value,
+                      std::string* new_value,
+                      bool* value_changed) const = 0;
+
+  // Returns a name that identifies this compaction filter.
+  // The name will be printed to LOG file on start up for diagnosis.
+  virtual const char* Name() const = 0;
+};
+
+// Each compaction will create a new CompactionFilter allowing the
+// application to know about different campactions
+class CompactionFilterFactory {
+  public:
+    virtual ~CompactionFilterFactory() { };
+    virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter() = 0;
+
+    // Returns a name that identifies this compaction filter factory.
+    virtual const char* Name() const = 0;
+};
+
+// Default implementaion of CompactionFilterFactory which does not
+// return any filter
+class DefaultCompactionFilterFactory : public CompactionFilterFactory {
+  public:
+    virtual std::unique_ptr<CompactionFilter>
+    CreateCompactionFilter() override {
+      return std::unique_ptr<CompactionFilter>(nullptr);
+    }
+
+    virtual const char* Name() const override {
+      return "DefaultCompactionFilterFactory";
+    }
+};
+
+}  // namespace leveldb
+
+#endif  // STORAGE_LEVELDB_INCLUDE_COMPACTION_FILTER_H_

include/leveldb/db.h

@@ -7,17 +7,20 @@
 
 #include <stdint.h>
 #include <stdio.h>
+#include <memory>
 #include <vector>
 #include "leveldb/iterator.h"
 #include "leveldb/options.h"
 #include "leveldb/types.h"
-#include "leveldb/transaction_log_iterator.h"
+#include "leveldb/transaction_log.h"
 
 namespace leveldb {
 
+using std::unique_ptr;
+
 // Update Makefile if you change these
-static const int kMajorVersion = 1;
-static const int kMinorVersion = 5;
+static const int kMajorVersion = 2;
+static const int kMinorVersion = 0;
 
 struct Options;
 struct ReadOptions;
@@ -50,7 +53,7 @@ class DB {
   // Open the database with the specified "name".
   // Stores a pointer to a heap-allocated database in *dbptr and returns
   // OK on success.
-  // Stores NULL in *dbptr and returns a non-OK status on error.
+  // Stores nullptr in *dbptr and returns a non-OK status on error.
   // Caller should delete *dbptr when it is no longer needed.
   static Status Open(const Options& options,
                      const std::string& name,
@@ -80,6 +83,14 @@ class DB {
   // Note: consider setting options.sync = true.
   virtual Status Delete(const WriteOptions& options, const Slice& key) = 0;
 
+  // Merge the database entry for "key" with "value".  Returns OK on success,
+  // and a non-OK status on error. The semantics of this operation is
+  // determined by the user provided merge_operator when opening DB.
+  // Note: consider setting options.sync = true.
+  virtual Status Merge(const WriteOptions& options,
+                       const Slice& key,
+                       const Slice& value) = 0;
+
   // Apply the specified updates to the database.
   // Returns OK on success, non-OK on failure.
   // Note: consider setting options.sync = true.
@@ -93,7 +104,39 @@ class DB {
   //
   // May return some other Status on an error.
   virtual Status Get(const ReadOptions& options,
-                     const Slice& key, std::string* value) = 0;
+                     const Slice& key,
+                     std::string* value) = 0;
+
+  // If keys[i] does not exist in the database, then the i'th returned
+  // status will be one for which Status::IsNotFound() is true, and
+  // (*values)[i] will be set to some arbitrary value (often ""). Otherwise,
+  // the i'th returned status will have Status::ok() true, and (*values)[i]
+  // will store the value associated with keys[i].
+  //
+  // (*values) will always be resized to be the same size as (keys).
+  // Similarly, the number of returned statuses will be the number of keys.
+  // Note: keys will not be "de-duplicated". Duplicate keys will return
+  // duplicate values in order.
+  virtual std::vector<Status> MultiGet(const ReadOptions& options,
+                                       const std::vector<Slice>& keys,
+                                       std::vector<std::string>* values) = 0;
+
+  // If the key definitely does not exist in the database, then this method
+  // returns false, else true. If the caller wants to obtain value when the key
+  // is found in memory, a bool for 'value_found' must be passed. 'value_found'
+  // will be true on return if value has been set properly.
+  // This check is potentially lighter-weight than invoking DB::Get(). One way
+  // to make this lighter weight is to avoid doing any IOs.
+  // Default implementation here returns true and sets 'value_found' to false
+  virtual bool KeyMayExist(const ReadOptions& options,
+                           const Slice& key,
+                           std::string* value,
+                           bool* value_found = nullptr) {
+    if (value_found != nullptr) {
+      *value_found = false;
+    }
+    return true;
+  }
 
   // Return a heap-allocated iterator over the contents of the database.
   // The result of NewIterator() is initially invalid (caller must
@@ -146,11 +189,18 @@ class DB {
   // needed to access the data.  This operation should typically only
   // be invoked by users who understand the underlying implementation.
   //
-  // begin==NULL is treated as a key before all keys in the database.
-  // end==NULL is treated as a key after all keys in the database.
+  // begin==nullptr is treated as a key before all keys in the database.
+  // end==nullptr is treated as a key after all keys in the database.
   // Therefore the following call will compact the entire database:
-  //    db->CompactRange(NULL, NULL);
-  virtual void CompactRange(const Slice* begin, const Slice* end) = 0;
+  //    db->CompactRange(nullptr, nullptr);
+  // Note that after the entire database is compacted, all data are pushed
+  // down to the last level containing any data. If the total data size
+  // after compaction is reduced, that level might not be appropriate for
+  // hosting all the files. In this case, client could set reduce_level
+  // to true, to move the files back to the minimum level capable of holding
+  // the data set.
+  virtual void CompactRange(const Slice* begin, const Slice* end,
+                            bool reduce_level = false) = 0;
 
   // Number of levels used for this DB.
   virtual int NumberLevels() = 0;
@@ -182,7 +232,15 @@ class DB {
   virtual Status GetLiveFiles(std::vector<std::string>&,
                               uint64_t* manifest_file_size) = 0;
 
-  // The sequence number of the most recent transaction. 
+  // Retrieve the sorted list of all wal files with earliest file first
+  virtual Status GetSortedWalFiles(VectorLogPtr& files) = 0;
+
+  // Delete wal files in files. These can be either live or archived.
+  // Returns Status::OK if all files could be deleted, otherwise Status::IOError
+  // which contains information about files that could not be deleted.
+  virtual Status DeleteWalFiles(const VectorLogPtr& files) = 0;
+
+  // The sequence number of the most recent transaction.
   virtual SequenceNumber GetLatestSequenceNumber() = 0;
 
   // Return's an iterator for all writes since the sequence number
@@ -197,7 +255,7 @@ class DB {
   // cleared aggressively and the iterator might keep getting invalid before
   // an update is read.
   virtual Status GetUpdatesSince(SequenceNumber seq_number,
-                                 TransactionLogIterator** iter) = 0;
+                                 unique_ptr<TransactionLogIterator>* iter) = 0;
  private:
   // No copying allowed
   DB(const DB&);

include/leveldb/env.h

@@ -15,6 +15,7 @@
 
 #include <cstdarg>
 #include <string>
+#include <memory>
 #include <vector>
 #include <stdint.h>
 #include "leveldb/status.h"
@@ -27,6 +28,40 @@ class RandomAccessFile;
 class SequentialFile;
 class Slice;
 class WritableFile;
+class Options;
+
+using std::unique_ptr;
+using std::shared_ptr;
+
+
+// Options while opening a file to read/write
+struct EnvOptions {
+
+  // construct with default Options
+  EnvOptions();
+
+  // construct from Options
+  EnvOptions(const Options& options);
+
+  // If true, then allow caching of data in environment buffers
+  bool use_os_buffer;
+
+   // If true, then use mmap to read data
+  bool use_mmap_reads;
+
+   // If true, then use mmap to write data
+  bool use_mmap_writes;
+
+  // If true, set the FD_CLOEXEC on open fd.
+  bool set_fd_cloexec;
+
+  // Allows OS to incrementally sync files to disk while they are being
+  // written, in the background. Issue one request for every bytes_per_sync
+  // written. 0 turns it off.
+  // Default: 0
+  uint64_t bytes_per_sync;
+
+};
 
 class Env {
  public:
@@ -42,32 +77,37 @@ class Env {
 
   // Create a brand new sequentially-readable file with the specified name.
   // On success, stores a pointer to the new file in *result and returns OK.
-  // On failure stores NULL in *result and returns non-OK.  If the file does
+  // On failure stores nullptr in *result and returns non-OK.  If the file does
   // not exist, returns a non-OK status.
   //
   // The returned file will only be accessed by one thread at a time.
   virtual Status NewSequentialFile(const std::string& fname,
-                                   SequentialFile** result) = 0;
+                                   unique_ptr<SequentialFile>* result,
+                                   const EnvOptions& options)
+                                   = 0;
 
   // Create a brand new random access read-only file with the
   // specified name.  On success, stores a pointer to the new file in
-  // *result and returns OK.  On failure stores NULL in *result and
+  // *result and returns OK.  On failure stores nullptr in *result and
   // returns non-OK.  If the file does not exist, returns a non-OK
   // status.
   //
   // The returned file may be concurrently accessed by multiple threads.
   virtual Status NewRandomAccessFile(const std::string& fname,
-                                     RandomAccessFile** result) = 0;
+                                     unique_ptr<RandomAccessFile>* result,
+                                     const EnvOptions& options)
+                                     = 0;
 
   // Create an object that writes to a new file with the specified
   // name.  Deletes any existing file with the same name and creates a
   // new file.  On success, stores a pointer to the new file in
-  // *result and returns OK.  On failure stores NULL in *result and
+  // *result and returns OK.  On failure stores nullptr in *result and
   // returns non-OK.
   //
   // The returned file will only be accessed by one thread at a time.
   virtual Status NewWritableFile(const std::string& fname,
-                                 WritableFile** result) = 0;
+                                 unique_ptr<WritableFile>* result,
+                                 const EnvOptions& options) = 0;
 
   // Returns true iff the named file exists.
   virtual bool FileExists(const std::string& fname) = 0;
@@ -102,7 +142,7 @@ class Env {
                             const std::string& target) = 0;
 
   // Lock the specified file.  Used to prevent concurrent access to
-  // the same db by multiple processes.  On failure, stores NULL in
+  // the same db by multiple processes.  On failure, stores nullptr in
   // *lock and returns non-OK.
   //
   // On success, stores a pointer to the object that represents the
@@ -143,7 +183,8 @@ class Env {
   virtual Status GetTestDirectory(std::string* path) = 0;
 
   // Create and return a log file for storing informational messages.
-  virtual Status NewLogger(const std::string& fname, Logger** result) = 0;
+  virtual Status NewLogger(const std::string& fname,
+                           shared_ptr<Logger>* result) = 0;
 
   // Returns the number of micro-seconds since some fixed point in time. Only
   // useful for computing deltas of time.
@@ -218,6 +259,32 @@ class RandomAccessFile {
   // Safe for concurrent use by multiple threads.
   virtual Status Read(uint64_t offset, size_t n, Slice* result,
                       char* scratch) const = 0;
+
+  // Tries to get an unique ID for this file that will be the same each time
+  // the file is opened (and will stay the same while the file is open).
+  // Furthermore, it tries to make this ID at most "max_size" bytes. If such an
+  // ID can be created this function returns the length of the ID and places it
+  // in "id"; otherwise, this function returns 0, in which case "id"
+  // may not have been modified.
+  //
+  // This function guarantees, for IDs from a given environment, two unique ids
+  // cannot be made equal to eachother by adding arbitrary bytes to one of
+  // them. That is, no unique ID is the prefix of another.
+  //
+  // This function guarantees that the returned ID will not be interpretable as
+  // a single varint.
+  //
+  // Note: these IDs are only valid for the duration of the process.
+  virtual size_t GetUniqueId(char* id, size_t max_size) const {
+    return 0; // Default implementation to prevent issues with backwards
+              // compatibility.
+  };
+
+
+  enum AccessPattern { NORMAL, RANDOM, SEQUENTIAL, WILLNEED, DONTNEED };
+
+  virtual void Hint(AccessPattern pattern) {}
+
 };
 
 // A file abstraction for sequential writing.  The implementation
@@ -225,7 +292,8 @@ class RandomAccessFile {
 // at a time to the file.
 class WritableFile {
  public:
-  WritableFile() { }
+  WritableFile() : last_preallocated_block_(0), preallocation_block_size_ (0) {
+  }
   virtual ~WritableFile();
 
   virtual Status Append(const Slice& data) = 0;
@@ -250,7 +318,67 @@ class WritableFile {
     return 0;
   }
 
+  /*
+   * Get and set the default pre-allocation block size for writes to
+   * this file.  If non-zero, then Allocate will be used to extend the
+   * underlying storage of a file (generally via fallocate) if the Env
+   * instance supports it.
+   */
+  void SetPreallocationBlockSize(size_t size) {
+    preallocation_block_size_ = size;
+  }
+
+  virtual void GetPreallocationStatus(size_t* block_size,
+                                      size_t* last_allocated_block) {
+    *last_allocated_block = last_preallocated_block_;
+    *block_size = preallocation_block_size_;
+  }
+
+ protected:
+  // PrepareWrite performs any necessary preparation for a write
+  // before the write actually occurs.  This allows for pre-allocation
+  // of space on devices where it can result in less file
+  // fragmentation and/or less waste from over-zealous filesystem
+  // pre-allocation.
+  void PrepareWrite(size_t offset, size_t len) {
+    if (preallocation_block_size_ == 0) {
+      return;
+    }
+    // If this write would cross one or more preallocation blocks,
+    // determine what the last preallocation block necesessary to
+    // cover this write would be and Allocate to that point.
+    const auto block_size = preallocation_block_size_;
+    size_t new_last_preallocated_block =
+      (offset + len + block_size - 1) / block_size;
+    if (new_last_preallocated_block > last_preallocated_block_) {
+      size_t num_spanned_blocks =
+        new_last_preallocated_block - last_preallocated_block_;
+      Allocate(block_size * last_preallocated_block_,
+               block_size * num_spanned_blocks);
+      last_preallocated_block_ = new_last_preallocated_block;
+    }
+  }
+
+  /*
+   * Pre-allocate space for a file.
+   */
+  virtual Status Allocate(off_t offset, off_t len) {
+    return Status::OK();
+  }
+
+  // Sync a file range with disk.
+  // offset is the starting byte of the file range to be synchronized.
+  // nbytes specifies the length of the range to be synchronized.
+  // This asks the OS to initiate flushing the cached data to disk,
+  // without waiting for completion.
+  // Default implementation does nothing.
+  virtual Status RangeSync(off64_t offset, off64_t nbytes) {
+    return Status::OK();
+  }
+
  private:
+  size_t last_preallocated_block_;
+  size_t preallocation_block_size_;
   // No copying allowed
   WritableFile(const WritableFile&);
   void operator=(const WritableFile&);
@@ -287,7 +415,13 @@ class FileLock {
   void operator=(const FileLock&);
 };
 
-// Log the specified data to *info_log if info_log is non-NULL.
+// Log the specified data to *info_log if info_log is non-nullptr.
+extern void Log(const shared_ptr<Logger>& info_log, const char* format, ...)
+#   if defined(__GNUC__) || defined(__clang__)
+    __attribute__((__format__ (__printf__, 2, 3)))
+#   endif
+    ;
+
 extern void Log(Logger* info_log, const char* format, ...)
 #   if defined(__GNUC__) || defined(__clang__)
     __attribute__((__format__ (__printf__, 2, 3)))
@@ -315,14 +449,19 @@ class EnvWrapper : public Env {
   Env* target() const { return target_; }
 
   // The following text is boilerplate that forwards all methods to target()
-  Status NewSequentialFile(const std::string& f, SequentialFile** r) {
-    return target_->NewSequentialFile(f, r);
+  Status NewSequentialFile(const std::string& f,
+                           unique_ptr<SequentialFile>* r,
+                           const EnvOptions& options) {
+    return target_->NewSequentialFile(f, r, options);
   }
-  Status NewRandomAccessFile(const std::string& f, RandomAccessFile** r) {
-    return target_->NewRandomAccessFile(f, r);
+  Status NewRandomAccessFile(const std::string& f,
+                             unique_ptr<RandomAccessFile>* r,
+                             const EnvOptions& options) {
+    return target_->NewRandomAccessFile(f, r, options);
   }
-  Status NewWritableFile(const std::string& f, WritableFile** r) {
-    return target_->NewWritableFile(f, r);
+  Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
+                         const EnvOptions& options) {
+    return target_->NewWritableFile(f, r, options);
   }
   bool FileExists(const std::string& f) { return target_->FileExists(f); }
   Status GetChildren(const std::string& dir, std::vector<std::string>* r) {
@@ -359,7 +498,8 @@ class EnvWrapper : public Env {
   virtual Status GetTestDirectory(std::string* path) {
     return target_->GetTestDirectory(path);
   }
-  virtual Status NewLogger(const std::string& fname, Logger** result) {
+  virtual Status NewLogger(const std::string& fname,
+                           shared_ptr<Logger>* result) {
     return target_->NewLogger(fname, result);
   }
   uint64_t NowMicros() {

include/leveldb/ldb_tool.h

@@ -0,0 +1,14 @@
+// Copyright 2008-present Facebook. All Rights Reserved.
+#ifndef STORAGE_LEVELDB_INCLUDE_LDB_TOOL_H
+#define STORAGE_LEVELDB_INCLUDE_LDB_TOOL_H
+#include "leveldb/options.h"
+
+namespace leveldb {
+
+class LDBTool {
+ public:
+  void Run(int argc, char** argv, Options = Options());
+};
+
+} // namespace leveldb
+#endif // STORAGE_LEVELDB_INCLUDE_LDB_TOOL_H

include/leveldb/memtablerep.h

@@ -0,0 +1,88 @@
+// This file contains the interface that must be implemented by any collection
+// to be used as the backing store for a MemTable. Such a collection must
+// satisfy the following properties:
+//  (1) It does not store duplicate items.
+//  (2) It uses MemTableRep::KeyComparator to compare items for iteration and
+//     equality.
+//  (3) It can be accessed concurrently by multiple readers but need not support
+//      concurrent writes.
+//  (4) Items are never deleted.
+// The liberal use of assertions is encouraged to enforce (1).
+
+#ifndef STORAGE_LEVELDB_DB_TABLE_H_
+#define STORAGE_LEVELDB_DB_TABLE_H_
+
+#include <memory>
+#include "leveldb/arena.h"
+
+namespace leveldb {
+
+class MemTableRep {
+ public:
+  // KeyComparator(a, b) returns a negative value if a is less than b, 0 if they
+  // are equal, and a positive value if b is greater than a
+  class KeyComparator {
+  public:
+    virtual int operator()(const char* a, const char* b) const = 0;
+    virtual ~KeyComparator() { }
+  };
+
+  // Insert key into the collection. (The caller will pack key and value into a
+  // single buffer and pass that in as the parameter to Insert)
+  // REQUIRES: nothing that compares equal to key is currently in the
+  // collection.
+  virtual void Insert(const char* key) = 0;
+
+  // Returns true iff an entry that compares equal to key is in the collection.
+  virtual bool Contains(const char* key) const = 0;
+
+  virtual ~MemTableRep() { }
+
+  // Iteration over the contents of a skip collection
+  class Iterator {
+   public:
+    // Initialize an iterator over the specified collection.
+    // The returned iterator is not valid.
+    // explicit Iterator(const MemTableRep* collection);
+    virtual ~Iterator() { };
+
+    // Returns true iff the iterator is positioned at a valid node.
+    virtual bool Valid() const = 0;
+
+    // Returns the key at the current position.
+    // REQUIRES: Valid()
+    virtual const char* key() const = 0;
+
+    // Advances to the next position.
+    // REQUIRES: Valid()
+    virtual void Next() = 0;
+
+    // Advances to the previous position.
+    // REQUIRES: Valid()
+    virtual void Prev() = 0;
+
+    // Advance to the first entry with a key >= target
+    virtual void Seek(const char* target) = 0;
+
+    // Position at the first entry in collection.
+    // Final state of iterator is Valid() iff collection is not empty.
+    virtual void SeekToFirst() = 0;
+
+    // Position at the last entry in collection.
+    // Final state of iterator is Valid() iff collection is not empty.
+    virtual void SeekToLast() = 0;
+  };
+
+  virtual std::shared_ptr<Iterator> GetIterator() = 0;
+};
+
+class MemTableRepFactory {
+ public:
+  virtual ~MemTableRepFactory() { };
+  virtual std::shared_ptr<MemTableRep> CreateMemTableRep(
+    MemTableRep::KeyComparator&, Arena* arena) = 0;
+};
+
+}
+
+#endif // STORAGE_LEVELDB_DB_TABLE_H_

include/leveldb/merge_operator.h

@@ -0,0 +1,147 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef STORAGE_LEVELDB_INCLUDE_MERGE_OPERATOR_H_
+#define STORAGE_LEVELDB_INCLUDE_MERGE_OPERATOR_H_
+
+#include <string>
+#include <deque>
+#include "leveldb/slice.h"  // TODO: Remove this when migration is done;
+
+namespace leveldb {
+
+class Slice;
+class Logger;
+
+// The Merge Operator
+//
+// Essentially, a MergeOperator specifies the SEMANTICS of a merge, which only
+// client knows. It could be numeric addition, list append, string
+// concatenation, edit data structure, ... , anything.
+// The library, on the other hand, is concerned with the exercise of this
+// interface, at the right time (during get, iteration, compaction...)
+//
+// To use merge, the client needs to provide an object implementing one of
+// the following interfaces:
+//  a) AssociativeMergeOperator - for most simple semantics (always take
+//    two values, and merge them into one value, which is then put back
+//    into rocksdb); numeric addition and string concatenation are examples;
+//
+//  b) MergeOperator - the generic class for all the more abstract / complex
+//    operations; one method (FullMerge) to merge a Put/Delete value with a
+//    merge operand; and another method (PartialMerge) that merges two
+//    operands together. this is especially useful if your key values have a
+//    complex structure but you would still like to support client-specific
+//    incremental updates.
+//
+// AssociativeMergeOperator is simpler to implement. MergeOperator is simply
+// more powerful.
+//
+// Refer to rocksdb-merge wiki for more details and example implementations.
+//
+class MergeOperator {
+ public:
+  virtual ~MergeOperator() {}
+
+  // Gives the client a way to express the read -> modify -> write semantics
+  // key:      (IN)    The key that's associated with this merge operation.
+  //                   Client could multiplex the merge operator based on it
+  //                   if the key space is partitioned and different subspaces
+  //                   refer to different types of data which have different
+  //                   merge operation semantics
+  // existing: (IN)    null indicates that the key does not exist before this op
+  // operand_list:(IN) the sequence of merge operations to apply, front() first.
+  // new_value:(OUT)   Client is responsible for filling the merge result here
+  // logger:   (IN)    Client could use this to log errors during merge.
+  //
+  // Return true on success.
+  // All values passed in will be client-specific values. So if this method
+  // returns false, it is because client specified bad data or there was
+  // internal corruption. This will be treated as an error by the library.
+  //
+  // Also make use of the *logger for error messages.
+  virtual bool FullMerge(const Slice& key,
+                         const Slice* existing_value,
+                         const std::deque<std::string>& operand_list,
+                         std::string* new_value,
+                         Logger* logger) const = 0;
+
+  // This function performs merge(left_op, right_op)
+  // when both the operands are themselves merge operation types
+  // that you would have passed to a DB::Merge() call in the same order
+  // (i.e.: DB::Merge(key,left_op), followed by DB::Merge(key,right_op)).
+  //
+  // PartialMerge should combine them into a single merge operation that is
+  // saved into *new_value, and then it should return true.
+  // *new_value should be constructed such that a call to
+  // DB::Merge(key, *new_value) would yield the same result as a call
+  // to DB::Merge(key, left_op) followed by DB::Merge(key, right_op).
+  //
+  // If it is impossible or infeasible to combine the two operations,
+  // leave new_value unchanged and return false. The library will
+  // internally keep track of the operations, and apply them in the
+  // correct order once a base-value (a Put/Delete/End-of-Database) is seen.
+  //
+  // TODO: Presently there is no way to differentiate between error/corruption
+  // and simply "return false". For now, the client should simply return
+  // false in any case it cannot perform partial-merge, regardless of reason.
+  // If there is corruption in the data, handle it in the FullMerge() function,
+  // and return false there.
+  virtual bool PartialMerge(const Slice& key,
+                            const Slice& left_operand,
+                            const Slice& right_operand,
+                            std::string* new_value,
+                            Logger* logger) const = 0;
+
+  // The name of the MergeOperator. Used to check for MergeOperator
+  // mismatches (i.e., a DB created with one MergeOperator is
+  // accessed using a different MergeOperator)
+  // TODO: the name is currently not stored persistently and thus
+  //       no checking is enforced. Client is responsible for providing
+  //       consistent MergeOperator between DB opens.
+  virtual const char* Name() const = 0;
+};
+
+// The simpler, associative merge operator.
+class AssociativeMergeOperator : public MergeOperator {
+ public:
+  virtual ~AssociativeMergeOperator() {}
+
+  // Gives the client a way to express the read -> modify -> write semantics
+  // key:           (IN) The key that's associated with this merge operation.
+  // existing_value:(IN) null indicates the key does not exist before this op
+  // value:         (IN) the value to update/merge the existing_value with
+  // new_value:    (OUT) Client is responsible for filling the merge result here
+  // logger:        (IN) Client could use this to log errors during merge.
+  //
+  // Return true on success.
+  // All values passed in will be client-specific values. So if this method
+  // returns false, it is because client specified bad data or there was
+  // internal corruption. The client should assume that this will be treated
+  // as an error by the library.
+  virtual bool Merge(const Slice& key,
+                     const Slice* existing_value,
+                     const Slice& value,
+                     std::string* new_value,
+                     Logger* logger) const = 0;
+
+
+ private:
+  // Default implementations of the MergeOperator functions
+  virtual bool FullMerge(const Slice& key,
+                         const Slice* existing_value,
+                         const std::deque<std::string>& operand_list,
+                         std::string* new_value,
+                         Logger* logger) const override;
+
+  virtual bool PartialMerge(const Slice& key,
+                            const Slice& left_operand,
+                            const Slice& right_operand,
+                            std::string* new_value,
+                            Logger* logger) const override;
+};
+
+}  // namespace leveldb
+
+#endif  // STORAGE_LEVELDB_INCLUDE_MERGE_OPERATOR_H_

include/leveldb/options.h

@@ -7,8 +7,15 @@
 
 #include <stddef.h>
 #include <string>
+#include <memory>
+#include <vector>
 #include <stdint.h>
 #include "leveldb/slice.h"
+#include "leveldb/statistics.h"
+#include "leveldb/universal_compaction.h"
+#include "leveldb/memtablerep.h"
+
+#include "leveldb/slice_transform.h"
 
 namespace leveldb {
 
@@ -17,8 +24,12 @@ class Comparator;
 class Env;
 class FilterPolicy;
 class Logger;
+class MergeOperator;
 class Snapshot;
-class Statistics;
+class CompactionFilter;
+class CompactionFilterFactory;
+
+using std::shared_ptr;
 
 // DB contents are stored in a set of blocks, each of which holds a
 // sequence of key,value pairs.  Each block may be compressed before
@@ -33,6 +44,11 @@ enum CompressionType {
   kBZip2Compression = 0x3
 };
 
+enum CompactionStyle {
+  kCompactionStyleLevel       = 0x0, // level based compaction style
+  kCompactionStyleUniversal   = 0x1  // Universal compaction style
+};
+
 // Compression options for different compression algorithms like Zlib
 struct CompressionOptions {
   int window_bits;
@@ -59,6 +75,28 @@ struct Options {
   // comparator provided to previous open calls on the same DB.
   const Comparator* comparator;
 
+  // REQUIRES: The client must provide a merge operator if Merge operation
+  // needs to be accessed. Calling Merge on a DB without a merge operator
+  // would result in Status::NotSupported. The client must ensure that the
+  // merge operator supplied here has the same name and *exactly* the same
+  // semantics as the merge operator provided to previous open calls on
+  // the same DB. The only exception is reserved for upgrade, where a DB
+  // previously without a merge operator is introduced to Merge operation
+  // for the first time. It's necessary to specify a merge operator when
+  // openning the DB in this case.
+  // Default: nullptr
+  shared_ptr<MergeOperator> merge_operator;
+
+  // The client must provide compaction_filter_factory if it requires a new
+  // compaction filter to be used for different compaction processes
+  // Allows an application to modify/delete a key-value during background
+  // compaction.
+  // Ideally, client should specify only one of filter or factory.
+  // compaction_filter takes precedence over compaction_filter_factory if
+  // client specifies both.
+  // Default: nullptr
+  const CompactionFilter* compaction_filter;
+
   // If true, the database will be created if it is missing.
   // Default: false
   bool create_if_missing;
@@ -81,10 +119,10 @@ struct Options {
   Env* env;
 
   // Any internal progress/error information generated by the db will
-  // be written to info_log if it is non-NULL, or to a file stored
-  // in the same directory as the DB contents if info_log is NULL.
-  // Default: NULL
-  Logger* info_log;
+  // be written to info_log if it is non-nullptr, or to a file stored
+  // in the same directory as the DB contents if info_log is nullptr.
+  // Default: nullptr
+  shared_ptr<Logger> info_log;
 
   // -------------------
   // Parameters that affect performance
@@ -108,6 +146,15 @@ struct Options {
   // Default: 2
   int max_write_buffer_number;
 
+  // The minimum number of write buffers that will be merged together
+  // before writing to storage.  If set to 1, then
+  // all write buffers are fushed to L0 as individual files and this increases
+  // read amplification because a get request has to check in all of these
+  // files. Also, an in-memory merge may result in writing lesser
+  // data to storage if there are duplicate records in each of these
+  // individual write buffers.  Default: 1
+  int min_write_buffer_number_to_merge;
+
   // Number of open files that can be used by the DB.  You may need to
   // increase this if your database has a large working set (budget
   // one open file per 2MB of working set).
@@ -118,10 +165,10 @@ struct Options {
   // Control over blocks (user data is stored in a set of blocks, and
   // a block is the unit of reading from disk).
 
-  // If non-NULL, use the specified cache for blocks.
+  // If non-NULL use the specified cache for blocks.
   // If NULL, leveldb will automatically create and use an 8MB internal cache.
-  // Default: NULL
-  Cache* block_cache;
+  // Default: nullptr
+  shared_ptr<Cache> block_cache;
 
   // Approximate size of user data packed per block.  Note that the
   // block size specified here corresponds to uncompressed data.  The
@@ -159,26 +206,48 @@ struct Options {
   // are cases where most lower levels would like to quick compression
   // algorithm while the higher levels (which have more data) use
   // compression algorithms that have better compression but could
-  // be slower. This array, if non NULL, should have an entry for
-  // each level of the database. This array, if non NULL, overides the
+  // be slower. This array, if non nullptr, should have an entry for
+  // each level of the database. This array, if non nullptr, overides the
   // value specified in the previous field 'compression'. The caller is
   // reponsible for allocating memory and initializing the values in it
   // before invoking Open(). The caller is responsible for freeing this
   // array and it could be freed anytime after the return from Open().
   // This could have been a std::vector but that makes the equivalent
   // java/C api hard to construct.
-  CompressionType* compression_per_level;
+  std::vector<CompressionType> compression_per_level;
 
   //different options for compression algorithms
   CompressionOptions compression_opts;
 
-  // If non-NULL, use the specified filter policy to reduce disk reads.
+  // If non-nullptr, use the specified filter policy to reduce disk reads.
   // Many applications will benefit from passing the result of
   // NewBloomFilterPolicy() here.
   //
-  // Default: NULL
+  // Default: nullptr
   const FilterPolicy* filter_policy;
 
+  // If non-nullptr, use the specified function to determine the
+  // prefixes for keys.  These prefixes will be placed in the filter.
+  // Depending on the workload, this can reduce the number of read-IOP
+  // cost for scans when a prefix is passed via ReadOptions to
+  // db.NewIterator().  For prefix filtering to work properly,
+  // "prefix_extractor" and "comparator" must be such that the following
+  // properties hold:
+  //
+  // 1) key.starts_with(prefix(key))
+  // 2) Compare(prefix(key), key) <= 0.
+  // 3) If Compare(k1, k2) <= 0, then Compare(prefix(k1), prefix(k2)) <= 0
+  // 4) prefix(prefix(key)) == prefix(key)
+  //
+  // Default: nullptr
+  const SliceTransform* prefix_extractor;
+
+  // If true, place whole keys in the filter (not just prefixes).
+  // This must generally be true for gets to be efficient.
+  //
+  // Default: true
+  bool whole_key_filtering;
+
   // Number of levels for this database
   int num_levels;
 
@@ -186,9 +255,9 @@ struct Options {
   // level-0 compaction will not be triggered by number of files at all.
   int level0_file_num_compaction_trigger;
 
-  // Soft limit on number of level-0 files. We slow down writes at this point.
-  // A value <0 means that no writing slow down will be triggered by number
-  // of files in level-0.
+  // Soft limit on number of level-0 files. We start slowing down writes at this
+  // point. A value <0 means that no writing slow down will be triggered by
+  // number of files in level-0.
   int level0_slowdown_writes_trigger;
 
   // Maximum number of level-0 files.  We stop writes at this point.
@@ -232,6 +301,12 @@ struct Options {
   // by default 'max_bytes_for_level_base' is 10.
   int max_bytes_for_level_multiplier;
 
+  // Different max-size multipliers for different levels.
+  // These are multiplied by max_bytes_for_level_multiplier to arrive
+  // at the max-size of each level.
+  // Default: 1
+  std::vector<int> max_bytes_for_level_multiplier_additional;
+
   // Maximum number of bytes in all compacted files.  We avoid expanding
   // the lower level file set of a compaction if it would make the
   // total compaction cover more than
@@ -252,7 +327,9 @@ struct Options {
   int max_grandparent_overlap_factor;
 
   // If non-null, then we should collect metrics about database operations
-  Statistics* statistics;
+  // Statistics objects should not be shared between DB instances as
+  // it does not use any locks to prevent concurrent updates.
+  shared_ptr<Statistics> statistics;
 
   // If true, then the contents of data files are not synced
   // to stable storage. Their contents remain in the OS buffers till the
@@ -304,41 +381,67 @@ struct Options {
   // log file.
   size_t max_log_file_size;
 
-  // Puts are delayed when any level has a compaction score that
-  // exceeds rate_limit. This is ignored when <= 1.0.
-  double rate_limit;
+  // Time for the info log file to roll (in seconds).
+  // If specified with non-zero value, log file will be rolled
+  // if it has been active longer than `log_file_time_to_roll`.
+  // Default: 0 (disabled)
+  size_t log_file_time_to_roll;
 
-  // Disable block cache. If this is set to false,
+  // Maximal info log files to be kept.
+  // Default: 1000
+  size_t keep_log_file_num;
+
+  // Puts are delayed 0-1 ms when any level has a compaction score that exceeds
+  // soft_rate_limit. This is ignored when == 0.0.
+  // CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not
+  // hold, RocksDB will set soft_rate_limit = hard_rate_limit
+  // Default: 0 (disabled)
+  double soft_rate_limit;
+
+  // Puts are delayed 1ms at a time when any level has a compaction score that
+  // exceeds hard_rate_limit. This is ignored when <= 1.0.
+  // Default: 0 (disabled)
+  double hard_rate_limit;
+
+  // Max time a put will be stalled when hard_rate_limit is enforced. If 0, then
+  // there is no limit.
+  // Default: 1000
+  unsigned int rate_limit_delay_max_milliseconds;
+
+  // manifest file is rolled over on reaching this limit.
+  // The older manifest file be deleted.
+  // The default value is MAX_INT so that roll-over does not take place.
+  uint64_t max_manifest_file_size;
+
+  // Disable block cache. If this is set to true,
   // then no block cache should be used, and the block_cache should
-  // point to a NULL object.
+  // point to a nullptr object.
+  // Default: false
   bool no_block_cache;
 
   // Number of shards used for table cache.
   int table_cache_numshardbits;
 
+  // size of one block in arena memory allocation.
+  // If <= 0, a proper value is automatically calculated (usually 1/10 of
+  // writer_buffer_size).
+  //
+  // Default: 0
+  size_t arena_block_size;
+
   // Create an Options object with default values for all fields.
   Options();
 
-  void Dump(Logger * log) const;
+  void Dump(Logger* log) const;
 
-  // This method allows an application to modify/delete a key-value at
-  // the time of compaction. The compaction process invokes this
-  // method for every kv that is being compacted. A return value
-  // of false indicates that the kv should be preserved in the
-  // output of this compaction run and a return value of true
-  // indicates that this key-value should be removed from the
-  // output of the compaction.  The application can inspect
-  // the existing value of the key, modify it if needed and
-  // return back the new value for this key. The application
-  // should allocate memory for the Slice object that is used to
-  // return the new value and the leveldb framework will
-  // free up that memory.
-  // The compaction_filter_args, if specified here, are passed
-  // back to the invocation of the CompactionFilter.
-  void* compaction_filter_args;
-  bool (*CompactionFilter)(void* compaction_filter_args,
-         int level, const Slice& key,
-         const Slice& existing_value, Slice** new_value);
+  // Set appropriate parameters for bulk loading.
+  // The reason that this is a function that returns "this" instead of a
+  // constructor is to enable chaining of multiple similar calls in the future.
+  //
+  // All data will be in level 0 without any automatic compaction.
+  // It's recommended to manually call CompactRange(NULL, NULL) before reading
+  // from the database, because otherwise the read can be very slow.
+  Options* PrepareForBulkLoad();
 
   // Disable automatic compactions. Manual compactions can still
   // be issued on this database.
@@ -353,6 +456,92 @@ struct Options {
   // deleted.
   // Default : 0
   uint64_t WAL_ttl_seconds;
+
+  // Number of bytes to preallocate (via fallocate) the manifest
+  // files.  Default is 4mb, which is reasonable to reduce random IO
+  // as well as prevent overallocation for mounts that preallocate
+  // large amounts of data (such as xfs's allocsize option).
+  size_t manifest_preallocation_size;
+
+  // Purge duplicate/deleted keys when a memtable is flushed to storage.
+  // Default: true
+  bool purge_redundant_kvs_while_flush;
+
+  // Data being read from file storage may be buffered in the OS
+  // Default: true
+  bool allow_os_buffer;
+
+  // Allow the OS to mmap file for reading sst tables. Default: false
+  bool allow_mmap_reads;
+
+  // Allow the OS to mmap file for writing. Default: true
+  bool allow_mmap_writes;
+
+  // Disable child process inherit open files. Default: true
+  bool is_fd_close_on_exec;
+
+  // Skip log corruption error on recovery (If client is ok with
+  // losing most recent changes)
+  // Default: false
+  bool skip_log_error_on_recovery;
+
+  // if not zero, dump leveldb.stats to LOG every stats_dump_period_sec
+  // Default: 3600 (1 hour)
+  unsigned int stats_dump_period_sec;
+
+  // This is used to close a block before it reaches the configured
+  // 'block_size'. If the percentage of free space in the current block is less
+  // than this specified number and adding a new record to the block will
+  // exceed the configured block size, then this block will be closed and the
+  // new record will be written to the next block.
+  // Default is 10.
+  int block_size_deviation;
+
+  // If set true, will hint the underlying file system that the file
+  // access pattern is random, when a sst file is opened.
+  // Default: true
+  bool advise_random_on_open;
+
+  // Specify the file access pattern once a compaction is started.
+  // It will be applied to all input files of a compaction.
+  // Default: NORMAL
+  enum { NONE, NORMAL, SEQUENTIAL, WILLNEED } access_hint_on_compaction_start;
+
+  // Use adaptive mutex, which spins in the user space before resorting
+  // to kernel. This could reduce context switch when the mutex is not
+  // heavily contended. However, if the mutex is hot, we could end up
+  // wasting spin time.
+  // Default: false
+  bool use_adaptive_mutex;
+
+  // Allows OS to incrementally sync files to disk while they are being
+  // written, asynchronously, in the background.
+  // Issue one request for every bytes_per_sync written. 0 turns it off.
+  // Default: 0
+  uint64_t bytes_per_sync;
+
+  // The compaction style. Default: kCompactionStyleLevel
+  CompactionStyle compaction_style;
+
+  // The options needed to support Universal Style compactions
+  CompactionOptionsUniversal compaction_options_universal;
+
+  // Use KeyMayExist API to filter deletes when this is true.
+  // If KeyMayExist returns false, i.e. the key definitely does not exist, then
+  // the delete is a noop. KeyMayExist only incurs in-memory look up.
+  // This optimization avoids writing the delete to storage when appropriate.
+  // Default: false
+  bool filter_deletes;
+
+  // This is a factory that provides MemTableRep objects.
+  // Default: a factory that provides a skip-list-based implementation of
+  // MemTableRep.
+  std::shared_ptr<MemTableRepFactory> memtable_factory;
+
+  // This is a factory that provides compaction filter objects which allow
+  // an application to modify/delete a key-value during background compaction.
+  // Default: a factory that doesn't provide any object
+  std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
 };
 
 // Options that control read operations
@@ -367,21 +556,35 @@ struct ReadOptions {
   // Default: true
   bool fill_cache;
 
-  // If "snapshot" is non-NULL, read as of the supplied snapshot
+  // If "snapshot" is non-nullptr, read as of the supplied snapshot
   // (which must belong to the DB that is being read and which must
-  // not have been released).  If "snapshot" is NULL, use an impliicit
+  // not have been released).  If "snapshot" is nullptr, use an impliicit
   // snapshot of the state at the beginning of this read operation.
-  // Default: NULL
+  // Default: nullptr
   const Snapshot* snapshot;
 
+  // If "prefix" is non-nullptr, and ReadOptions is being passed to
+  // db.NewIterator, only return results when the key begins with this
+  // prefix.  This field is ignored by other calls (e.g., Get).
+  // Options.prefix_extractor must also be set, and
+  // prefix_extractor.InRange(prefix) must be true.  The iterator
+  // returned by NewIterator when this option is set will behave just
+  // as if the underlying store did not contain any non-matching keys,
+  // with two exceptions.  Seek() only accepts keys starting with the
+  // prefix, and SeekToLast() is not supported.  prefix filter with this
+  // option will sometimes reduce the number of read IOPs.
+  // Default: nullptr
+  const Slice* prefix;
+
   ReadOptions()
       : verify_checksums(false),
         fill_cache(true),
-        snapshot(NULL) {
+        snapshot(nullptr),
+        prefix(nullptr) {
   }
   ReadOptions(bool cksum, bool cache) :
               verify_checksums(cksum), fill_cache(cache),
-              snapshot(NULL) {
+              snapshot(nullptr), prefix(nullptr) {
   }
 };
 

include/leveldb/perf_context.h

@@ -0,0 +1,24 @@
+#ifndef STORAGE_LEVELDB_INCLUDE_PERF_CONTEXT_H
+#define STORAGE_LEVELDB_INCLUDE_PERF_CONTEXT_H
+
+#include <stdint.h>
+
+namespace leveldb {
+
+// A thread local context for gathering performance counter efficiently
+// and transparently.
+
+struct PerfContext {
+
+  void Reset(); // reset all performance counters to zero
+
+
+  uint64_t user_key_comparison_count; // total number of user key comparisons
+};
+
+extern __thread PerfContext perf_context;
+
+}
+
+
+#endif

include/leveldb/slice_transform.h

@@ -0,0 +1,41 @@
+// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// Class for specifying user-defined functions which perform a
+// transformation on a slice.  It is not required that every slice
+// belong to the domain and/or range of a function.  Subclasses should
+// define InDomain and InRange to determine which slices are in either
+// of these sets respectively.
+
+#ifndef STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_
+#define STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_
+
+#include <string>
+
+namespace leveldb {
+
+class Slice;
+
+class SliceTransform {
+ public:
+  virtual ~SliceTransform() {};
+
+  // Return the name of this transformation.
+  virtual const char* Name() const = 0;
+
+  // transform a src in domain to a dst in the range
+  virtual Slice Transform(const Slice& src) const = 0;
+
+  // determine whether this is a valid src upon the function applies
+  virtual bool InDomain(const Slice& src) const = 0;
+
+  // determine whether dst=Transform(src) for some src
+  virtual bool InRange(const Slice& dst) const = 0;
+};
+
+extern const SliceTransform* NewFixedPrefixTransform(size_t prefix_len);
+
+}
+
+#endif  // STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_

include/leveldb/statistics.h

@@ -5,12 +5,21 @@
 #ifndef STORAGE_LEVELDB_INCLUDE_STATISTICS_H_
 #define STORAGE_LEVELDB_INCLUDE_STATISTICS_H_
 
+#include <atomic>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <memory>
+#include <vector>
+
 namespace leveldb {
 
 /**
  * Keep adding ticker's here.
  * Any ticker should have a value less than TICKER_ENUM_MAX.
  * Add a new ticker by assigning it the current value of TICKER_ENUM_MAX
+ * Add a string representation in TickersNameMap below.
  * And incrementing TICKER_ENUM_MAX.
  */
 enum Tickers {
@@ -24,9 +33,144 @@ enum Tickers {
   COMPACTION_KEY_DROP_NEWER_ENTRY = 3, // key was written with a newer value.
   COMPACTION_KEY_DROP_OBSOLETE = 4, // The key is obsolete.
   COMPACTION_KEY_DROP_USER = 5, // user compaction function has dropped the key.
-  TICKER_ENUM_MAX = 6,
+  // Number of keys written to the database via the Put and Write call's
+  NUMBER_KEYS_WRITTEN = 6,
+  // Number of Keys read,
+  NUMBER_KEYS_READ = 7,
+  // Bytes written / read
+  BYTES_WRITTEN = 8,
+  BYTES_READ = 9,
+  NO_FILE_CLOSES = 10,
+  NO_FILE_OPENS = 11,
+  NO_FILE_ERRORS = 12,
+  // Time system had to wait to do LO-L1 compactions
+  STALL_L0_SLOWDOWN_MICROS = 13,
+  // Time system had to wait to move memtable to L1.
+  STALL_MEMTABLE_COMPACTION_MICROS = 14,
+  // write throttle because of too many files in L0
+  STALL_L0_NUM_FILES_MICROS = 15,
+  RATE_LIMIT_DELAY_MILLIS = 16,
+
+  NO_ITERATORS = 17, // number of iterators currently open
+
+  // Number of MultiGet calls, keys read, and bytes read
+  NUMBER_MULTIGET_CALLS = 18,
+  NUMBER_MULTIGET_KEYS_READ = 19,
+  NUMBER_MULTIGET_BYTES_READ = 20,
+
+  NUMBER_FILTERED_DELETES = 21,
+  NUMBER_MERGE_FAILURES = 22,
+  SEQUENCE_NUMBER = 23,
+
+  TICKER_ENUM_MAX = 24
 };
 
+const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
+  { BLOCK_CACHE_MISS, "rocksdb.block.cache.miss" },
+  { BLOCK_CACHE_HIT, "rocksdb.block.cache.hit" },
+  { BLOOM_FILTER_USEFUL, "rocksdb.bloom.filter.useful" },
+  { COMPACTION_KEY_DROP_NEWER_ENTRY, "rocksdb.compaction.key.drop.new" },
+  { COMPACTION_KEY_DROP_OBSOLETE, "rocksdb.compaction.key.drop.obsolete" },
+  { COMPACTION_KEY_DROP_USER, "rocksdb.compaction.key.drop.user" },
+  { NUMBER_KEYS_WRITTEN, "rocksdb.number.keys.written" },
+  { NUMBER_KEYS_READ, "rocksdb.number.keys.read" },
+  { BYTES_WRITTEN, "rocksdb.bytes.written" },
+  { BYTES_READ, "rocksdb.bytes.read" },
+  { NO_FILE_CLOSES, "rocksdb.no.file.closes" },
+  { NO_FILE_OPENS, "rocksdb.no.file.opens" },
+  { NO_FILE_ERRORS, "rocksdb.no.file.errors" },
+  { STALL_L0_SLOWDOWN_MICROS, "rocksdb.l0.slowdown.micros" },
+  { STALL_MEMTABLE_COMPACTION_MICROS, "rocksdb.memtable.compaction.micros" },
+  { STALL_L0_NUM_FILES_MICROS, "rocksdb.l0.num.files.stall.micros" },
+  { RATE_LIMIT_DELAY_MILLIS, "rocksdb.rate.limit.delay.millis" },
+  { NO_ITERATORS, "rocksdb.num.iterators" },
+  { NUMBER_MULTIGET_CALLS, "rocksdb.number.multiget.get" },
+  { NUMBER_MULTIGET_KEYS_READ, "rocksdb.number.multiget.keys.read" },
+  { NUMBER_MULTIGET_BYTES_READ, "rocksdb.number.multiget.bytes.read" },
+  { NUMBER_FILTERED_DELETES, "rocksdb.number.deletes.filtered" },
+  { NUMBER_MERGE_FAILURES, "rocksdb.number.merge.failures" },
+  { SEQUENCE_NUMBER, "rocksdb.sequence.number" }
+};
+
+/**
+ * Keep adding histogram's here.
+ * Any histogram whould have value less than HISTOGRAM_ENUM_MAX
+ * Add a new Histogram by assigning it the current value of HISTOGRAM_ENUM_MAX
+ * Add a string representation in HistogramsNameMap below
+ * And increment HISTOGRAM_ENUM_MAX
+ */
+enum Histograms {
+  DB_GET = 0,
+  DB_WRITE = 1,
+  COMPACTION_TIME = 2,
+  TABLE_SYNC_MICROS = 3,
+  COMPACTION_OUTFILE_SYNC_MICROS = 4,
+  WAL_FILE_SYNC_MICROS = 5,
+  MANIFEST_FILE_SYNC_MICROS = 6,
+  // TIME SPENT IN IO DURING TABLE OPEN
+  TABLE_OPEN_IO_MICROS = 7,
+  DB_MULTIGET = 8,
+  READ_BLOCK_COMPACTION_MICROS = 9,
+  READ_BLOCK_GET_MICROS = 10,
+  WRITE_RAW_BLOCK_MICROS = 11,
+
+  STALL_L0_SLOWDOWN_COUNT = 12,
+  STALL_MEMTABLE_COMPACTION_COUNT = 13,
+  STALL_L0_NUM_FILES_COUNT = 14,
+  HARD_RATE_LIMIT_DELAY_COUNT = 15,
+  SOFT_RATE_LIMIT_DELAY_COUNT = 16,
+  NUM_FILES_IN_SINGLE_COMPACTION = 17,
+  HISTOGRAM_ENUM_MAX = 18
+};
+
+const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
+  { DB_GET, "rocksdb.db.get.micros" },
+  { DB_WRITE, "rocksdb.db.write.micros" },
+  { COMPACTION_TIME, "rocksdb.compaction.times.micros" },
+  { TABLE_SYNC_MICROS, "rocksdb.table.sync.micros" },
+  { COMPACTION_OUTFILE_SYNC_MICROS, "rocksdb.compaction.outfile.sync.micros" },
+  { WAL_FILE_SYNC_MICROS, "rocksdb.wal.file.sync.micros" },
+  { MANIFEST_FILE_SYNC_MICROS, "rocksdb.manifest.file.sync.micros" },
+  { TABLE_OPEN_IO_MICROS, "rocksdb.table.open.io.micros" },
+  { DB_MULTIGET, "rocksdb.db.multiget.micros" },
+  { READ_BLOCK_COMPACTION_MICROS, "rocksdb.read.block.compaction.micros" },
+  { READ_BLOCK_GET_MICROS, "rocksdb.read.block.get.micros" },
+  { WRITE_RAW_BLOCK_MICROS, "rocksdb.write.raw.block.micros" },
+  { STALL_L0_SLOWDOWN_COUNT, "rocksdb.l0.slowdown.count"},
+  { STALL_MEMTABLE_COMPACTION_COUNT, "rocksdb.memtable.compaction.count"},
+  { STALL_L0_NUM_FILES_COUNT, "rocksdb.num.files.stall.count"},
+  { HARD_RATE_LIMIT_DELAY_COUNT, "rocksdb.hard.rate.limit.delay.count"},
+  { SOFT_RATE_LIMIT_DELAY_COUNT, "rocksdb.soft.rate.limit.delay.count"},
+  { NUM_FILES_IN_SINGLE_COMPACTION, "rocksdb.numfiles.in.singlecompaction" }
+};
+
+struct HistogramData {
+  double median;
+  double percentile95;
+  double percentile99;
+  double average;
+  double standard_deviation;
+};
+
+
+class Histogram {
+ public:
+  // clear's the histogram
+  virtual void Clear() = 0;
+  virtual ~Histogram();
+  // Add a value to be recorded in the histogram.
+  virtual void Add(uint64_t value) = 0;
+
+  virtual std::string ToString() const = 0;
+
+  // Get statistics
+  virtual double Median() const = 0;
+  virtual double Percentile(double p) const = 0;
+  virtual double Average() const = 0;
+  virtual double StandardDeviation() const = 0;
+  virtual void Data(HistogramData * const data) const = 0;
+
+};
 
 /**
  * A dumb ticker which keeps incrementing through its life time.
@@ -36,8 +180,12 @@ class Ticker {
  public:
   Ticker() : count_(0) { }
 
-  inline void recordTick() {
-    count_++;
+  inline void setTickerCount(uint64_t count) {
+    count_ = count;
+  }
+
+  inline void recordTick(int count = 1) {
+    count_ += count;
   }
 
   inline uint64_t getCount() {
@@ -45,41 +193,47 @@ class Ticker {
   }
 
  private:
-  uint64_t count_;
-
+  std::atomic_uint_fast64_t count_;
 };
 
 // Analyze the performance of a db
 class Statistics {
  public:
-  // Create an Statistics object with default values for all fields.
-  Statistics() : numFileOpens_(0), numFileCloses_(0),
-                 numFileErrors_(0) {}
-
-  virtual void incNumFileOpens() = 0;
-  virtual void incNumFileCloses() = 0;
-  virtual void incNumFileErrors() = 0;
-
-  virtual long getNumFileOpens() { return numFileOpens_;}
-  virtual long getNumFileCloses() { return numFileCloses_;}
-  virtual long getNumFileErrors() { return numFileErrors_;}
-  virtual ~Statistics() {}
 
   virtual long getTickerCount(Tickers tickerType) = 0;
-  virtual void recordTick(Tickers tickerType) = 0;
+  virtual void recordTick(Tickers tickerType, uint64_t count = 0) = 0;
+  virtual void setTickerCount(Tickers tickerType, uint64_t count) = 0;
+  virtual void measureTime(Histograms histogramType, uint64_t time) = 0;
 
- protected:
-  long numFileOpens_;
-  long numFileCloses_;
-  long numFileErrors_;
+  virtual void histogramData(Histograms type, HistogramData * const data) = 0;
+  // String representation of the statistic object.
+  std::string ToString();
 };
 
+// Create a concrete DBStatistics object
+std::shared_ptr<Statistics> CreateDBStatistics();
+
 // Ease of Use functions
-inline void RecordTick(Statistics* const statistics, Tickers ticker) {
-  if (statistics != NULL) {
-    statistics->recordTick(ticker);
+inline void RecordTick(std::shared_ptr<Statistics> statistics,
+                       Tickers ticker,
+                       uint64_t count = 1) {
+  assert(HistogramsNameMap.size() == HISTOGRAM_ENUM_MAX);
+  assert(TickersNameMap.size() == TICKER_ENUM_MAX);
+  if (statistics) {
+    statistics->recordTick(ticker, count);
   }
-};
+}
+
+inline void SetTickerCount(std::shared_ptr<Statistics> statistics,
+                           Tickers ticker,
+                           uint64_t count) {
+  assert(HistogramsNameMap.size() == HISTOGRAM_ENUM_MAX);
+  assert(TickersNameMap.size() == TICKER_ENUM_MAX);
+  if (statistics) {
+    statistics->setTickerCount(ticker, count);
+  }
+}
+
 }  // namespace leveldb
 
 #endif  // STORAGE_LEVELDB_INCLUDE_STATISTICS_H_

include/leveldb/status.h

@@ -21,7 +21,7 @@ namespace leveldb {
 class Status {
  public:
   // Create a success status.
-  Status() : state_(NULL) { }
+  Status() : state_(nullptr) { }
   ~Status() { delete[] state_; }
 
   // Copy the specified status.
@@ -47,9 +47,12 @@ class Status {
   static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) {
     return Status(kIOError, msg, msg2);
   }
+  static Status MergeInProgress(const Slice& msg, const Slice& msg2 = Slice()) {
+    return Status(kMergeInProgress, msg, msg2);
+  }
 
   // Returns true iff the status indicates success.
-  bool ok() const { return (state_ == NULL); }
+  bool ok() const { return (state_ == nullptr); }
 
   // Returns true iff the status indicates a NotFound error.
   bool IsNotFound() const { return code() == kNotFound; }
@@ -57,15 +60,24 @@ class Status {
   // Returns true iff the status indicates a Corruption error.
   bool IsCorruption() const { return code() == kCorruption; }
 
+  // Returns true iff the status indicates a NotSupported error.
+  bool IsNotSupported() const { return code() == kNotSupported; }
+
+  // Returns true iff the status indicates an InvalidArgument error.
+  bool IsInvalidArgument() const { return code() == kInvalidArgument; }
+
   // Returns true iff the status indicates an IOError.
   bool IsIOError() const { return code() == kIOError; }
 
+  // Returns true iff the status indicates an MergeInProgress.
+  bool IsMergeInProgress() const { return code() == kMergeInProgress; }
+
   // Return a string representation of this status suitable for printing.
   // Returns the string "OK" for success.
   std::string ToString() const;
 
  private:
-  // OK status has a NULL state_.  Otherwise, state_ is a new[] array
+  // OK status has a nullptr state_.  Otherwise, state_ is a new[] array
   // of the following form:
   //    state_[0..3] == length of message
   //    state_[4]    == code
@@ -78,11 +90,12 @@ class Status {
     kCorruption = 2,
     kNotSupported = 3,
     kInvalidArgument = 4,
-    kIOError = 5
+    kIOError = 5,
+    kMergeInProgress = 6
   };
 
   Code code() const {
-    return (state_ == NULL) ? kOk : static_cast<Code>(state_[4]);
+    return (state_ == nullptr) ? kOk : static_cast<Code>(state_[4]);
   }
 
   Status(Code code, const Slice& msg, const Slice& msg2);
@@ -90,14 +103,14 @@ class Status {
 };
 
 inline Status::Status(const Status& s) {
-  state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
+  state_ = (s.state_ == nullptr) ? nullptr : CopyState(s.state_);
 }
 inline void Status::operator=(const Status& s) {
   // The following condition catches both aliasing (when this == &s),
   // and the common case where both s and *this are ok.
   if (state_ != s.state_) {
     delete[] state_;
-    state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
+    state_ = (s.state_ == nullptr) ? nullptr : CopyState(s.state_);
   }
 }
 

include/leveldb/transaction_log.h

@@ -0,0 +1,78 @@
+// Copyright 2008-present Facebook. All Rights Reserved.
+#ifndef STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
+#define STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
+
+#include "leveldb/status.h"
+#include "leveldb/types.h"
+#include "leveldb/write_batch.h"
+
+namespace leveldb {
+
+class LogFile;
+typedef std::vector<std::unique_ptr<LogFile>> VectorLogPtr;
+
+enum  WalFileType {
+  /* Indicates that WAL file is in archive directory. WAL files are moved from
+   * the main db directory to archive directory once they are not live and stay
+   * there for a duration of WAL_ttl_seconds which can be set in Options
+   */
+  kArchivedLogFile = 0,
+
+  /* Indicates that WAL file is live and resides in the main db directory */
+  kAliveLogFile = 1
+} ;
+
+class LogFile {
+ public:
+  LogFile() {}
+  virtual ~LogFile() {}
+
+  // Returns log file's name excluding the db path
+  virtual std::string Filename() const = 0;
+
+  // Primary identifier for log file.
+  // This is directly proportional to creation time of the log file
+  virtual uint64_t LogNumber() const = 0;
+
+  // Log file can be either alive or archived
+  virtual WalFileType Type() const = 0;
+
+  // Starting sequence number of writebatch written in this log file
+  virtual SequenceNumber StartSequence() const = 0;
+
+  // Size of log file on disk in Bytes
+  virtual uint64_t SizeFileBytes() const = 0;
+};
+
+struct BatchResult {
+  SequenceNumber sequence;
+  std::unique_ptr<WriteBatch> writeBatchPtr;
+};
+
+//  A TransactionLogIterator is used to iterate over the Transaction's in a db.
+class TransactionLogIterator {
+ public:
+  TransactionLogIterator() {}
+  virtual ~TransactionLogIterator() {}
+
+  // An iterator is either positioned at a WriteBatch or not valid.
+  // This method returns true if the iterator is valid.
+  // Can read data from a valid iterator.
+  virtual bool Valid() = 0;
+
+  // Moves the iterator to the next WriteBatch.
+  // REQUIRES: Valid() to be true.
+  virtual void Next() = 0;
+
+  // Return's ok if the iterator is valid.
+  // Return the Error when something has gone wrong.
+  virtual Status status() = 0;
+
+  // If valid return's the current write_batch and the sequence number of the
+  // latest transaction contained in the batch.
+  // ONLY use if Valid() is true and status() is OK.
+  virtual BatchResult GetBatch() = 0;
+};
+} //  namespace leveldb
+
+#endif  // STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_

include/leveldb/transaction_log_iterator.h

@@ -1,35 +0,0 @@
-// Copyright 2008-present Facebook. All Rights Reserved.
-#ifndef STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
-#define STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
-
-#include "leveldb/status.h"
-#include "leveldb/write_batch.h"
-
-namespace leveldb {
-
-
-//  A TransactionLogIterator is used to iterate over the Transaction's in a db.
-class TransactionLogIterator {
- public:
-  TransactionLogIterator() {}
-  virtual ~TransactionLogIterator() {}
-
-  // An iterator is either positioned at a WriteBatch or not valid.
-  // This method returns true if the iterator is valid.
-  virtual bool Valid() = 0;
-
-  // Moves the iterator to the next WriteBatch.
-  // REQUIRES: Valid() to be true.
-  virtual void Next() = 0;
-
-  // Return's ok if the iterator is in a valid stated.
-  // Return the Error Status when the iterator is not Valid.
-  virtual Status status() = 0;
-
-  // If valid return's the current write_batch and the sequence number of the
-  // latest transaction contained in the batch.
-  virtual void GetBatch(WriteBatch* batch, SequenceNumber* seq) = 0;
-};
-} //  namespace leveldb
-
-#endif  // STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_

include/leveldb/types.h

@@ -7,7 +7,7 @@ namespace leveldb {
 
 // Define all public custom types here.
 
-//  Represents a sequence number in a WAL file.
+// Represents a sequence number in a WAL file.
 typedef uint64_t SequenceNumber;
 
 }  //  namespace leveldb

include/leveldb/universal_compaction.h

@@ -0,0 +1,57 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H
+#define STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H
+
+#include <stddef.h>
+#include <string>
+#include <memory>
+#include <vector>
+#include <stdint.h>
+#include <climits>
+#include "leveldb/slice.h"
+#include "leveldb/statistics.h"
+
+namespace leveldb {
+
+//
+// Algorithm used to make a compaction request stop picking new files
+// into a single compaction run
+//
+enum CompactionStopStyle {
+  kCompactionStopStyleSimilarSize, // pick files of similar size
+  kCompactionStopStyleTotalSize    // total size of picked files > next file
+};
+
+class CompactionOptionsUniversal {
+ public:
+
+  // Percentage flexibilty while comparing file size. If the candidate file(s)
+  // size is 1% smaller than the next file's size, then include next file into
+  // this candidate set. // Default: 1
+  unsigned int size_ratio;
+
+  // The minimum number of files in a single compaction run. Default: 2
+  unsigned int min_merge_width;
+
+  // The maximum number of files in a single compaction run. Default: INT_MAX
+  unsigned int max_merge_width;
+
+  // The algorithm used to stop picking files into a single compaction run
+  // Default: kCompactionStopStyleTotalSize
+  CompactionStopStyle stop_style;
+
+  // Default set of parameters
+  CompactionOptionsUniversal() :
+    size_ratio(1),
+    min_merge_width(2),
+    max_merge_width(UINT_MAX),
+    stop_style(kCompactionStopStyleTotalSize) {
+  }
+};
+
+}  // namespace leveldb
+
+#endif  // STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H

include/leveldb/write_batch.h

@@ -36,9 +36,24 @@ class WriteBatch {
   // Store the mapping "key->value" in the database.
   void Put(const Slice& key, const Slice& value);
 
+  // Merge "value" with the existing value of "key" in the database.
+  // "key->merge(existing, value)"
+  void Merge(const Slice& key, const Slice& value);
+
   // If the database contains a mapping for "key", erase it.  Else do nothing.
   void Delete(const Slice& key);
 
+  // Append a blob of arbitrary to the records in this batch. The blob will be
+  // stored in the transaction log but not in any other file. In particular, it
+  // will not be persisted to the SST files. When iterating over this
+  // WriteBatch, WriteBatch::Handler::LogData will be called with the contents
+  // of the blob as it is encountered. Blobs, puts, deletes, and merges will be
+  // encountered in the same order in thich they were inserted.
+  //
+  // Example application: add timestamps to the transaction log for use in
+  // replication.
+  void PutLogData(const Slice& blob);
+
   // Clear all updates buffered in this batch.
   void Clear();
 
@@ -47,15 +62,24 @@ class WriteBatch {
    public:
     virtual ~Handler();
     virtual void Put(const Slice& key, const Slice& value) = 0;
+    // Merge and LogData are not pure virtual. Otherwise, we would break
+    // existing clients of Handler on a source code level. The default
+    // implementation of Merge simply throws a runtime exception.
+    virtual void Merge(const Slice& key, const Slice& value);
+    // The default implementation of LogData does nothing.
+    virtual void LogData(const Slice& blob);
     virtual void Delete(const Slice& key) = 0;
   };
   Status Iterate(Handler* handler) const;
 
-  // Retrive the serialized version of this batch.
+  // Retrieve the serialized version of this batch.
   std::string Data() { return rep_; }
 
+  // Returns the number of updates in the batch
+  int Count() const;
+
   // Constructor with a serialized string object
-  WriteBatch(std::string rep): rep_(rep) {}
+  explicit WriteBatch(std::string rep): rep_(rep) {}
 
  private:
   friend class WriteBatchInternal;

include/utilities/stackable_db.h

@@ -0,0 +1,163 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef LEVELDB_INCLUDE_UTILITIES_STACKABLE_DB_H_
+#define LEVELDB_INCLUDE_UTILITIES_STACKABLE_DB_H_
+
+#include "leveldb/db.h"
+
+namespace leveldb {
+
+// This class contains APIs to stack rocksdb wrappers.Eg. Stack TTL over base d
+class StackableDB : public DB {
+ public:
+  explicit StackableDB(StackableDB* sdb) : sdb_(sdb) {}
+
+  // Returns the DB object that is the lowermost component in the stack of DBs
+  virtual DB* GetRawDB() {
+    return sdb_->GetRawDB();
+  }
+
+  // convert a DB to StackableDB
+  static StackableDB* DBToStackableDB(DB* db) {
+    class NewStackableDB : public StackableDB {
+     public:
+      NewStackableDB(DB* db)
+        : StackableDB(nullptr),
+          db_(db) {}
+
+      DB* GetRawDB() {
+        return db_;
+      }
+
+     private:
+      DB* db_;
+    };
+    return new NewStackableDB(db);
+  }
+
+  virtual Status Put(const WriteOptions& options,
+                     const Slice& key,
+                     const Slice& val) override {
+    return sdb_->Put(options, key, val);
+  }
+
+  virtual Status Get(const ReadOptions& options,
+                     const Slice& key,
+                     std::string* value) override {
+    return sdb_->Get(options, key, value);
+  }
+
+  virtual std::vector<Status> MultiGet(const ReadOptions& options,
+                                       const std::vector<Slice>& keys,
+                                       std::vector<std::string>* values)
+    override {
+      return sdb_->MultiGet(options, keys, values);
+  }
+
+  virtual bool KeyMayExist(const ReadOptions& options,
+                           const Slice& key,
+                           std::string* value,
+                           bool* value_found = nullptr) override {
+    return sdb_->KeyMayExist(options, key, value, value_found);
+  }
+
+  virtual Status Delete(const WriteOptions& wopts, const Slice& key) override {
+    return sdb_->Delete(wopts, key);
+  }
+
+  virtual Status Merge(const WriteOptions& options,
+                       const Slice& key,
+                       const Slice& value) override {
+    return sdb_->Merge(options, key, value);
+  }
+
+
+  virtual Status Write(const WriteOptions& opts, WriteBatch* updates)
+    override {
+      return sdb_->Write(opts, updates);
+  }
+
+  virtual Iterator* NewIterator(const ReadOptions& opts) override {
+    return sdb_->NewIterator(opts);
+  }
+
+  virtual const Snapshot* GetSnapshot() override {
+    return sdb_->GetSnapshot();
+  }
+
+  virtual void ReleaseSnapshot(const Snapshot* snapshot) override {
+    return sdb_->ReleaseSnapshot(snapshot);
+  }
+
+  virtual bool GetProperty(const Slice& property, std::string* value)
+    override {
+      return sdb_->GetProperty(property, value);
+  }
+
+  virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes)
+    override {
+      return sdb_->GetApproximateSizes(r, n, sizes);
+  }
+
+  virtual void CompactRange(const Slice* begin, const Slice* end,
+                            bool reduce_level = false) override {
+    return sdb_->CompactRange(begin, end, reduce_level);
+  }
+
+  virtual int NumberLevels() override {
+    return sdb_->NumberLevels();
+  }
+
+  virtual int MaxMemCompactionLevel() override {
+    return sdb_->MaxMemCompactionLevel();
+  }
+
+  virtual int Level0StopWriteTrigger() override {
+    return sdb_->Level0StopWriteTrigger();
+  }
+
+  virtual Status Flush(const FlushOptions& fopts) override {
+    return sdb_->Flush(fopts);
+  }
+
+  virtual Status DisableFileDeletions() override {
+    return sdb_->DisableFileDeletions();
+  }
+
+  virtual Status EnableFileDeletions() override {
+    return sdb_->EnableFileDeletions();
+  }
+
+  virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs)
+    override {
+      return sdb_->GetLiveFiles(vec, mfs);
+  }
+
+  virtual SequenceNumber GetLatestSequenceNumber() override {
+    return sdb_->GetLatestSequenceNumber();
+  }
+
+  virtual Status GetSortedWalFiles(VectorLogPtr& files) override {
+    return sdb_->GetSortedWalFiles(files);
+  }
+
+  virtual Status DeleteWalFiles(const VectorLogPtr& files)
+    override{
+      return sdb_->DeleteWalFiles(files);
+  }
+
+  virtual Status GetUpdatesSince(SequenceNumber seq_number,
+                                 unique_ptr<TransactionLogIterator>* iter)
+    override {
+      return sdb_->GetUpdatesSince(seq_number, iter);
+  }
+
+ protected:
+  StackableDB* sdb_;
+};
+
+} //  namespace leveldb
+
+#endif  //  LEVELDB_INCLUDE_UTILITIES_STACKABLE_DB_H_

include/utilities/utility_db.h

@@ -0,0 +1,54 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef LEVELDB_INCLUDE_UTILITIES_UTILITY_DB_H_
+#define LEVELDB_INCLUDE_UTILITIES_UTILITY_DB_H_
+
+#include "stackable_db.h"
+
+namespace leveldb {
+
+// This class contains APIs to open leveldb with specific support eg. TTL
+class UtilityDB {
+
+  public:
+    // Open the database with TTL support.
+    //
+    // USE-CASES:
+    // This API should be used to open the db when key-values inserted are
+    //  meant to be removed from the db in a non-strict 'ttl' amount of time
+    //  Therefore, this guarantees that key-values inserted will remain in the
+    //  db for >= ttl amount of time and the db will make efforts to remove the
+    //  key-values as soon as possible after ttl seconds of their insertion.
+    //
+    // BEHAVIOUR:
+    // TTL is accepted in seconds
+    // (int32_t)Timestamp(creation) is suffixed to values in Put internally
+    // Expired TTL values deleted in compaction only:(Timestamp+ttl<time_now)
+    // Get/Iterator may return expired entries(compaction not run on them yet)
+    // Different TTL may be used during different Opens
+    // Example: Open1 at t=0 with ttl=4 and insert k1,k2, close at t=2
+    //          Open2 at t=3 with ttl=5. Now k1,k2 should be deleted at t>=5
+    // read_only=true opens in the usual read-only mode. Compactions will not be
+    //  triggered(neither manual nor automatic), so no expired entries removed
+    //
+    // CONSTRAINTS:
+    // Not specifying/passing or non-positive TTL behaves like TTL = infinity
+    //
+    // !!!WARNING!!!:
+    // Calling DB::Open directly to re-open a db created by this API will get
+    //  corrupt values(timestamp suffixed) and no ttl effect will be there
+    //  during the second Open, so use this API consistently to open the db
+    // Be careful when passing ttl with a small positive value because the
+    //  whole database may be deleted in a small amount of time
+    static Status OpenTtlDB(const Options& options,
+                            const std::string& name,
+                            StackableDB** dbptr,
+                            int32_t ttl = 0,
+                            bool read_only = false);
+};
+
+} //  namespace leveldb
+
+#endif  //  LEVELDB_INCLUDE_UTILITIES_UTILITY_DB_H_