Disable error as warning

The previuos change that use gcc-5 set USE_SSE to wrong flag by mistake. Fix it.

.gitignore

@@ -42,7 +42,6 @@ unity.a
 tags
 rocksdb_dump
 rocksdb_undump
-db_test2
 
 java/out
 java/target

CMakeLists.txt

@@ -185,7 +185,6 @@ set(SOURCES
         table/table_properties.cc
         table/two_level_iterator.cc
         tools/sst_dump_tool.cc
-        tools/db_bench_tool.cc
         tools/dump/db_dump_tool.cc
         util/arena.cc
         util/bloom.cc
@@ -207,7 +206,6 @@ set(SOURCES
         util/filter_policy.cc
         util/hash.cc
         util/histogram.cc
-        util/histogram_windowing.cc
         util/instrumented_mutex.cc
         util/iostats_context.cc
         tools/ldb_cmd.cc
@@ -239,7 +237,6 @@ set(SOURCES
         util/thread_status_updater.cc
         util/thread_status_util.cc
         util/thread_status_util_debug.cc
-        util/transaction_test_util.cc
         util/xfunc.cc
         util/xxhash.cc
         utilities/backupable/backupable_db.cc
@@ -304,7 +301,7 @@ else()
 endif()
 
 set(APPS
-        tools/db_bench.cc
+        db/db_bench.cc
         db/memtablerep_bench.cc
         table/table_reader_bench.cc
         tools/db_stress.cc
@@ -339,8 +336,6 @@ set(TESTS
         db/db_table_properties_test.cc
         db/db_tailing_iter_test.cc
         db/db_test.cc
-        db/db_test2.cc
-        db/db_block_cache_test.cc
         db/db_universal_compaction_test.cc
         db/db_wal_test.cc
         db/dbformat_test.cc
@@ -360,7 +355,6 @@ set(TESTS
         db/perf_context_test.cc
         db/plain_table_db_test.cc
         db/prefix_test.cc
-        db/repair_test.cc
         db/skiplist_test.cc
         db/table_properties_collector_test.cc
         db/version_builder_test.cc
@@ -396,10 +390,8 @@ set(TESTS
         util/file_reader_writer_test.cc
         util/heap_test.cc
         util/histogram_test.cc
-        util/iostats_context_test.cc
         util/memenv_test.cc
         util/mock_env_test.cc
-        util/options_settable_test.cc
         util/options_test.cc
         util/rate_limiter_test.cc
         util/slice_transform_test.cc

DEFAULT_OPTIONS_HISTORY.md

@@ -1,9 +0,0 @@
-# RocksDB default options change log
-## 4.7.0 (4/8/2016)
-* options.write_buffer_size changes from 4MB to 64MB
-* options.target_file_size_base changes from 2MB to 64MB
-* options.max_bytes_for_level_base changes from 10MB to 256MB
-* options.soft_pending_compaction_bytes_limit changes from 0 (disabled) to 64GB
-* options.hard_pending_compaction_bytes_limit changes from 0 (disabled) to 256GB
-* table_cache_numshardbits changes from 4 to 6
-* max_file_opening_threads changes from 1 to 16 

HISTORY.md

@@ -1,18 +1,9 @@
 # Rocksdb Change Log
-## 4.7.0 (4/8/2016)
-### Public API Change
-* rename options compaction_measure_io_stats to report_bg_io_stats and include flush too.
-* Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.
+## Unreleased
 
-## 4.6.0 (3/10/2016)
-### Public API Changes
-* Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier.
-* Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signiture of Cache::Insert() is updated accordingly.
-* Tickers [NUMBER_DB_NEXT, NUMBER_DB_PREV, NUMBER_DB_NEXT_FOUND, NUMBER_DB_PREV_FOUND, ITER_BYTES_READ] are not updated immediately. The are updated when the Iterator is deleted.  
-* Add monotonically increasing counter (DB property "rocksdb.current-super-version-number") that increments upon any change to the LSM tree.
-### New Features
-* Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification.
-* Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned"
+## 4.5.1 (3/25/2016)
+### Bug Fixes
+* Fix failures caused by the destorying order of singleton objects.
 
 ## 4.5.0 (2/5/2016)
 ### Public API Changes

LANGUAGE-BINDINGS.md

@@ -1,11 +0,0 @@
-This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it.
-
-* Java - https://github.com/facebook/rocksdb/tree/master/java
-* Python - http://pyrocksdb.readthedocs.org/en/latest/
-* Perl - https://metacpan.org/pod/RocksDB
-* Node.js - https://npmjs.org/package/rocksdb
-* Go - https://github.com/tecbot/gorocksdb
-* Ruby - http://rubygems.org/gems/rocksdb-ruby
-* Haskell - https://hackage.haskell.org/package/rocksdb-haskell
-* PHP - https://github.com/Photonios/rocksdb-php
-* C# - https://github.com/warrenfalk/rocksdb-sharp

LICENSE

@@ -2,7 +2,7 @@ BSD License
 
 For rocksdb software
 
-Copyright (c) 2011-present, Facebook, Inc.
+Copyright (c) 2014, Facebook, Inc.
 All rights reserved.
 ---------------------------------------------------------------------
 

Makefile

@@ -84,11 +84,12 @@ endif
 # compile with -O2 if debug level is not 2
 ifneq ($(DEBUG_LEVEL), 2)
 OPT += -O2 -fno-omit-frame-pointer
-# Skip for archs that don't support -momit-leaf-frame-pointer
-ifeq (,$(shell $(CXX) -fsyntax-only -momit-leaf-frame-pointer -xc /dev/null 2>&1))
+ifeq (,$(findstring ppc64,$(MACHINE))) # ppc64[le] doesn't support -momit-leaf-frame-pointer
+ifneq ($(MACHINE),armv7l)
 OPT += -momit-leaf-frame-pointer
 endif
 endif
+endif
 
 # if we're compiling for release, compile without debug code (-DNDEBUG) and
 # don't treat warnings as errors
@@ -174,15 +175,6 @@ else
 	pg = -pg
 endif
 
-# USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
-ifdef COMPILE_WITH_UBSAN
-	DISABLE_JEMALLOC=1
-	EXEC_LDFLAGS += -fsanitize=undefined
-	PLATFORM_CCFLAGS += -fsanitize=undefined
-	PLATFORM_CXXFLAGS += -fsanitize=undefined
-endif
-
-
 ifndef DISABLE_JEMALLOC
 	EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
@@ -246,12 +238,8 @@ VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)
 
 VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full
 
-BENCHTOOLOBJECTS = $(BENCH_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
-
 TESTS = \
 	db_test \
-	db_test2 \
-	db_block_cache_test \
 	db_iter_test \
 	db_log_iter_test \
 	db_compaction_filter_test \
@@ -326,7 +314,6 @@ TESTS = \
 	rate_limiter_test \
 	delete_scheduler_test \
 	options_test \
-	options_settable_test \
 	options_util_test \
 	event_logger_test \
 	cuckoo_table_builder_test \
@@ -347,8 +334,7 @@ TESTS = \
 	compact_on_deletion_collector_test \
 	compaction_job_stats_test \
 	transaction_test \
-	ldb_cmd_test \
-	iostats_context_test
+	ldb_cmd_test
 
 SUBSET :=  $(shell echo $(TESTS) |sed s/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/)
 
@@ -606,7 +592,7 @@ ldb_tests: ldb
 crash_test: whitebox_crash_test blackbox_crash_test
 
 blackbox_crash_test: db_stress
-	python -u tools/db_crashtest.py --simple blackbox
+	python -u tools/db_crashtest.py --simple blackbox 
 	python -u tools/db_crashtest.py blackbox
 
 whitebox_crash_test: db_stress
@@ -623,18 +609,8 @@ asan_crash_test:
 	COMPILE_WITH_ASAN=1 $(MAKE) crash_test
 	$(MAKE) clean
 
-ubsan_check:
-	$(MAKE) clean
-	COMPILE_WITH_UBSAN=1 $(MAKE) check -j32
-	$(MAKE) clean
-
-ubsan_crash_test:
-	$(MAKE) clean
-	COMPILE_WITH_UBSAN=1 $(MAKE) crash_test
-	$(MAKE) clean
-
 valgrind_check: $(TESTS)
-	for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \
+	for t in $(filter-out %skiplist_test,$(TESTS)); do \
 		$(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \
 		ret_code=$$?; \
 		if [ $$ret_code -ne 0 ]; then \
@@ -664,6 +640,8 @@ parloop:
 	exit $$ret_bad;
 endif
 
+all_tests:=$(shell $(test_names))
+
 parallel_check: $(TESTS)
 	$(AM_V_GEN)if test "$(J)" > 1                                  \
 	    && (parallel --gnu --help 2>/dev/null) |                    \
@@ -677,7 +655,7 @@ parallel_check: $(TESTS)
 	echo $(J);\
 	echo Test Dir: $(TMPD); \
         seq $(J) | parallel --gnu 's=$(TMPD)/rdb-{}; rm -rf $$s; mkdir $$s'; \
-	$(MAKE)  PAR_TEST="$(shell $(test_names))" TMPD=$(TMPD) \
+	$(MAKE)  PAR_TEST="$(all_tests)" TMPD=$(TMPD) \
 		J=$(J) db_test=1 parloop; \
 	$(MAKE) PAR_TEST="$(filter-out db_test, $(TESTS))" \
 		TMPD=$(TMPD) J=$(J) db_test=0 parloop;
@@ -733,7 +711,7 @@ $(LIBRARY): $(LIBOBJECTS)
 	$(AM_V_AR)rm -f $@
 	$(AM_V_at)$(AR) $(ARFLAGS) $@ $(LIBOBJECTS)
 
-db_bench: tools/db_bench.o $(BENCHTOOLOBJECTS)
+db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL)
 	$(AM_LINK)
 
 cache_bench: util/cache_bench.o $(LIBOBJECTS) $(TESTUTIL)
@@ -808,12 +786,6 @@ slice_transform_test: util/slice_transform_test.o $(LIBOBJECTS) $(TESTHARNESS)
 db_test: db/db_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-db_test2: db/db_test2.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-db_block_cache_test: db/db_block_cache_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_log_iter_test: db/db_log_iter_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1021,9 +993,6 @@ compact_files_test: db/compact_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
 options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-options_settable_test: util/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1066,18 +1035,12 @@ transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TEST
 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 	$(AM_LINK)
 
-repair_test: db/repair_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 ldb_cmd_test: tools/ldb_cmd_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 ldb: tools/ldb.o $(LIBOBJECTS)
 	$(AM_LINK)
 
-iostats_context_test: util/iostats_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
-
 #-------------------------------------------------
 # make install related stuff
 INSTALL_PATH ?= /usr/local
@@ -1240,7 +1203,7 @@ jdb_bench:
 
 commit_prereq: build_tools/rocksdb-lego-determinator \
                build_tools/precommit_checker.py
-	J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit tsan asan ubsan lite
+	build_tools/precommit_checker.py unit unit_481 clang_unit tsan asan lite
 	$(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;
 
 xfunc:

USERS.md

@@ -52,7 +52,3 @@ Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtub
 
 ## VWO, Wingify
 [VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed.
-
-## quasardb
-[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark. 
-quasardb uses a heavily tuned RocksDB as its persistence layer.

arcanist_util/cpp_linter/cpplint.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+# Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.

build_tools/build_detect_platform

@@ -184,19 +184,17 @@ if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
     # Also don't need any compilation tests if compiling on fbcode
     true
 else
-    if ! test $ROCKSDB_DISABLE_FALLOCATE; then
-        # Test whether fallocate is available
-        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <fcntl.h>
-          #include <linux/falloc.h>
-          int main() {
-      int fd = open("/dev/null", 0);
-      fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
-          }
+    # Test whether fallocate is available
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <fcntl.h>
+      #include <linux/falloc.h>
+      int main() {
+	int fd = open("/dev/null", 0);
+  fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
     fi
 
     # Test whether Snappy library is installed
@@ -360,8 +358,6 @@ elif test -z "$PORTABLE"; then
     # Tune for this POWER processor, treating '+' models as base models
     POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
     COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
-    COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
   else
     COMMON_FLAGS="$COMMON_FLAGS -march=native "
   fi

build_tools/dependencies_4.8.1.sh

@@ -1,16 +1,16 @@
 GCC_BASE=/mnt/gvfs/third-party2/gcc/ebc96bc2fb751b5a0300b8d91a95bdf24ac1d88b/4.8.1/centos6-native/cc6c9dc
-CLANG_BASE=/mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/9d9ecb9/
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/f97108c12512b3b0789ac4515d836bdb1eae1142/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f5484f168c0e4d19823d41df052c5870c6e575a4/2.17/gcc-4.8.1-glibc-2.17/99df8fc
+CLANG_BASE=/mnt/vol/engshare/fbcode/third-party2/clang/3.7.1/centos6-native/9d9ecb9/
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/5710d6a0fb0d12820aac0bffcd7fcb8646e7fff7/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/0600c95b31226b5e535614c590677d87c62d8016/2.17/gcc-4.8.1-glibc-2.17/99df8fc
 SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
 ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
 BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
 LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.8.1-glibc-2.17/c3f970a
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/cb6c4880fcb4fee471574ba6af63a3882155a16a/0.5.1/gcc-4.8.1-glibc-2.17/c3f970a
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/69d56740ffb89d8bc81ded8ec428c01a813ea948/0.4.7/gcc-4.8.1-glibc-2.17/c3f970a
 GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/d9acac8a9a2c9378eb696e22ffa8bd0e97d9206b/master/gcc-4.8.1-glibc-2.17/8d31e51
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/5f0be745ddc0f86f22c8c8bb64b6b1163c93df91/4.0.3/gcc-4.8.1-glibc-2.17/8d31e51
 NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/41bfa3759ce52c071f5fd547ec9ecd2522929f0a/trunk/gcc-4.8.1-glibc-2.17/675d945
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/121f1a75c4414683aea8c70b761bfaf187f7c1a3/trunk/gcc-4.8.1-glibc-2.17/675d945
 KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.8.1-glibc-2.17/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/71454c53fffcb716a0beb9a90047aff7fb5c984a/2.26/centos6-native/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/75670d0d8ef4891fd1ec2a7513ef01cd002c823b/2.25/centos6-native/da39a3e
 VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/af85c56f424cd5edfc2c97588299b44ecdec96bb/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a

build_tools/make_new_version.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-#  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+#  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 #  This source code is licensed under the BSD-style license found in the
 #  LICENSE file in the root directory of this source tree. An additional grant
 #  of patent rights can be found in the PATENTS file in the same directory.
@@ -10,7 +10,7 @@ then
   GIT="git"
 fi
 
-# Print out the colored progress info so that it can be brainlessly
+# Print out the colored progress info so that it can be brainlessly 
 # distinguished by users.
 function title() {
   echo -e "\033[1;32m$*\033[0m"

build_tools/precommit_checker.py

@@ -184,7 +184,7 @@ parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.')
 
 # <test ....>
 parser.add_argument('test', nargs='+',
-                    help='CI test(s) to run. e.g: unit punit asan tsan ubsan')
+                    help='CI test(s) to run. e.g: unit punit asan tsan')
 
 print("Please follow log %s" % Log.LOG_FILE)
 

build_tools/rocksdb-lego-determinator

@@ -67,7 +67,6 @@ ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
 LITE="OPT=-DROCKSDB_LITE"
 TSAN="COMPILE_WITH_TSAN=1"
-UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 PARSER="'parser':'egrep \'Failure|^#|Abort|Expected|Actual|GoogleTestFailure|^==\''"
 
@@ -377,48 +376,6 @@ ASAN_CRASH_TEST_COMMANDS="[
     }
 ]"
 
-#
-# RocksDB test under undefined behavior sanitizer
-#
-UBSAN_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb Unit Test under UBSAN',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Test RocksDB debug under UBSAN',
-                'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make J=1 ubsan_check',
-                'user':'root',
-                $PARSER
-            }
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB crash testing under udnefined behavior sanitizer
-#
-UBSAN_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb crash test under UBSAN',
-        'oncall':'$ONCALL',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug ubsan_crash_test',
-                'timeout': 86400,
-                'shell':'$SHM $DEBUG make J=1 ubsan_crash_test',
-                'user':'root',
-                $PARSER
-            },
-        ],
-        $REPORT
-    }
-]"
-
 #
 # RocksDB unit test under valgrind
 #
@@ -505,9 +462,9 @@ run_format_compatible()
   if [ -e "build_detect_platform" ]
   then
     sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
-    rm -rf build_detect_platform
+    rm -rf build_detect_platform 
     cp $TEST_TMPDIR/temp_build_file build_detect_platform
-    chmod +x build_detect_platform
+    chmod +x build_detect_platform 
   fi
 
   make ldb -j32
@@ -675,12 +632,6 @@ case $1 in
   asan_crash)
     echo $ASAN_CRASH_TEST_COMMANDS
     ;;
-  ubsan)
-    echo $UBSAN_TEST_COMMANDS
-    ;;
-  ubsan_crash)
-    echo $UBSAN_CRASH_TEST_COMMANDS
-    ;;
   valgrind)
     echo $VALGRIND_TEST_COMMANDS
     ;;

build_tools/run_ci_db_test.ps1

@@ -9,7 +9,7 @@ Param(
   [string]$WorkFolder = "",  # Direct tests to use that folder
   [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
   [string]$Exclude = "", # Expect a comma separated list, no spaces
-  [string]$Run = "db_test",  # Run db_test|db_test2|tests|testname1,testname2...
+  [string]$Run = "db_test",  # Run db_test|tests|testname1,testname2...
    # Number of async tasks that would run concurrently. Recommend a number below 64.
    # However, CPU utlization really depends on the storage media. Recommend ram based disk.
    # a value of 1 will run everything serially
@@ -46,25 +46,15 @@ if($WorkFolder -eq "") {
   $Env:TEST_TMPDIR = $WorkFolder
 }
 
-Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
-
 # Use JEMALLOC executables
-if($Run -ceq "db_test" -or
-   $Run -ceq "db_test2" ) {
-
-   $file_name = $Run
-
-   if($EnableJE) {
-     $file_name += "_je"
-   }
-
-   $file_name += ".exe"
-
-   $db_test = -Join ($BinariesFolder, $file_name)
-
-   Write-Output "Binaries: $BinariesFolder db_test: $db_test"
+if($EnableJE) {
+    $db_test = -Join ($BinariesFolder, "db_test_je.exe")
+} else {
+    $db_test = -Join ($BinariesFolder, "db_test.exe")
 }
 
+Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
+Write-Output "Binaries: $BinariesFolder db_test: $db_test"
 
 #Exclusions that we do not want to run
 $ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
@@ -163,8 +153,7 @@ function Discover-TestBinaries([string]$Pattern, $HashTable) {
 
 $TestsToRun = [ordered]@{}
 
-if($Run -ceq "db_test" -or
-   $Run -ceq "db_test2") {
+if($Run -ceq "db_test") {
     Normalize-DbTests -HashTable $TestsToRun
 } elseif($Run -ceq "tests") {
     if($EnableJE) {
@@ -236,8 +225,7 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
             Write-Host "Starting $k"
             $log_path = ($TestToLog.$k)
 
-            if($Run -ceq "db_test" -or
-               $Run -ceq "db_test2") {
+            if($Run -ceq "db_test") {
               $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
             } else {
               [string]$Exe =  -Join ($BinariesFolder, $k)

build_tools/update_dependencies.sh

@@ -65,7 +65,7 @@ echo "Writing dependencies to $OUTPUT"
 
 # Compilers locations
 GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.9.x/centos6-native/*/`
-CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
 
 log_variable GCC_BASE
 log_variable CLANG_BASE
@@ -102,7 +102,7 @@ echo "Writing 4.8.1 dependencies to $OUTPUT"
 
 # Compilers locations
 GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/`
-CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
 
 log_variable GCC_BASE
 log_variable CLANG_BASE

db/auto_roll_logger.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/auto_roll_logger.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -14,7 +14,6 @@
 #include "port/port.h"
 #include "port/util_logger.h"
 #include "util/sync_point.h"
-#include "util/mutexlock.h"
 
 namespace rocksdb {
 

db/auto_roll_logger_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/builder.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -41,17 +41,13 @@ TableBuilder* NewTableBuilder(
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
-    WritableFileWriter* file, const CompressionType compression_type,
+    uint32_t column_family_id, WritableFileWriter* file,
+    const CompressionType compression_type,
     const CompressionOptions& compression_opts, const bool skip_filters) {
-  assert((column_family_id ==
-          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
-         column_family_name.empty());
   return ioptions.table_factory->NewTableBuilder(
       TableBuilderOptions(ioptions, internal_comparator,
                           int_tbl_prop_collector_factories, compression_type,
-                          compression_opts, skip_filters,
-                          column_family_name),
+                          compression_opts, skip_filters),
       column_family_id, file);
 }
 
@@ -62,16 +58,12 @@ Status BuildTable(
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
-    std::vector<SequenceNumber> snapshots,
+    uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
     const CompressionType compression,
     const CompressionOptions& compression_opts, bool paranoid_file_checks,
     InternalStats* internal_stats, const Env::IOPriority io_priority,
-    TableProperties* table_properties, int level) {
-  assert((column_family_id ==
-          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
-         column_family_name.empty());
+    TableProperties* table_properties) {
   // Reports the IOStats for flush for every following bytes.
   const size_t kReportFlushIOStatsEvery = 1048576;
   Status s;
@@ -95,8 +87,7 @@ Status BuildTable(
 
       builder = NewTableBuilder(
           ioptions, internal_comparator, int_tbl_prop_collector_factories,
-          column_family_id, column_family_name, file_writer.get(), compression,
-          compression_opts);
+          column_family_id, file_writer.get(), compression, compression_opts);
     }
 
     MergeHelper merge(env, internal_comparator.user_comparator(),
@@ -158,8 +149,7 @@ Status BuildTable(
           ReadOptions(), env_options, internal_comparator, meta->fd, nullptr,
           (internal_stats == nullptr) ? nullptr
                                       : internal_stats->GetFileReadHist(0),
-          false /* for_compaction */, nullptr /* arena */,
-          false /* skip_filter */, level));
+          false));
       s = it->status();
       if (s.ok() && paranoid_file_checks) {
         for (it->SeekToFirst(); it->Valid(); it->Next()) {

db/builder.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -33,16 +33,13 @@ class WritableFileWriter;
 class InternalStats;
 class InternalIterator;
 
-// @param column_family_name Name of the column family that is also identified
-//    by column_family_id, or empty string if unknown. It must outlive the
-//    TableBuilder returned by this function.
 TableBuilder* NewTableBuilder(
     const ImmutableCFOptions& options,
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
-    WritableFileWriter* file, const CompressionType compression_type,
+    uint32_t column_family_id, WritableFileWriter* file,
+    const CompressionType compression_type,
     const CompressionOptions& compression_opts,
     const bool skip_filters = false);
 
@@ -51,9 +48,6 @@ TableBuilder* NewTableBuilder(
 // *meta will be filled with metadata about the generated table.
 // If no data is present in *iter, meta->file_size will be set to
 // zero, and no Table file will be produced.
-//
-// @param column_family_name Name of the column family that is also identified
-//    by column_family_id, or empty string if unknown.
 extern Status BuildTable(
     const std::string& dbname, Env* env, const ImmutableCFOptions& options,
     const EnvOptions& env_options, TableCache* table_cache,
@@ -61,13 +55,12 @@ extern Status BuildTable(
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
-    std::vector<SequenceNumber> snapshots,
+    uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
     const CompressionType compression,
     const CompressionOptions& compression_opts, bool paranoid_file_checks,
     InternalStats* internal_stats,
     const Env::IOPriority io_priority = Env::IO_HIGH,
-    TableProperties* table_properties = nullptr, int level = -1);
+    TableProperties* table_properties = nullptr);
 
 }  // namespace rocksdb

db/c.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1288,11 +1288,6 @@ void rocksdb_block_based_options_set_cache_index_and_filter_blocks(
   options->rep.cache_index_and_filter_blocks = v;
 }
 
-void rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(
-    rocksdb_block_based_table_options_t* options, unsigned char v) {
-  options->rep.pin_l0_filter_and_index_blocks_in_cache = v;
-}
-
 void rocksdb_block_based_options_set_skip_table_builder_flush(
     rocksdb_block_based_table_options_t* options, unsigned char v) {
   options->rep.skip_table_builder_flush = v;

db/column_family.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -147,10 +147,13 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
                                     const ColumnFamilyOptions& src) {
   ColumnFamilyOptions result = src;
   result.comparator = icmp;
-  size_t clamp_max = std::conditional<
-      sizeof(size_t) == 4, std::integral_constant<size_t, 0xffffffff>,
-      std::integral_constant<size_t, 64ull << 30>>::type::value;
-  ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, clamp_max);
+#ifdef OS_MACOSX
+  // TODO(icanadi) make write_buffer_size uint64_t instead of size_t
+  ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, ((size_t)1) << 30);
+#else
+  ClipToRange(&result.write_buffer_size,
+              ((size_t)64) << 10, ((size_t)64) << 30);
+#endif
   // if user sets arena_block_size, we trust user to use this value. Otherwise,
   // calculate a proper value from writer_buffer_size;
   if (result.arena_block_size <= 0) {

db/column_family.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -465,8 +465,6 @@ class ColumnFamilySet {
   // Don't call while iterating over ColumnFamilySet
   void FreeDeadColumnFamilies();
 
-  Cache* get_table_cache() { return table_cache_; }
-
  private:
   friend class ColumnFamilyData;
   // helper function that gets called from cfd destructor

db/column_family_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1952,27 +1952,13 @@ TEST_F(ColumnFamilyTest, ReadDroppedColumnFamily) {
     PutRandomData(1, kKeysNum, 100);
     PutRandomData(2, kKeysNum, 100);
 
-    {
-      std::unique_ptr<Iterator> iterator(
-          db_->NewIterator(ReadOptions(), handles_[2]));
-      iterator->SeekToFirst();
-
-      if (iter == 0) {
-        // Drop CF two
-        ASSERT_OK(db_->DropColumnFamily(handles_[2]));
-      } else {
-        // delete CF two
-        delete handles_[2];
-        handles_[2] = nullptr;
-      }
-      // Make sure iterator created can still be used.
-      int count = 0;
-      for (; iterator->Valid(); iterator->Next()) {
-        ASSERT_OK(iterator->status());
-        ++count;
-      }
-      ASSERT_OK(iterator->status());
-      ASSERT_EQ(count, kKeysNum);
+    if (iter == 0) {
+      // Drop CF two
+      ASSERT_OK(db_->DropColumnFamily(handles_[2]));
+    } else {
+      // delete CF two
+      delete handles_[2];
+      handles_[2] = nullptr;
     }
 
     // Add bunch more data to other CFs
@@ -2556,36 +2542,6 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
   ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
 }
-
-TEST_F(ColumnFamilyTest, LogSyncConflictFlush) {
-  Open();
-  CreateColumnFamiliesAndReopen({"one", "two"});
-
-  Put(0, "", "");
-  Put(1, "foo", "bar");
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"DBImpl::SyncWAL:BeforeMarkLogsSynced:1",
-        "ColumnFamilyTest::LogSyncConflictFlush:1"},
-       {"ColumnFamilyTest::LogSyncConflictFlush:2",
-        "DBImpl::SyncWAL:BeforeMarkLogsSynced:2"}});
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  std::thread thread([&] { db_->SyncWAL(); });
-
-  TEST_SYNC_POINT("ColumnFamilyTest::LogSyncConflictFlush:1");
-  Flush(1);
-  Put(1, "foo", "bar");
-  Flush(1);
-
-  TEST_SYNC_POINT("ColumnFamilyTest::LogSyncConflictFlush:2");
-
-  thread.join();
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  Close();
-}
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/compact_files_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/compacted_db_impl.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/compacted_db_impl.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/compaction.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -162,6 +162,9 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
       deletion_compaction_(_deletion_compaction),
       inputs_(std::move(_inputs)),
       grandparents_(std::move(_grandparents)),
+      grandparent_index_(0),
+      seen_key_(false),
+      overlapped_bytes_(0),
       score_(_score),
       bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
       is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
@@ -287,6 +290,32 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
   return true;
 }
 
+bool Compaction::ShouldStopBefore(const Slice& internal_key) {
+  // Scan to find earliest grandparent file that contains key.
+  const InternalKeyComparator* icmp = &cfd_->internal_comparator();
+  while (grandparent_index_ < grandparents_.size() &&
+      icmp->Compare(internal_key,
+                    grandparents_[grandparent_index_]->largest.Encode()) > 0) {
+    if (seen_key_) {
+      overlapped_bytes_ += grandparents_[grandparent_index_]->fd.GetFileSize();
+    }
+    assert(grandparent_index_ + 1 >= grandparents_.size() ||
+           icmp->Compare(grandparents_[grandparent_index_]->largest.Encode(),
+                         grandparents_[grandparent_index_+1]->smallest.Encode())
+                         < 0);
+    grandparent_index_++;
+  }
+  seen_key_ = true;
+
+  if (overlapped_bytes_ > max_grandparent_overlap_bytes_) {
+    // Too much overlap for current output; start new output
+    overlapped_bytes_ = 0;
+    return true;
+  } else {
+    return false;
+  }
+}
+
 // Mark (or clear) each file that is being compacted
 void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
   for (size_t i = 0; i < num_input_levels(); i++) {
@@ -392,7 +421,7 @@ void Compaction::Summary(char* output, int len) {
   snprintf(output + write, len - write, "]");
 }
 
-uint64_t Compaction::OutputFilePreallocationSize() const {
+uint64_t Compaction::OutputFilePreallocationSize() {
   uint64_t preallocation_size = 0;
 
   if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||

db/compaction.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -104,7 +104,7 @@ class Compaction {
   }
 
   // Returns the LevelFilesBrief of the specified compaction input level.
-  const LevelFilesBrief* input_levels(size_t compaction_input_level) const {
+  LevelFilesBrief* input_levels(size_t compaction_input_level) {
     return &input_levels_[compaction_input_level];
   }
 
@@ -132,10 +132,12 @@ class Compaction {
   bool KeyNotExistsBeyondOutputLevel(const Slice& user_key,
                                      std::vector<size_t>* level_ptrs) const;
 
+  // Returns true iff we should stop building the current output
+  // before processing "internal_key".
+  bool ShouldStopBefore(const Slice& internal_key);
+
   // Clear all files to indicate that they are not being compacted
   // Delete this compaction from the list of running compactions.
-  //
-  // Requirement: DB mutex held
   void ReleaseCompactionFiles(Status status);
 
   // Returns the summary of the compaction in "output" with maximum "len"
@@ -147,13 +149,13 @@ class Compaction {
   double score() const { return score_; }
 
   // Is this compaction creating a file in the bottom most level?
-  bool bottommost_level() const { return bottommost_level_; }
+  bool bottommost_level() { return bottommost_level_; }
 
   // Does this compaction include all sst files?
-  bool is_full_compaction() const { return is_full_compaction_; }
+  bool is_full_compaction() { return is_full_compaction_; }
 
   // Was this compaction triggered manually by the client?
-  bool is_manual_compaction() const { return is_manual_compaction_; }
+  bool is_manual_compaction() { return is_manual_compaction_; }
 
   // Used when allow_trivial_move option is set in
   // Universal compaction. If all the input files are
@@ -166,21 +168,19 @@ class Compaction {
   // Used when allow_trivial_move option is set in
   // Universal compaction. Returns true, if the input files
   // are non-overlapping and can be trivially moved.
-  bool is_trivial_move() const { return is_trivial_move_; }
+  bool is_trivial_move() { return is_trivial_move_; }
 
   // How many total levels are there?
   int number_levels() const { return number_levels_; }
 
   // Return the MutableCFOptions that should be used throughout the compaction
   // procedure
-  const MutableCFOptions* mutable_cf_options() const {
-    return &mutable_cf_options_;
-  }
+  const MutableCFOptions* mutable_cf_options() { return &mutable_cf_options_; }
 
   // Returns the size in bytes that the output file should be preallocated to.
   // In level compaction, that is max_file_size_. In universal compaction, that
   // is the sum of all input file sizes.
-  uint64_t OutputFilePreallocationSize() const;
+  uint64_t OutputFilePreallocationSize();
 
   void SetInputVersion(Version* input_version);
 
@@ -223,14 +223,6 @@ class Compaction {
 
   CompactionReason compaction_reason() { return compaction_reason_; }
 
-  const std::vector<FileMetaData*>& grandparents() const {
-    return grandparents_;
-  }
-
-  uint64_t max_grandparent_overlap_bytes() const {
-    return max_grandparent_overlap_bytes_;
-  }
-
  private:
   // mark (or clear) all files that are being compacted
   void MarkFilesBeingCompacted(bool mark_as_compacted);
@@ -274,6 +266,10 @@ class Compaction {
   // State used to check for number of of overlapping grandparent files
   // (grandparent == "output_level_ + 1")
   std::vector<FileMetaData*> grandparents_;
+  size_t grandparent_index_;   // Index in grandparent_starts_
+  bool seen_key_;              // Some output key has been seen
+  uint64_t overlapped_bytes_;  // Bytes of overlap between current output
+                               // and grandparent files
   const double score_;         // score that was used to pick this compaction.
 
   // Is this compaction creating a file in the bottom most level?

db/compaction_iterator.cc

@@ -1,6 +1,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -14,7 +14,7 @@ CompactionIterator::CompactionIterator(
     InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
     SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
     SequenceNumber earliest_write_conflict_snapshot, Env* env,
-    bool expect_valid_internal_key, const Compaction* compaction,
+    bool expect_valid_internal_key, Compaction* compaction,
     const CompactionFilter* compaction_filter, LogBuffer* log_buffer)
     : input_(input),
       cmp_(cmp),

db/compaction_iterator.h

@@ -1,6 +1,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -42,7 +42,7 @@ class CompactionIterator {
                      std::vector<SequenceNumber>* snapshots,
                      SequenceNumber earliest_write_conflict_snapshot, Env* env,
                      bool expect_valid_internal_key,
-                     const Compaction* compaction = nullptr,
+                     Compaction* compaction = nullptr,
                      const CompactionFilter* compaction_filter = nullptr,
                      LogBuffer* log_buffer = nullptr);
 
@@ -92,7 +92,7 @@ class CompactionIterator {
   const SequenceNumber earliest_write_conflict_snapshot_;
   Env* env_;
   bool expect_valid_internal_key_;
-  const Compaction* compaction_;
+  Compaction* compaction_;
   const CompactionFilter* compaction_filter_;
   LogBuffer* log_buffer_;
   bool bottommost_level_;

db/compaction_iterator_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/compaction_job.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -63,7 +63,7 @@ namespace rocksdb {
 
 // Maintains state for each sub-compaction
 struct CompactionJob::SubcompactionState {
-  const Compaction* compaction;
+  Compaction* compaction;
   std::unique_ptr<CompactionIterator> c_iter;
 
   // The boundaries of the key-range this compaction is interested in. No two
@@ -104,14 +104,6 @@ struct CompactionJob::SubcompactionState {
   uint64_t num_output_records;
   CompactionJobStats compaction_job_stats;
   uint64_t approx_size;
-  // An index that used to speed up Compaction::ShouldStopBefore().
-  size_t grandparent_index = 0;
-  // The number of bytes overlapping between the current output and
-  // grandparent files used in Compaction::ShouldStopBefore().
-  uint64_t overlapped_bytes = 0;
-  // A flag determine whether the key has been seen in
-  // Compaction::ShouldStopBefore()
-  bool seen_key = false;
 
   SubcompactionState(Compaction* c, Slice* _start, Slice* _end,
                      uint64_t size = 0)
@@ -123,10 +115,7 @@ struct CompactionJob::SubcompactionState {
         total_bytes(0),
         num_input_records(0),
         num_output_records(0),
-        approx_size(size),
-        grandparent_index(0),
-        overlapped_bytes(0),
-        seen_key(false) {
+        approx_size(size) {
     assert(compaction != nullptr);
   }
 
@@ -145,9 +134,6 @@ struct CompactionJob::SubcompactionState {
     num_output_records = std::move(o.num_output_records);
     compaction_job_stats = std::move(o.compaction_job_stats);
     approx_size = std::move(o.approx_size);
-    grandparent_index = std::move(o.grandparent_index);
-    overlapped_bytes = std::move(o.overlapped_bytes);
-    seen_key = std::move(o.seen_key);
     return *this;
   }
 
@@ -155,38 +141,6 @@ struct CompactionJob::SubcompactionState {
   SubcompactionState(const SubcompactionState&) = delete;
 
   SubcompactionState& operator=(const SubcompactionState&) = delete;
-
-  // Returns true iff we should stop building the current output
-  // before processing "internal_key".
-  bool ShouldStopBefore(const Slice& internal_key) {
-    const InternalKeyComparator* icmp =
-        &compaction->column_family_data()->internal_comparator();
-    const std::vector<FileMetaData*>& grandparents = compaction->grandparents();
-
-    // Scan to find earliest grandparent file that contains key.
-    while (grandparent_index < grandparents.size() &&
-           icmp->Compare(internal_key,
-                         grandparents[grandparent_index]->largest.Encode()) >
-               0) {
-      if (seen_key) {
-        overlapped_bytes += grandparents[grandparent_index]->fd.GetFileSize();
-      }
-      assert(grandparent_index + 1 >= grandparents.size() ||
-             icmp->Compare(
-                 grandparents[grandparent_index]->largest.Encode(),
-                 grandparents[grandparent_index + 1]->smallest.Encode()) < 0);
-      grandparent_index++;
-    }
-    seen_key = true;
-
-    if (overlapped_bytes > compaction->max_grandparent_overlap_bytes()) {
-      // Too much overlap for current output; start new output
-      overlapped_bytes = 0;
-      return true;
-    }
-
-    return false;
-  }
 };
 
 // Maintains state for the entire compaction
@@ -258,7 +212,6 @@ CompactionJob::CompactionJob(
     const EnvOptions& env_options, VersionSet* versions,
     std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
     Directory* db_directory, Directory* output_directory, Statistics* stats,
-    InstrumentedMutex* db_mutex, Status* db_bg_error,
     std::vector<SequenceNumber> existing_snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
     std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
@@ -278,8 +231,6 @@ CompactionJob::CompactionJob(
       db_directory_(db_directory),
       output_directory_(output_directory),
       stats_(stats),
-      db_mutex_(db_mutex),
-      db_bg_error_(db_bg_error),
       existing_snapshots_(std::move(existing_snapshots)),
       earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
       table_cache_(std::move(table_cache)),
@@ -409,7 +360,7 @@ void CompactionJob::GenSubcompactionBoundaries() {
       size_t num_files = flevel->num_files;
 
       if (num_files == 0) {
-        continue;
+        break;
       }
 
       if (lvl == 0) {
@@ -548,11 +499,16 @@ Status CompactionJob::Run() {
   }
 
   TablePropertiesCollection tp;
+  auto sfm =
+      static_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
   for (const auto& state : compact_->sub_compact_states) {
     for (const auto& output : state.outputs) {
       auto fn = TableFileName(db_options_.db_paths, output.meta.fd.GetNumber(),
                               output.meta.fd.GetPathId());
       tp[fn] = output.table_properties;
+      if (sfm && output.meta.fd.GetPathId() == 0) {
+        sfm->OnAddFile(fn);
+      }
     }
   }
   compact_->compaction->SetOutputTableProperties(std::move(tp));
@@ -568,17 +524,18 @@ Status CompactionJob::Run() {
   return status;
 }
 
-Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
+Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
+                              InstrumentedMutex* db_mutex) {
   AutoThreadOperationStageUpdater stage_updater(
       ThreadStatus::STAGE_COMPACTION_INSTALL);
-  db_mutex_->AssertHeld();
+  db_mutex->AssertHeld();
   Status status = compact_->status;
   ColumnFamilyData* cfd = compact_->compaction->column_family_data();
   cfd->internal_stats()->AddCompactionStats(
       compact_->compaction->output_level(), compaction_stats_);
 
   if (status.ok()) {
-    status = InstallCompactionResults(mutable_cf_options);
+    status = InstallCompactionResults(mutable_cf_options, db_mutex);
   }
   VersionStorageInfo::LevelSummaryStorage tmp;
   auto vstorage = cfd->current()->storage_info();
@@ -715,7 +672,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
     if (end != nullptr &&
         cfd->user_comparator()->Compare(c_iter->user_key(), *end) >= 0) {
       break;
-    } else if (sub_compact->ShouldStopBefore(key) &&
+    } else if (sub_compact->compaction->ShouldStopBefore(key) &&
                sub_compact->builder != nullptr) {
       status = FinishCompactionOutputFile(input->status(), sub_compact);
       if (!status.ok()) {
@@ -904,33 +861,13 @@ Status CompactionJob::FinishCompactionOutputFile(
           event_logger_, cfd->ioptions()->listeners, meta->fd, info);
     }
   }
-
-  // Report new file to SstFileManagerImpl
-  auto sfm =
-      static_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
-  if (sfm && meta->fd.GetPathId() == 0) {
-    ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
-    auto fn = TableFileName(cfd->ioptions()->db_paths, meta->fd.GetNumber(),
-                            meta->fd.GetPathId());
-    sfm->OnAddFile(fn);
-    if (sfm->IsMaxAllowedSpaceReached()) {
-      InstrumentedMutexLock l(db_mutex_);
-      if (db_bg_error_->ok()) {
-        s = Status::IOError("Max allowed space was reached");
-        *db_bg_error_ = s;
-        TEST_SYNC_POINT(
-            "CompactionJob::FinishCompactionOutputFile:MaxAllowedSpaceReached");
-      }
-    }
-  }
-
   sub_compact->builder.reset();
   return s;
 }
 
 Status CompactionJob::InstallCompactionResults(
-    const MutableCFOptions& mutable_cf_options) {
-  db_mutex_->AssertHeld();
+    const MutableCFOptions& mutable_cf_options, InstrumentedMutex* db_mutex) {
+  db_mutex->AssertHeld();
 
   auto* compaction = compact_->compaction;
   // paranoia: verify that the files that we started with
@@ -965,7 +902,7 @@ Status CompactionJob::InstallCompactionResults(
   }
   return versions_->LogAndApply(compaction->column_family_data(),
                                 mutable_cf_options, compaction->edit(),
-                                db_mutex_, db_directory_);
+                                db_mutex, db_directory_);
 }
 
 void CompactionJob::RecordCompactionIOStats() {
@@ -1019,7 +956,7 @@ Status CompactionJob::OpenCompactionOutputFile(
       cfd->ioptions()->optimize_filters_for_hits && bottommost_level_;
   sub_compact->builder.reset(NewTableBuilder(
       *cfd->ioptions(), cfd->internal_comparator(),
-      cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
+      cfd->int_tbl_prop_collector_factories(), cfd->GetID(),
       sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
       cfd->ioptions()->compression_opts, skip_filters));
   LogFlush(db_options_.info_log);

db/compaction_job.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -56,8 +56,7 @@ class CompactionJob {
                 const EnvOptions& env_options, VersionSet* versions,
                 std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
                 Directory* db_directory, Directory* output_directory,
-                Statistics* stats, InstrumentedMutex* db_mutex,
-                Status* db_bg_error,
+                Statistics* stats,
                 std::vector<SequenceNumber> existing_snapshots,
                 SequenceNumber earliest_write_conflict_snapshot,
                 std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
@@ -78,7 +77,8 @@ class CompactionJob {
   Status Run();
 
   // REQUIRED: mutex held
-  Status Install(const MutableCFOptions& mutable_cf_options);
+  Status Install(const MutableCFOptions& mutable_cf_options,
+                 InstrumentedMutex* db_mutex);
 
  private:
   struct SubcompactionState;
@@ -95,7 +95,8 @@ class CompactionJob {
 
   Status FinishCompactionOutputFile(const Status& input_status,
                                     SubcompactionState* sub_compact);
-  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options);
+  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options,
+                                  InstrumentedMutex* db_mutex);
   void RecordCompactionIOStats();
   Status OpenCompactionOutputFile(SubcompactionState* sub_compact);
   void CleanupCompaction();
@@ -129,8 +130,6 @@ class CompactionJob {
   Directory* db_directory_;
   Directory* output_directory_;
   Statistics* stats_;
-  InstrumentedMutex* db_mutex_;
-  Status* db_bg_error_;
   // If there were two snapshots with seq numbers s1 and
   // s2 and s1 < s2, and if we find two instances of a key k1 then lies
   // entirely within s1 and s2, then the earlier version of k1 can be safely

db/compaction_job_stats_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -660,7 +660,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
   options.max_subcompactions = max_subcompactions_;
   options.bytes_per_sync = 512 * 1024;
 
-  options.report_bg_io_stats = true;
+  options.compaction_measure_io_stats = true;
   for (int test = 0; test < 2; ++test) {
     DestroyAndReopen(options);
     CreateAndReopenWithCF({"pikachu"}, options);

db/compaction_job_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -250,9 +250,9 @@ class CompactionJobTest : public testing::Test {
     EventLogger event_logger(db_options_.info_log.get());
     CompactionJob compaction_job(
         0, &compaction, db_options_, env_options_, versions_.get(),
-        &shutting_down_, &log_buffer, nullptr, nullptr, nullptr, &mutex_,
-        &bg_error_, snapshots, earliest_write_conflict_snapshot, table_cache_,
-        &event_logger, false, false, dbname_, &compaction_job_stats_);
+        &shutting_down_, &log_buffer, nullptr, nullptr, nullptr, snapshots,
+        earliest_write_conflict_snapshot, table_cache_, &event_logger, false,
+        false, dbname_, &compaction_job_stats_);
 
     VerifyInitializationOfCompactionJobStats(compaction_job_stats_);
 
@@ -262,7 +262,8 @@ class CompactionJobTest : public testing::Test {
     s = compaction_job.Run();
     ASSERT_OK(s);
     mutex_.Lock();
-    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions()));
+    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions(),
+                                     &mutex_));
     mutex_.Unlock();
 
     if (expected_results.size() == 0) {
@@ -294,7 +295,6 @@ class CompactionJobTest : public testing::Test {
   ColumnFamilyData* cfd_;
   std::unique_ptr<CompactionFilter> compaction_filter_;
   std::shared_ptr<MergeOperator> merge_op_;
-  Status bg_error_;
 };
 
 TEST_F(CompactionJobTest, Simple) {

db/compaction_picker.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -141,8 +141,7 @@ CompactionPicker::~CompactionPicker() {}
 
 // Delete this compaction from the list of running compactions.
 void CompactionPicker::ReleaseCompactionFiles(Compaction* c, Status status) {
-  if (c->start_level() == 0 ||
-      ioptions_.compaction_style == kCompactionStyleUniversal) {
+  if (c->start_level() == 0) {
     level0_compactions_in_progress_.erase(c);
   }
   if (!status.ok()) {
@@ -613,17 +612,6 @@ Compaction* CompactionPicker::CompactRange(
   if (input_level == 0) {
     level0_compactions_in_progress_.insert(compaction);
   }
-
-  // Creating a compaction influences the compaction score because the score
-  // takes running compactions into account (by skipping files that are already
-  // being compacted). Since we just changed compaction score, we recalculate it
-  // here
-  {  // this piece of code recomputes compaction score
-    CompactionOptionsFIFO dummy_compaction_options_fifo;
-    vstorage->ComputeCompactionScore(mutable_cf_options,
-                                     dummy_compaction_options_fifo);
-  }
-
   return compaction;
 }
 

db/compaction_picker.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -83,8 +83,6 @@ class CompactionPicker {
 #endif  // ROCKSDB_LITE
 
   // Free up the files that participated in a compaction
-  //
-  // Requirement: DB mutex held
   void ReleaseCompactionFiles(Compaction* c, Status status);
 
   // Returns true if any one of the specified files are being compacted

db/compaction_picker_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -195,7 +195,6 @@ TEST_F(CompactionPickerTest, LevelMaxScore) {
   NewVersionStorage(6, kCompactionStyleLevel);
   mutable_cf_options_.target_file_size_base = 10000000;
   mutable_cf_options_.target_file_size_multiplier = 10;
-  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
   Add(0, 1U, "150", "200", 1000000000U);
   // Level 1 score 1.2
   Add(1, 66U, "150", "200", 6000000U);
@@ -488,90 +487,6 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
 }
 #endif  // ROCKSDB_LITE
 
-TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.target_file_size_base = 10000000;
-  mutable_cf_options_.target_file_size_multiplier = 10;
-  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
-  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
-
-  Add(2, 6U, "150", "179", 50000000U);
-  Add(2, 7U, "180", "220", 50000000U);
-  Add(2, 8U, "321", "400", 50000000U);  // File not overlapping
-  Add(2, 9U, "721", "800", 50000000U);
-
-  Add(3, 26U, "150", "170", 260000000U);
-  Add(3, 27U, "171", "179", 260000000U);
-  Add(3, 28U, "191", "220", 260000000U);
-  Add(3, 29U, "221", "300", 260000000U);
-  Add(3, 30U, "750", "900", 260000000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  // Pick file 8 because it overlaps with 0 files on level 3.
-  ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.target_file_size_base = 10000000;
-  mutable_cf_options_.target_file_size_multiplier = 10;
-  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
-  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
-
-  Add(2, 6U, "150", "175",
-      60000000U);  // Overlaps with file 26, 27, total size 521M
-  Add(2, 7U, "176", "200", 60000000U);  // Overlaps with file 27, 28, total size
-                                        // 520M, the smalelst overlapping
-  Add(2, 8U, "201", "300",
-      60000000U);  // Overlaps with file 28, 29, total size 521M
-
-  Add(3, 26U, "100", "110", 261000000U);
-  Add(3, 26U, "150", "170", 261000000U);
-  Add(3, 27U, "171", "179", 260000000U);
-  Add(3, 28U, "191", "220", 260000000U);
-  Add(3, 29U, "221", "300", 261000000U);
-  Add(3, 30U, "321", "400", 261000000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  // Picking file 7 because overlapping ratio is the biggest.
-  ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_bytes_for_level_base = 10000000;
-  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
-  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
-
-  // file 7 and 8 over lap with the same file, but file 8 is smaller so
-  // it will be picked.
-  Add(2, 6U, "150", "167", 60000000U);  // Overlaps with file 26, 27
-  Add(2, 7U, "168", "169", 60000000U);  // Overlaps with file 27
-  Add(2, 8U, "201", "300", 61000000U);  // Overlaps with file 28, but the file
-                                        // itself is larger. Should be picked.
-
-  Add(3, 26U, "160", "165", 260000000U);
-  Add(3, 27U, "166", "170", 260000000U);
-  Add(3, 28U, "180", "400", 260000000U);
-  Add(3, 29U, "401", "500", 260000000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  // Picking file 8 because overlapping ratio is the biggest.
-  ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
-}
-
 // This test exhibits the bug where we don't properly reset parent_index in
 // PickCompaction()
 TEST_F(CompactionPickerTest, ParentIndexResetBug) {

db/comparator_db_test.cc

@@ -1,6 +1,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/convenience.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/corruption_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/cuckoo_table_db_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_bench.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -11,7 +11,14 @@
 #define __STDC_FORMAT_MACROS
 #endif
 
-#ifdef GFLAGS
+#ifndef GFLAGS
+#include <cstdio>
+int main() {
+  fprintf(stderr, "Please install gflags to run rocksdb tools\n");
+  return 1;
+}
+#else
+
 #ifdef NUMA
 #include <numa.h>
 #include <numaif.h>
@@ -36,41 +43,39 @@
 
 #include "db/db_impl.h"
 #include "db/version_set.h"
-#include "hdfs/env_hdfs.h"
-#include "port/port.h"
-#include "port/stack_trace.h"
+#include "rocksdb/options.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
-#include "rocksdb/filter_policy.h"
 #include "rocksdb/memtablerep.h"
-#include "rocksdb/options.h"
-#include "rocksdb/perf_context.h"
-#include "rocksdb/rate_limiter.h"
+#include "rocksdb/write_batch.h"
 #include "rocksdb/slice.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/rate_limiter.h"
 #include "rocksdb/slice_transform.h"
+#include "rocksdb/perf_context.h"
 #include "rocksdb/utilities/flashcache.h"
-#include "rocksdb/utilities/optimistic_transaction_db.h"
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
-#include "rocksdb/write_batch.h"
-#include "util/compression.h"
+#include "rocksdb/utilities/optimistic_transaction_db.h"
+#include "port/port.h"
+#include "port/stack_trace.h"
 #include "util/crc32c.h"
+#include "util/compression.h"
 #include "util/histogram.h"
 #include "util/mutexlock.h"
 #include "util/random.h"
-#include "util/statistics.h"
 #include "util/string_util.h"
+#include "util/statistics.h"
 #include "util/testutil.h"
-#include "util/transaction_test_util.h"
 #include "util/xxhash.h"
+#include "hdfs/env_hdfs.h"
 #include "utilities/merge_operators.h"
 
 #ifdef OS_WIN
 #include <io.h>  // open/close
 #endif
 
-namespace {
 using GFLAGS::ParseCommandLineFlags;
 using GFLAGS::RegisterFlagValidator;
 using GFLAGS::SetUsageMessage;
@@ -340,9 +345,6 @@ DEFINE_int64(cache_size, -1, "Number of bytes to use as a cache of uncompressed"
 DEFINE_bool(cache_index_and_filter_blocks, false,
             "Cache index/filter blocks in block cache.");
 
-DEFINE_bool(pin_l0_filter_and_index_blocks_in_cache, false,
-            "Pin index/filter blocks of L0 files in block cache.");
-
 DEFINE_int32(block_size,
              static_cast<int32_t>(rocksdb::BlockBasedTableOptions().block_size),
              "Number of bytes in a block.");
@@ -516,9 +518,10 @@ DEFINE_uint64(transaction_lock_timeout, 100,
               " milliseconds before failing a transaction waiting on a lock");
 #endif  // ROCKSDB_LITE
 
-DEFINE_bool(report_bg_io_stats, false,
+DEFINE_bool(compaction_measure_io_stats, false,
             "Measure times spents on I/Os while in compactions. ");
 
+namespace {
 enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
   assert(ctype);
 
@@ -538,7 +541,7 @@ enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
     return rocksdb::kZSTDNotFinalCompression;
 
   fprintf(stdout, "Cannot parse compression type '%s'\n", ctype);
-  return rocksdb::kSnappyCompression;  // default value
+  return rocksdb::kSnappyCompression; //default value
 }
 
 std::string ColumnFamilyName(size_t i) {
@@ -550,6 +553,7 @@ std::string ColumnFamilyName(size_t i) {
     return std::string(name);
   }
 }
+}  // namespace
 
 DEFINE_string(compression_type, "snappy",
               "Algorithm to use to compress the database");
@@ -760,6 +764,7 @@ enum RepFactory {
   kCuckoo
 };
 
+namespace {
 enum RepFactory StringToRepFactory(const char* ctype) {
   assert(ctype);
 
@@ -777,6 +782,7 @@ enum RepFactory StringToRepFactory(const char* ctype) {
   fprintf(stdout, "Cannot parse memreptable %s\n", ctype);
   return kSkipList;
 }
+}  // namespace
 
 static enum RepFactory FLAGS_rep_factory;
 DEFINE_string(memtablerep, "skip_list", "");
@@ -828,7 +834,6 @@ static const bool FLAGS_deletepercent_dummy __attribute__((unused)) =
 static const bool FLAGS_table_cache_numshardbits_dummy __attribute__((unused)) =
     RegisterFlagValidator(&FLAGS_table_cache_numshardbits,
                           &ValidateTableCacheNumshardbits);
-}  // namespace
 
 namespace rocksdb {
 
@@ -1209,7 +1214,7 @@ class Stats {
   uint64_t bytes_;
   uint64_t last_op_finish_;
   uint64_t last_report_finish_;
-  std::unordered_map<OperationType, std::shared_ptr<HistogramImpl>,
+  std::unordered_map<OperationType, HistogramImpl,
                      std::hash<unsigned char>> hist_;
   std::string message_;
   bool exclude_from_merge_;
@@ -1246,7 +1251,7 @@ class Stats {
     for (auto it = other.hist_.begin(); it != other.hist_.end(); ++it) {
       auto this_it = hist_.find(it->first);
       if (this_it != hist_.end()) {
-        this_it->second->Merge(*(other.hist_.at(it->first)));
+        this_it->second.Merge(other.hist_.at(it->first));
       } else {
         hist_.insert({ it->first, it->second });
       }
@@ -1320,10 +1325,10 @@ class Stats {
 
       if (hist_.find(op_type) == hist_.end())
       {
-        auto hist_temp = std::make_shared<HistogramImpl>();
-        hist_.insert({op_type, std::move(hist_temp)});
+        HistogramImpl hist_temp;
+        hist_.insert({op_type, hist_temp});
       }
-      hist_[op_type]->Add(micros);
+      hist_[op_type].Add(micros);
 
       if (micros > 20000 && !FLAGS_stats_interval) {
         fprintf(stderr, "long op: %" PRIu64 " micros%30s\r", micros, "");
@@ -1456,7 +1461,7 @@ class Stats {
       for (auto it = hist_.begin(); it != hist_.end(); ++it) {
         fprintf(stdout, "Microseconds per %s:\n%s\n",
                 OperationTypeString[it->first].c_str(),
-                it->second->ToString().c_str());
+                it->second.ToString().c_str());
       }
     }
     if (FLAGS_report_file_operations) {
@@ -2244,7 +2249,7 @@ class Benchmark {
       count++;
       thread->stats.FinishedOps(nullptr, nullptr, 1, kOthers);
     }
-    if (ptr == nullptr) exit(1);  // Disable unused variable warning.
+    if (ptr == nullptr) exit(1); // Disable unused variable warning.
   }
 
   void Compress(ThreadState *thread) {
@@ -2514,8 +2519,6 @@ class Benchmark {
       }
       block_based_options.cache_index_and_filter_blocks =
           FLAGS_cache_index_and_filter_blocks;
-      block_based_options.pin_l0_filter_and_index_blocks_in_cache =
-          FLAGS_pin_l0_filter_and_index_blocks_in_cache;
       block_based_options.block_cache = cache_;
       block_based_options.block_cache_compressed = compressed_cache_;
       block_based_options.block_size = FLAGS_block_size;
@@ -2599,7 +2602,7 @@ class Benchmark {
       exit(1);
     }
     options.max_successive_merges = FLAGS_max_successive_merges;
-    options.report_bg_io_stats = FLAGS_report_bg_io_stats;
+    options.compaction_measure_io_stats = FLAGS_compaction_measure_io_stats;
 
     // set universal style compaction configurations, if applicable
     if (FLAGS_universal_size_ratio != 0) {
@@ -3769,22 +3772,18 @@ class Benchmark {
     ReadOptions options(FLAGS_verify_checksum, true);
     Duration duration(FLAGS_duration, readwrites_);
     ReadOptions read_options(FLAGS_verify_checksum, true);
-    uint16_t num_prefix_ranges = static_cast<uint16_t>(FLAGS_transaction_sets);
+    std::string value;
+    DB* db = db_.db;
     uint64_t transactions_done = 0;
+    uint64_t transactions_aborted = 0;
+    Status s;
+    uint64_t num_prefix_ranges = FLAGS_transaction_sets;
 
     if (num_prefix_ranges == 0 || num_prefix_ranges > 9999) {
       fprintf(stderr, "invalid value for transaction_sets\n");
       abort();
     }
 
-    TransactionOptions txn_options;
-    txn_options.lock_timeout = FLAGS_transaction_lock_timeout;
-    txn_options.set_snapshot = FLAGS_transaction_set_snapshot;
-
-    RandomTransactionInserter inserter(&thread->rand, write_options_,
-                                       read_options, FLAGS_num,
-                                       num_prefix_ranges);
-
     if (FLAGS_num_multi_db > 1) {
       fprintf(stderr,
               "Cannot run RandomTransaction benchmark with "
@@ -3793,26 +3792,126 @@ class Benchmark {
     }
 
     while (!duration.Done(1)) {
-      bool success;
+      Transaction* txn = nullptr;
+      WriteBatch* batch = nullptr;
 
-      // RandomTransactionInserter will attempt to insert a key for each
-      // # of FLAGS_transaction_sets
       if (FLAGS_optimistic_transaction_db) {
-        success = inserter.OptimisticTransactionDBInsert(db_.opt_txn_db);
+        txn = db_.opt_txn_db->BeginTransaction(write_options_);
+        assert(txn);
       } else if (FLAGS_transaction_db) {
         TransactionDB* txn_db = reinterpret_cast<TransactionDB*>(db_.db);
-        success = inserter.TransactionDBInsert(txn_db, txn_options);
+
+        TransactionOptions txn_options;
+        txn_options.lock_timeout = FLAGS_transaction_lock_timeout;
+
+        txn = txn_db->BeginTransaction(write_options_, txn_options);
+        assert(txn);
       } else {
-        success = inserter.DBInsert(db_.db);
+        batch = new WriteBatch();
       }
 
-      if (!success) {
-        fprintf(stderr, "Unexpected error: %s\n",
-                inserter.GetLastStatus().ToString().c_str());
-        abort();
+      if (txn && FLAGS_transaction_set_snapshot) {
+        txn->SetSnapshot();
+      }
+
+      // pick a random number to use to increment a key in each set
+      uint64_t incr = (thread->rand.Next() % 100) + 1;
+
+      bool failed = false;
+      // For each set, pick a key at random and increment it
+      for (uint8_t i = 0; i < num_prefix_ranges; i++) {
+        uint64_t int_value;
+        char prefix_buf[5];
+
+        // key format:  [SET#][random#]
+        std::string rand_key = ToString(thread->rand.Next() % FLAGS_num);
+        Slice base_key(rand_key);
+
+        // Pad prefix appropriately so we can iterate over each set
+        snprintf(prefix_buf, sizeof(prefix_buf), "%04d", i + 1);
+        std::string full_key = std::string(prefix_buf) + base_key.ToString();
+        Slice key(full_key);
+
+        if (txn) {
+          s = txn->GetForUpdate(read_options, key, &value);
+        } else {
+          s = db->Get(read_options, key, &value);
+        }
+
+        if (s.ok()) {
+          int_value = std::stoull(value);
+
+          if (int_value == 0 || int_value == ULONG_MAX) {
+            fprintf(stderr, "Get returned unexpected value: %s\n",
+                    value.c_str());
+            abort();
+          }
+        } else if (s.IsNotFound()) {
+          int_value = 0;
+        } else if (!(s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) {
+          fprintf(stderr, "Get returned an unexpected error: %s\n",
+                  s.ToString().c_str());
+          abort();
+        } else {
+          failed = true;
+          break;
+        }
+
+        if (FLAGS_transaction_sleep > 0) {
+          FLAGS_env->SleepForMicroseconds(thread->rand.Next() %
+                                          FLAGS_transaction_sleep);
+        }
+
+        std::string sum = ToString(int_value + incr);
+        if (txn) {
+          s = txn->Put(key, sum);
+          if (!s.ok()) {
+            // Since we did a GetForUpdate, Put should not fail.
+            fprintf(stderr, "Put returned an unexpected error: %s\n",
+                    s.ToString().c_str());
+            abort();
+          }
+        } else {
+          batch->Put(key, sum);
+        }
+      }
+
+      if (txn) {
+        if (failed) {
+          transactions_aborted++;
+          txn->Rollback();
+          s = Status::OK();
+        } else {
+          s = txn->Commit();
+        }
+      } else {
+        s = db->Write(write_options_, batch);
+      }
+
+      if (!s.ok()) {
+        failed = true;
+
+        // Ideally, we'd want to run this stress test with enough concurrency
+        // on a small enough set of keys that we get some failed transactions
+        // due to conflicts.
+        if (FLAGS_optimistic_transaction_db &&
+            (s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) {
+          transactions_aborted++;
+        } else if (FLAGS_transaction_db && s.IsExpired()) {
+          transactions_aborted++;
+        } else {
+          fprintf(stderr, "Unexpected write error: %s\n", s.ToString().c_str());
+          abort();
+        }
+      }
+
+      delete txn;
+      delete batch;
+
+      if (!failed) {
+        thread->stats.FinishedOps(nullptr, db, 1, kOthers);
       }
 
-      thread->stats.FinishedOps(nullptr, db_.db, 1, kOthers);
       transactions_done++;
     }
 
@@ -3820,7 +3919,7 @@ class Benchmark {
     if (FLAGS_optimistic_transaction_db || FLAGS_transaction_db) {
       snprintf(msg, sizeof(msg),
                "( transactions:%" PRIu64 " aborts:%" PRIu64 ")",
-               transactions_done, inserter.GetFailureCount());
+               transactions_done, transactions_aborted);
     } else {
       snprintf(msg, sizeof(msg), "( batches:%" PRIu64 " )", transactions_done);
     }
@@ -3840,15 +3939,50 @@ class Benchmark {
       return;
     }
 
-    Status s =
-        RandomTransactionInserter::Verify(db_.db,
-                            static_cast<uint16_t>(FLAGS_transaction_sets));
+    uint64_t prev_total = 0;
 
-    if (s.ok()) {
-      fprintf(stdout, "RandomTransactionVerify Success.\n");
-    } else {
-      fprintf(stdout, "RandomTransactionVerify FAILED!!\n");
+    // For each set of keys with the same prefix, sum all the values
+    for (uint32_t i = 0; i < FLAGS_transaction_sets; i++) {
+      char prefix_buf[5];
+      snprintf(prefix_buf, sizeof(prefix_buf), "%04u", i + 1);
+      uint64_t total = 0;
+
+      Iterator* iter = db_.db->NewIterator(ReadOptions());
+
+      for (iter->Seek(Slice(prefix_buf, 4)); iter->Valid(); iter->Next()) {
+        Slice key = iter->key();
+
+        // stop when we reach a different prefix
+        if (key.ToString().compare(0, 4, prefix_buf) != 0) {
+          break;
+        }
+
+        Slice value = iter->value();
+        uint64_t int_value = std::stoull(value.ToString());
+        if (int_value == 0 || int_value == ULONG_MAX) {
+          fprintf(stderr, "Iter returned unexpected value: %s\n",
+                  value.ToString().c_str());
+          abort();
+        }
+
+        total += int_value;
+      }
+      delete iter;
+
+      if (i > 0) {
+        if (total != prev_total) {
+          fprintf(stderr,
+                  "RandomTransactionVerify found inconsistent totals. "
+                  "Set[%" PRIu32 "]: %" PRIu64 ", Set[%" PRIu32 "]: %" PRIu64
+                  " \n",
+                  i - 1, prev_total, i, total);
+          abort();
+        }
+      }
+      prev_total = total;
     }
+
+    fprintf(stdout, "RandomTransactionVerify Success!\n");
   }
 #endif  // ROCKSDB_LITE
 
@@ -3938,7 +4072,9 @@ class Benchmark {
   }
 };
 
-int db_bench_tool(int argc, char** argv) {
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
   rocksdb::port::InstallStackTraceHandler();
   SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
                   " [OPTIONS]...");
@@ -4007,5 +4143,5 @@ int db_bench_tool(int argc, char** argv) {
   benchmark.Run();
   return 0;
 }
-}  // namespace rocksdb
-#endif
+
+#endif  // GFLAGS

db/db_block_cache_test.cc

@@ -1,240 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include <cstdlib>
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-
-namespace rocksdb {
-
-static uint64_t TestGetTickerCount(const Options& options,
-                                   Tickers ticker_type) {
-  return options.statistics->getTickerCount(ticker_type);
-}
-
-class DBBlockCacheTest : public DBTestBase {
- private:
-  size_t miss_count_ = 0;
-  size_t hit_count_ = 0;
-  size_t insert_count_ = 0;
-  size_t failure_count_ = 0;
-  size_t compressed_miss_count_ = 0;
-  size_t compressed_hit_count_ = 0;
-  size_t compressed_insert_count_ = 0;
-  size_t compressed_failure_count_ = 0;
-
- public:
-  const size_t kNumBlocks = 10;
-  const size_t kValueSize = 100;
-
-  DBBlockCacheTest() : DBTestBase("/db_block_cache_test") {}
-
-  BlockBasedTableOptions GetTableOptions() {
-    BlockBasedTableOptions table_options;
-    // Set a small enough block size so that each key-value get its own block.
-    table_options.block_size = 1;
-    return table_options;
-  }
-
-  Options GetOptions(const BlockBasedTableOptions& table_options) {
-    Options options = CurrentOptions();
-    options.create_if_missing = true;
-    // options.compression = kNoCompression;
-    options.statistics = rocksdb::CreateDBStatistics();
-    options.table_factory.reset(new BlockBasedTableFactory(table_options));
-    return options;
-  }
-
-  void InitTable(const Options& options) {
-    std::string value(kValueSize, 'a');
-    for (size_t i = 0; i < kNumBlocks; i++) {
-      ASSERT_OK(Put(ToString(i), value.c_str()));
-    }
-  }
-
-  void RecordCacheCounters(const Options& options) {
-    miss_count_ = TestGetTickerCount(options, BLOCK_CACHE_MISS);
-    hit_count_ = TestGetTickerCount(options, BLOCK_CACHE_HIT);
-    insert_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD);
-    failure_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES);
-    compressed_miss_count_ =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS);
-    compressed_hit_count_ =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT);
-    compressed_insert_count_ =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD);
-    compressed_failure_count_ =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD_FAILURES);
-  }
-
-  void CheckCacheCounters(const Options& options, size_t expected_misses,
-                          size_t expected_hits, size_t expected_inserts,
-                          size_t expected_failures) {
-    size_t new_miss_count = TestGetTickerCount(options, BLOCK_CACHE_MISS);
-    size_t new_hit_count = TestGetTickerCount(options, BLOCK_CACHE_HIT);
-    size_t new_insert_count = TestGetTickerCount(options, BLOCK_CACHE_ADD);
-    size_t new_failure_count =
-        TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES);
-    ASSERT_EQ(miss_count_ + expected_misses, new_miss_count);
-    ASSERT_EQ(hit_count_ + expected_hits, new_hit_count);
-    ASSERT_EQ(insert_count_ + expected_inserts, new_insert_count);
-    ASSERT_EQ(failure_count_ + expected_failures, new_failure_count);
-    miss_count_ = new_miss_count;
-    hit_count_ = new_hit_count;
-    insert_count_ = new_insert_count;
-    failure_count_ = new_failure_count;
-  }
-
-  void CheckCompressedCacheCounters(const Options& options,
-                                    size_t expected_misses,
-                                    size_t expected_hits,
-                                    size_t expected_inserts,
-                                    size_t expected_failures) {
-    size_t new_miss_count =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS);
-    size_t new_hit_count =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT);
-    size_t new_insert_count =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD);
-    size_t new_failure_count =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD_FAILURES);
-    ASSERT_EQ(compressed_miss_count_ + expected_misses, new_miss_count);
-    ASSERT_EQ(compressed_hit_count_ + expected_hits, new_hit_count);
-    ASSERT_EQ(compressed_insert_count_ + expected_inserts, new_insert_count);
-    ASSERT_EQ(compressed_failure_count_ + expected_failures, new_failure_count);
-    compressed_miss_count_ = new_miss_count;
-    compressed_hit_count_ = new_hit_count;
-    compressed_insert_count_ = new_insert_count;
-    compressed_failure_count_ = new_failure_count;
-  }
-};
-
-TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
-  ReadOptions read_options;
-  auto table_options = GetTableOptions();
-  auto options = GetOptions(table_options);
-  InitTable(options);
-
-  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
-  table_options.block_cache = cache;
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  Reopen(options);
-  RecordCacheCounters(options);
-
-  std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1);
-  Iterator* iter = nullptr;
-
-  // Load blocks into cache.
-  for (size_t i = 0; i < kNumBlocks - 1; i++) {
-    iter = db_->NewIterator(read_options);
-    iter->Seek(ToString(i));
-    ASSERT_OK(iter->status());
-    CheckCacheCounters(options, 1, 0, 1, 0);
-    iterators[i].reset(iter);
-  }
-  size_t usage = cache->GetUsage();
-  ASSERT_LT(0, usage);
-  cache->SetCapacity(usage);
-  ASSERT_EQ(usage, cache->GetPinnedUsage());
-
-  // Test with strict capacity limit.
-  cache->SetStrictCapacityLimit(true);
-  iter = db_->NewIterator(read_options);
-  iter->Seek(ToString(kNumBlocks - 1));
-  ASSERT_TRUE(iter->status().IsIncomplete());
-  CheckCacheCounters(options, 1, 0, 0, 1);
-  delete iter;
-  iter = nullptr;
-
-  // Release interators and access cache again.
-  for (size_t i = 0; i < kNumBlocks - 1; i++) {
-    iterators[i].reset();
-    CheckCacheCounters(options, 0, 0, 0, 0);
-  }
-  ASSERT_EQ(0, cache->GetPinnedUsage());
-  for (size_t i = 0; i < kNumBlocks - 1; i++) {
-    iter = db_->NewIterator(read_options);
-    iter->Seek(ToString(i));
-    ASSERT_OK(iter->status());
-    CheckCacheCounters(options, 0, 1, 0, 0);
-    iterators[i].reset(iter);
-  }
-}
-
-#ifdef SNAPPY
-TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
-  ReadOptions read_options;
-  auto table_options = GetTableOptions();
-  auto options = GetOptions(table_options);
-  options.compression = CompressionType::kSnappyCompression;
-  InitTable(options);
-
-  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
-  std::shared_ptr<Cache> compressed_cache = NewLRUCache(0, 0, false);
-  table_options.block_cache = cache;
-  table_options.block_cache_compressed = compressed_cache;
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  Reopen(options);
-  RecordCacheCounters(options);
-
-  std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1);
-  Iterator* iter = nullptr;
-
-  // Load blocks into cache.
-  for (size_t i = 0; i < kNumBlocks - 1; i++) {
-    iter = db_->NewIterator(read_options);
-    iter->Seek(ToString(i));
-    ASSERT_OK(iter->status());
-    CheckCacheCounters(options, 1, 0, 1, 0);
-    CheckCompressedCacheCounters(options, 1, 0, 1, 0);
-    iterators[i].reset(iter);
-  }
-  size_t usage = cache->GetUsage();
-  ASSERT_LT(0, usage);
-  ASSERT_EQ(usage, cache->GetPinnedUsage());
-  size_t compressed_usage = compressed_cache->GetUsage();
-  ASSERT_LT(0, compressed_usage);
-  // Compressed block cache cannot be pinned.
-  ASSERT_EQ(0, compressed_cache->GetPinnedUsage());
-
-  // Set strict capacity limit flag. Now block will only load into compressed
-  // block cache.
-  cache->SetCapacity(usage);
-  cache->SetStrictCapacityLimit(true);
-  ASSERT_EQ(usage, cache->GetPinnedUsage());
-  // compressed_cache->SetCapacity(compressed_usage);
-  compressed_cache->SetCapacity(0);
-  // compressed_cache->SetStrictCapacityLimit(true);
-  iter = db_->NewIterator(read_options);
-  iter->Seek(ToString(kNumBlocks - 1));
-  ASSERT_TRUE(iter->status().IsIncomplete());
-  CheckCacheCounters(options, 1, 0, 0, 1);
-  CheckCompressedCacheCounters(options, 1, 0, 1, 0);
-  delete iter;
-  iter = nullptr;
-
-  // Clear strict capacity limit flag. This time we shall hit compressed block
-  // cache.
-  cache->SetStrictCapacityLimit(false);
-  iter = db_->NewIterator(read_options);
-  iter->Seek(ToString(kNumBlocks - 1));
-  ASSERT_OK(iter->status());
-  CheckCacheCounters(options, 1, 0, 1, 0);
-  CheckCompressedCacheCounters(options, 0, 1, 0, 0);
-  delete iter;
-  iter = nullptr;
-}
-#endif
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  rocksdb::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/db_compaction_filter_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -368,10 +368,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
 // entries are deleted. The compaction should create bunch of 'DeleteFile'
 // entries in VersionEdit, but none of the 'AddFile's.
 TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
   options.disable_auto_compactions = true;
   options.create_if_missing = true;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   // put some data
@@ -399,10 +400,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
 
 TEST_F(DBTestCompactionFilter, CompactionFilterWithValueChange) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.num_levels = 3;
     options.compaction_filter_factory =
       std::make_shared<ChangeFilterFactory>();
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Write 100K+1 keys, these are written to a few files
@@ -465,7 +467,8 @@ TEST_F(DBTestCompactionFilter, CompactionFilterWithMergeOperator) {
   PutFixed64(&three, 3);
   PutFixed64(&four, 4);
 
-  Options options = CurrentOptions();
+  Options options;
+  options = CurrentOptions(options);
   options.create_if_missing = true;
   options.merge_operator = MergeOperators::CreateUInt64AddOperator();
   options.num_levels = 3;
@@ -618,10 +621,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextCfId) {
 // Compaction filters should only be applied to records that are newer than the
 // latest snapshot. This test inserts records and applies a delete filter.
 TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
   options.disable_auto_compactions = true;
   options.create_if_missing = true;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   // Put some data.
@@ -655,10 +659,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
 // records newer than the snapshot will also be processed
 TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
   std::string five = ToString(5);
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_filter_factory = std::make_shared<DeleteISFilterFactory>();
   options.disable_auto_compactions = true;
   options.create_if_missing = true;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   // Put some data.

db/db_compaction_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -288,7 +288,8 @@ TEST_F(DBCompactionTest, SkipStatsUpdateTest) {
 }
 
 TEST_F(DBCompactionTest, TestTableReaderForCompaction) {
-  Options options = CurrentOptions();
+  Options options;
+  options = CurrentOptions(options);
   options.env = env_;
   options.new_table_reader_for_compaction_inputs = true;
   options.max_open_files = 100;
@@ -492,13 +493,14 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
 TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
   const int kNumKeysPerFile = 100;
 
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 110 << 10;  // 110KB
   options.arena_block_size = 4 << 10;
   options.num_levels = 3;
   options.level0_file_num_compaction_trigger = 3;
   options.max_subcompactions = max_subcompactions_;
   options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+  options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
   Random rnd(301);
@@ -536,7 +538,7 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
   // and see number of compactions scheduled to be less than allowed.
   const int kNumKeysPerFile = 100;
 
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 110 << 10;  // 110KB
   options.arena_block_size = 4 << 10;
   options.num_levels = 3;
@@ -547,6 +549,7 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
   options.base_background_compactions = 1;
   options.max_background_compactions = 3;
   options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+  options = CurrentOptions(options);
 
   // Block all threads in thread pool.
   const size_t kTotalTasks = 4;
@@ -629,9 +632,10 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
 }
 
 TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 100000000;        // Large write buffer
   options.max_subcompactions = max_subcompactions_;
+  options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
   Random rnd(301);
@@ -658,8 +662,9 @@ TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
 
 TEST_F(DBCompactionTest, MinorCompactionsHappen) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.write_buffer_size = 10000;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     const int N = 500;
@@ -684,7 +689,7 @@ TEST_F(DBCompactionTest, MinorCompactionsHappen) {
 }
 
 TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleLevel;
   options.level0_file_num_compaction_trigger = 3;
 
@@ -698,6 +703,7 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
   const size_t key_len =
     static_cast<size_t>(compact_opt.output_file_size_limit) / 5;
 
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   std::vector<const Snapshot*> snaps;
@@ -741,8 +747,10 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
 // if the database is shutdown during the memtable compaction.
 TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.env = env_;
+    options.write_buffer_size = 1000000;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Trigger a long memtable compaction and reopen the database during it
@@ -766,9 +774,10 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveOneFile) {
       [&](void* arg) { trivial_move++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 100000000;
   options.max_subcompactions = max_subcompactions_;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   int32_t num_keys = 80;
@@ -1360,9 +1369,10 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveToLastLevelWithFiles) {
       [&](void* arg) { non_trivial_move++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 100000000;
   options.max_subcompactions = max_subcompactions_;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   int32_t value_size = 10 * 1024;  // 10 KB
@@ -1651,7 +1661,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
   int max_key_universal_insert = 600;
 
   // Stage 1: generate a db with level compaction
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 110 << 10;  // 110KB
   options.arena_block_size = 4 << 10;
   options.num_levels = 4;
@@ -1661,6 +1671,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
   options.target_file_size_base = 200 << 10;  // 200KB
   options.target_file_size_multiplier = 1;
   options.max_subcompactions = max_subcompactions_;
+  options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
   for (int i = 0; i <= max_key_level_insert; i++) {
@@ -2375,9 +2386,10 @@ TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
       [&](void* arg) { non_trivial_move++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 100000000;
   options.max_subcompactions = max_subcompactions_;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   int32_t value_size = 10 * 1024;  // 10 KB
@@ -2467,14 +2479,14 @@ class CompactionPriTest : public DBTestBase,
 };
 
 TEST_P(CompactionPriTest, Test) {
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 16 * 1024;
   options.compaction_pri = static_cast<CompactionPri>(compaction_pri_);
   options.hard_pending_compaction_bytes_limit = 256 * 1024;
   options.max_bytes_for_level_base = 64 * 1024;
   options.max_bytes_for_level_multiplier = 4;
   options.compression = kNoCompression;
-
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   Random rnd(301);
@@ -2495,12 +2507,8 @@ TEST_P(CompactionPriTest, Test) {
   }
 }
 
-INSTANTIATE_TEST_CASE_P(
-    CompactionPriTest, CompactionPriTest,
-    ::testing::Values(CompactionPri::kByCompensatedSize,
-                      CompactionPri::kOldestLargestSeqFirst,
-                      CompactionPri::kOldestSmallestSeqFirst,
-                      CompactionPri::kMinOverlappingRatio));
+INSTANTIATE_TEST_CASE_P(CompactionPriTest, CompactionPriTest,
+                        ::testing::Values(0, 1, 2));
 
 #endif // !defined(ROCKSDB_LITE)
 }  // namespace rocksdb

db/db_dynamic_level_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_filesnapshot.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_impl.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -282,7 +282,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
 #endif  // ROCKSDB_LITE
       event_logger_(db_options_.info_log.get()),
       bg_work_paused_(0),
-      bg_compaction_paused_(0),
       refitting_level_(false),
       opened_successfully_(false) {
   env_->GetAbsolutePath(dbname, &db_absolute_path_);
@@ -405,21 +404,6 @@ DBImpl::~DBImpl() {
   }
   logs_.clear();
 
-  // Table cache may have table handles holding blocks from the block cache.
-  // We need to release them before the block cache is destroyed. The block
-  // cache may be destroyed inside versions_.reset(), when column family data
-  // list is destroyed, so leaving handles in table cache after
-  // versions_.reset() may cause issues.
-  // Here we clean all unreferenced handles in table cache.
-  // Now we assume all user queries have finished, so only version set itself
-  // can possibly hold the blocks from block cache. After releasing unreferenced
-  // handles here, only handles held by version set left and inside
-  // versions_.reset(), we will release them. There, we need to make sure every
-  // time a handle is released, we erase it from the cache too. By doing that,
-  // we can guarantee that after versions_.reset(), table cache is empty
-  // so the cache can be safely destroyed.
-  table_cache_->EraseUnRefEntries();
-
   // versions need to be destroyed before table_cache since it can hold
   // references to table_cache.
   versions_.reset();
@@ -587,7 +571,6 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
   // Get obsolete files.  This function will also update the list of
   // pending files in VersionSet().
   versions_->GetObsoleteFiles(&job_context->sst_delete_files,
-                              &job_context->manifest_delete_files,
                               job_context->min_pending_output);
 
   // store the current filenum, lognum, etc
@@ -705,9 +688,9 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
   }
 
   auto candidate_files = state.full_scan_candidate_files;
-  candidate_files.reserve(
-      candidate_files.size() + state.sst_delete_files.size() +
-      state.log_delete_files.size() + state.manifest_delete_files.size());
+  candidate_files.reserve(candidate_files.size() +
+                          state.sst_delete_files.size() +
+                          state.log_delete_files.size());
   // We may ignore the dbname when generating the file names.
   const char* kDumbDbName = "";
   for (auto file : state.sst_delete_files) {
@@ -723,9 +706,6 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
                                    0);
     }
   }
-  for (const auto& filename : state.manifest_delete_files) {
-    candidate_files.emplace_back(filename, 0);
-  }
 
   // dedup state.candidate_files so we don't try to delete the same
   // file twice
@@ -844,8 +824,7 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
 
   // Delete old info log files.
   size_t old_info_log_file_count = old_info_log_files.size();
-  if (old_info_log_file_count != 0 &&
-      old_info_log_file_count >= db_options_.keep_log_file_num) {
+  if (old_info_log_file_count >= db_options_.keep_log_file_num) {
     std::sort(old_info_log_files.begin(), old_info_log_files.end());
     size_t end = old_info_log_file_count - db_options_.keep_log_file_num;
     for (unsigned int i = 0; i <= end; i++) {
@@ -1118,23 +1097,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
     stream.EndArray();
   }
 
-#ifndef ROCKSDB_LITE
-  if (db_options_.wal_filter != nullptr) {
-    std::map<std::string, uint32_t> cf_name_id_map;
-    std::map<uint32_t, uint64_t> cf_lognumber_map;
-    for (auto cfd : *versions_->GetColumnFamilySet()) {
-      cf_name_id_map.insert(
-        std::make_pair(cfd->GetName(), cfd->GetID()));
-      cf_lognumber_map.insert(
-        std::make_pair(cfd->GetID(), cfd->GetLogNumber()));
-    }
-
-    db_options_.wal_filter->ColumnFamilyLogNumberMap(
-      cf_lognumber_map,
-      cf_name_id_map);
-  }
-#endif
-
   bool continue_replay_log = true;
   for (auto log_number : log_numbers) {
     // The previous incarnation may not have written any MANIFEST
@@ -1201,7 +1163,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
         continue_replay_log &&
         reader.ReadRecord(&record, &scratch, db_options_.wal_recovery_mode) &&
         status.ok()) {
-      if (record.size() < WriteBatchInternal::kHeader) {
+      if (record.size() < 12) {
         reporter.Corruption(record.size(),
                             Status::Corruption("log record too small"));
         continue;
@@ -1214,8 +1176,8 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
         bool batch_changed = false;
 
         WalFilter::WalProcessingOption wal_processing_option =
-            db_options_.wal_filter->LogRecordFound(log_number, fname, batch,
-                                              &new_batch, &batch_changed);
+            db_options_.wal_filter->LogRecord(batch, &new_batch,
+                                              &batch_changed);
 
         switch (wal_processing_option) {
           case WalFilter::WalProcessingOption::kContinueProcessing:
@@ -1424,9 +1386,9 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
   mutex_.AssertHeld();
   const uint64_t start_micros = env_->NowMicros();
   FileMetaData meta;
+  meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0);
   auto pending_outputs_inserted_elem =
       CaptureCurrentFileNumberInPendingOutputs();
-  meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0);
   ReadOptions ro;
   ro.total_order_seek = true;
   Arena arena;
@@ -1452,8 +1414,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
       s = BuildTable(
           dbname_, env_, *cfd->ioptions(), env_options_, cfd->table_cache(),
           iter.get(), &meta, cfd->internal_comparator(),
-          cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
-          snapshot_seqs, earliest_write_conflict_snapshot,
+          cfd->int_tbl_prop_collector_factories(), cfd->GetID(), snapshot_seqs,
+          earliest_write_conflict_snapshot,
           GetCompressionFlush(*cfd->ioptions()),
           cfd->ioptions()->compression_opts, paranoid_file_checks,
           cfd->internal_stats(), Env::IO_HIGH, &info.table_properties);
@@ -1513,13 +1475,12 @@ Status DBImpl::FlushMemTableToOutputFile(
   std::vector<SequenceNumber> snapshot_seqs =
       snapshots_.GetAll(&earliest_write_conflict_snapshot);
 
-  FlushJob flush_job(dbname_, cfd, db_options_, mutable_cf_options,
-                     env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     snapshot_seqs, earliest_write_conflict_snapshot,
-                     job_context, log_buffer, directories_.GetDbDir(),
-                     directories_.GetDataDir(0U),
-                     GetCompressionFlush(*cfd->ioptions()), stats_,
-                     &event_logger_, mutable_cf_options.report_bg_io_stats);
+  FlushJob flush_job(
+      dbname_, cfd, db_options_, mutable_cf_options, env_options_,
+      versions_.get(), &mutex_, &shutting_down_, snapshot_seqs,
+      earliest_write_conflict_snapshot, job_context, log_buffer,
+      directories_.GetDbDir(), directories_.GetDataDir(0U),
+      GetCompressionFlush(*cfd->ioptions()), stats_, &event_logger_);
 
   FileMetaData file_meta;
 
@@ -1549,12 +1510,11 @@ Status DBImpl::FlushMemTableToOutputFile(
     bg_error_ = s;
   }
   RecordFlushIOStats();
-  if (s.ok()) {
 #ifndef ROCKSDB_LITE
+  if (s.ok()) {
     // may temporarily unlock and lock the mutex.
     NotifyOnFlushCompleted(cfd, &file_meta, mutable_cf_options,
                            job_context->job_id, flush_job.GetTableProperties());
-#endif  // ROCKSDB_LITE
     auto sfm =
         static_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
     if (sfm) {
@@ -1562,13 +1522,9 @@ Status DBImpl::FlushMemTableToOutputFile(
       std::string file_path = MakeTableFileName(db_options_.db_paths[0].path,
                                                 file_meta.fd.GetNumber());
       sfm->OnAddFile(file_path);
-      if (sfm->IsMaxAllowedSpaceReached() && bg_error_.ok()) {
-        bg_error_ = Status::IOError("Max allowed space was reached");
-        TEST_SYNC_POINT(
-            "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached");
-      }
     }
   }
+#endif  // ROCKSDB_LITE
   return s;
 }
 
@@ -1865,10 +1821,10 @@ Status DBImpl::CompactFilesImpl(
   CompactionJob compaction_job(
       job_context->job_id, c.get(), db_options_, env_options_, versions_.get(),
       &shutting_down_, log_buffer, directories_.GetDbDir(),
-      directories_.GetDataDir(c->output_path_id()), stats_, &mutex_, &bg_error_,
-      snapshot_seqs, earliest_write_conflict_snapshot, table_cache_,
-      &event_logger_, c->mutable_cf_options()->paranoid_file_checks,
-      c->mutable_cf_options()->report_bg_io_stats, dbname_,
+      directories_.GetDataDir(c->output_path_id()), stats_, snapshot_seqs,
+      earliest_write_conflict_snapshot, table_cache_, &event_logger_,
+      c->mutable_cf_options()->paranoid_file_checks,
+      c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
       nullptr);  // Here we pass a nullptr for CompactionJobStats because
                  // CompactFiles does not trigger OnCompactionCompleted(),
                  // which is the only place where CompactionJobStats is
@@ -1882,17 +1838,6 @@ Status DBImpl::CompactFilesImpl(
                  // support for CompactFiles, we should have CompactFiles API
                  // pass a pointer of CompactionJobStats as the out-value
                  // instead of using EventListener.
-
-  // Creating a compaction influences the compaction score because the score
-  // takes running compactions into account (by skipping files that are already
-  // being compacted). Since we just changed compaction score, we recalculate it
-  // here.
-  {
-    CompactionOptionsFIFO dummy_compaction_options_fifo;
-    version->storage_info()->ComputeCompactionScore(
-        *c->mutable_cf_options(), dummy_compaction_options_fifo);
-  }
-
   compaction_job.Prepare();
 
   mutex_.Unlock();
@@ -1903,7 +1848,7 @@ Status DBImpl::CompactFilesImpl(
   TEST_SYNC_POINT("CompactFilesImpl:3");
   mutex_.Lock();
 
-  Status status = compaction_job.Install(*c->mutable_cf_options());
+  Status status = compaction_job.Install(*c->mutable_cf_options(), &mutex_);
   if (status.ok()) {
     InstallSuperVersionAndScheduleWorkWrapper(
         c->column_family_data(), job_context, *c->mutable_cf_options());
@@ -1939,11 +1884,10 @@ Status DBImpl::CompactFilesImpl(
 
 Status DBImpl::PauseBackgroundWork() {
   InstrumentedMutexLock guard_lock(&mutex_);
-  bg_compaction_paused_++;
+  bg_work_paused_++;
   while (bg_compaction_scheduled_ > 0 || bg_flush_scheduled_ > 0) {
     bg_cv_.Wait();
   }
-  bg_work_paused_++;
   return Status::OK();
 }
 
@@ -1953,11 +1897,7 @@ Status DBImpl::ContinueBackgroundWork() {
     return Status::InvalidArgument();
   }
   assert(bg_work_paused_ > 0);
-  assert(bg_compaction_paused_ > 0);
-  bg_compaction_paused_--;
   bg_work_paused_--;
-  // It's sufficient to check just bg_work_paused_ here since
-  // bg_work_paused_ is always no greater than bg_compaction_paused_
   if (bg_work_paused_ == 0) {
     MaybeScheduleFlushOrCompaction();
   }
@@ -2257,9 +2197,6 @@ Status DBImpl::SyncWAL() {
     status = directories_.GetWalDir()->Fsync();
   }
 
-  TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:1");
-  TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:2");
-
   {
     InstrumentedMutexLock l(&mutex_);
     MarkLogsSynced(current_log_number, need_log_dir_sync, status);
@@ -2287,8 +2224,7 @@ void DBImpl::MarkLogsSynced(
       ++it;
     }
   }
-  assert(logs_.empty() || logs_[0].number > up_to ||
-         (logs_.size() == 1 && !logs_[0].getting_synced));
+  assert(logs_.empty() || (logs_.size() == 1 && !logs_[0].getting_synced));
   log_sync_cv_.SignalAll();
 }
 
@@ -2540,11 +2476,6 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
     }
   }
 
-  if (bg_compaction_paused_ > 0) {
-    // we paused the background compaction
-    return;
-  }
-
   if (HasExclusiveManualCompaction()) {
     // only manual compactions are allowed to run. don't schedule automatic
     // compactions
@@ -3070,11 +3001,10 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
     CompactionJob compaction_job(
         job_context->job_id, c.get(), db_options_, env_options_,
         versions_.get(), &shutting_down_, log_buffer, directories_.GetDbDir(),
-        directories_.GetDataDir(c->output_path_id()), stats_, &mutex_,
-        &bg_error_, snapshot_seqs, earliest_write_conflict_snapshot,
-        table_cache_, &event_logger_,
+        directories_.GetDataDir(c->output_path_id()), stats_, snapshot_seqs,
+        earliest_write_conflict_snapshot, table_cache_, &event_logger_,
         c->mutable_cf_options()->paranoid_file_checks,
-        c->mutable_cf_options()->report_bg_io_stats, dbname_,
+        c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
         &compaction_job_stats);
     compaction_job.Prepare();
 
@@ -3083,7 +3013,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
     TEST_SYNC_POINT("DBImpl::BackgroundCompaction:NonTrivial:AfterRun");
     mutex_.Lock();
 
-    status = compaction_job.Install(*c->mutable_cf_options());
+    status = compaction_job.Install(*c->mutable_cf_options(), &mutex_);
     if (status.ok()) {
       InstallSuperVersionAndScheduleWorkWrapper(
           c->column_family_data(), job_context, *c->mutable_cf_options());
@@ -3647,16 +3577,8 @@ Status DBImpl::AddFile(ColumnFamilyHandle* column_family,
     return Status::InvalidArgument(
         "Non zero sequence numbers are not supported");
   }
-
   // Generate a location for the new table
-  std::list<uint64_t>::iterator pending_outputs_inserted_elem;
-  {
-    InstrumentedMutexLock l(&mutex_);
-    pending_outputs_inserted_elem = CaptureCurrentFileNumberInPendingOutputs();
-    meta.fd =
-        FileDescriptor(versions_->NewFileNumber(), 0, file_info->file_size);
-  }
-
+  meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, file_info->file_size);
   std::string db_fname = TableFileName(
       db_options_.db_paths, meta.fd.GetNumber(), meta.fd.GetPathId());
 
@@ -3669,7 +3591,6 @@ Status DBImpl::AddFile(ColumnFamilyHandle* column_family,
   } else {
     status = CopyFile(env_, file_info->file_path, db_fname, 0);
   }
-  TEST_SYNC_POINT("DBImpl::AddFile:FileCopied");
   if (!status.ok()) {
     return status;
   }
@@ -3733,7 +3654,6 @@ Status DBImpl::AddFile(ColumnFamilyHandle* column_family,
       delete InstallSuperVersionAndScheduleWork(cfd, nullptr,
                                                 mutable_cf_options);
     }
-    ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
   }
 
   if (!status.ok()) {
@@ -3982,8 +3902,8 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
         env_, *cfd->ioptions(), cfd->user_comparator(), iter,
         kMaxSequenceNumber,
         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-        sv->version_number, read_options.iterate_upper_bound,
-        read_options.prefix_same_as_start, read_options.pin_data);
+        read_options.iterate_upper_bound, read_options.prefix_same_as_start,
+        read_options.pin_data);
 #endif
   } else {
     SequenceNumber latest_snapshot = versions_->LastSequence();
@@ -4040,8 +3960,8 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
     ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
         env_, *cfd->ioptions(), cfd->user_comparator(), snapshot,
         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-        sv->version_number, read_options.iterate_upper_bound,
-        read_options.prefix_same_as_start, read_options.pin_data);
+        read_options.iterate_upper_bound, read_options.prefix_same_as_start,
+        read_options.pin_data);
 
     InternalIterator* internal_iter =
         NewInternalIterator(read_options, cfd, sv, db_iter->GetArena());
@@ -4094,8 +4014,8 @@ Status DBImpl::NewIterators(
       iterators->push_back(NewDBIterator(
           env_, *cfd->ioptions(), cfd->user_comparator(), iter,
           kMaxSequenceNumber,
-          sv->mutable_cf_options.max_sequential_skip_in_iterations,
-          sv->version_number, nullptr, false, read_options.pin_data));
+          sv->mutable_cf_options.max_sequential_skip_in_iterations, nullptr,
+          false, read_options.pin_data));
     }
 #endif
   } else {
@@ -4114,8 +4034,8 @@ Status DBImpl::NewIterators(
 
       ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
           env_, *cfd->ioptions(), cfd->user_comparator(), snapshot,
-          sv->mutable_cf_options.max_sequential_skip_in_iterations,
-          sv->version_number, nullptr, false, read_options.pin_data);
+          sv->mutable_cf_options.max_sequential_skip_in_iterations, nullptr,
+          false, read_options.pin_data);
       InternalIterator* internal_iter =
           NewInternalIterator(read_options, cfd, sv, db_iter->GetArena());
       db_iter->SetIterUnderDBIter(internal_iter);
@@ -4207,6 +4127,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   }
 
   Status status;
+  bool callback_failed = false;
 
   bool xfunc_attempted_write = false;
   XFUNC_TEST("transaction", "transaction_xftest_write_impl",
@@ -4224,7 +4145,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   w.sync = write_options.sync;
   w.disableWAL = write_options.disableWAL;
   w.in_batch_group = false;
-  w.callback = callback;
+  w.has_callback = (callback != nullptr) ? true : false;
 
   if (!write_options.disableWAL) {
     RecordTick(stats_, WRITE_WITH_WAL);
@@ -4237,32 +4158,30 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     // we are a non-leader in a parallel group
     PERF_TIMER_GUARD(write_memtable_time);
 
-    if (!w.CallbackFailed()) {
-      ColumnFamilyMemTablesImpl column_family_memtables(
-          versions_->GetColumnFamilySet());
-      WriteBatchInternal::SetSequence(w.batch, w.sequence);
-      w.status = WriteBatchInternal::InsertInto(
-          w.batch, &column_family_memtables, &flush_scheduler_,
-          write_options.ignore_missing_column_families, 0 /*log_number*/, this,
-          true /*dont_filter_deletes*/, true /*concurrent_memtable_writes*/);
-    }
+    ColumnFamilyMemTablesImpl column_family_memtables(
+        versions_->GetColumnFamilySet());
+    WriteBatchInternal::SetSequence(w.batch, w.sequence);
+    w.status = WriteBatchInternal::InsertInto(
+        w.batch, &column_family_memtables, &flush_scheduler_,
+        write_options.ignore_missing_column_families, 0 /*log_number*/, this,
+        true /*dont_filter_deletes*/, true /*concurrent_memtable_writes*/);
 
     if (write_thread_.CompleteParallelWorker(&w)) {
       // we're responsible for early exit
-      auto last_sequence = w.parallel_group->last_sequence;
+      auto last_sequence =
+          w.parallel_group->last_writer->sequence +
+          WriteBatchInternal::Count(w.parallel_group->last_writer->batch) - 1;
       SetTickerCount(stats_, SEQUENCE_NUMBER, last_sequence);
       versions_->SetLastSequence(last_sequence);
       write_thread_.EarlyExitParallelGroup(&w);
     }
     assert(w.state == WriteThread::STATE_COMPLETED);
     // STATE_COMPLETED conditional below handles exit
-
-    status = w.FinalStatus();
   }
   if (w.state == WriteThread::STATE_COMPLETED) {
     // write is complete and leader has updated sequence
     RecordTick(stats_, WRITE_DONE_BY_OTHER);
-    return w.FinalStatus();
+    return w.status;
   }
   // else we are the leader of the write batch group
   assert(w.state == WriteThread::STATE_GROUP_LEADER);
@@ -4368,7 +4287,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 
   uint64_t last_sequence = versions_->LastSequence();
   WriteThread::Writer* last_writer = &w;
-  autovector<WriteThread::Writer*> write_group;
+  autovector<WriteBatch*> write_batch_group;
   bool need_log_sync = !write_options.disableWAL && write_options.sync;
   bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
 
@@ -4387,15 +4306,24 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     // during this phase since &w is currently responsible for logging
     // and protects against concurrent loggers and concurrent writes
     // into memtables
-  }
 
-  mutex_.Unlock();
+    mutex_.Unlock();
+
+    if (callback != nullptr) {
+      // If this write has a validation callback, check to see if this write
+      // is able to be written.  Must be called on the write thread.
+      status = callback->Callback(this);
+      callback_failed = true;
+    }
+  } else {
+    mutex_.Unlock();
+  }
 
   // At this point the mutex is unlocked
 
   bool exit_completed_early = false;
-  last_batch_group_size_ =
-      write_thread_.EnterAsBatchGroupLeader(&w, &last_writer, &write_group);
+  last_batch_group_size_ = write_thread_.EnterAsBatchGroupLeader(
+      &w, &last_writer, &write_batch_group);
 
   if (status.ok()) {
     // Rules for when we can update the memtable concurrently
@@ -4411,17 +4339,15 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     // assumed to be true.  Rule 4 is checked for each batch.  We could
     // relax rules 2 and 3 if we could prevent write batches from referring
     // more than once to a particular key.
-    bool parallel =
-        db_options_.allow_concurrent_memtable_write && write_group.size() > 1;
+    bool parallel = db_options_.allow_concurrent_memtable_write &&
+                    write_batch_group.size() > 1;
     int total_count = 0;
     uint64_t total_byte_size = 0;
-    for (auto writer : write_group) {
-      if (writer->CheckCallback(this)) {
-        total_count += WriteBatchInternal::Count(writer->batch);
-        total_byte_size = WriteBatchInternal::AppendedByteSize(
-            total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
-        parallel = parallel && !writer->batch->HasMerge();
-      }
+    for (auto b : write_batch_group) {
+      total_count += WriteBatchInternal::Count(b);
+      total_byte_size = WriteBatchInternal::AppendedByteSize(
+          total_byte_size, WriteBatchInternal::ByteSize(b));
+      parallel = parallel && !b->HasMerge();
     }
 
     const SequenceNumber current_sequence = last_sequence + 1;
@@ -4442,22 +4368,21 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
       PERF_TIMER_GUARD(write_wal_time);
 
       WriteBatch* merged_batch = nullptr;
-      if (write_group.size() == 1 && !write_group[0]->CallbackFailed()) {
-        merged_batch = write_group[0]->batch;
+      if (write_batch_group.size() == 1) {
+        merged_batch = write_batch_group[0];
       } else {
         // WAL needs all of the batches flattened into a single batch.
         // We could avoid copying here with an iov-like AddRecord
         // interface
         merged_batch = &tmp_batch_;
-        for (auto writer : write_group) {
-          if (!writer->CallbackFailed()) {
-            WriteBatchInternal::Append(merged_batch, writer->batch);
-          }
+        for (auto b : write_batch_group) {
+          WriteBatchInternal::Append(merged_batch, b);
         }
       }
       WriteBatchInternal::SetSequence(merged_batch, current_sequence);
 
       assert(WriteBatchInternal::Count(merged_batch) == total_count);
+      assert(WriteBatchInternal::ByteSize(merged_batch) == total_byte_size);
 
       Slice log_entry = WriteBatchInternal::Contents(merged_batch);
       status = logs_.back().writer->AddRecord(log_entry);
@@ -4512,7 +4437,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
           }
           stats->AddDBStats(InternalStats::WAL_FILE_BYTES, log_size);
         }
-        uint64_t for_other = write_group.size() - 1;
+        uint64_t for_other = write_batch_group.size() - 1;
         if (for_other > 0) {
           stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER, for_other);
           if (!write_options.disableWAL) {
@@ -4523,50 +4448,43 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 
       if (!parallel) {
         status = WriteBatchInternal::InsertInto(
-            write_group, current_sequence, column_family_memtables_.get(),
+            write_batch_group, current_sequence, column_family_memtables_.get(),
             &flush_scheduler_, write_options.ignore_missing_column_families,
             0 /*log_number*/, this, false /*dont_filter_deletes*/);
-
-        if (status.ok()) {
-          // There were no write failures. Set leader's status
-          // in case the write callback returned a non-ok status.
-          status = w.FinalStatus();
-        }
-
       } else {
         WriteThread::ParallelGroup pg;
         pg.leader = &w;
         pg.last_writer = last_writer;
-        pg.last_sequence = last_sequence;
         pg.early_exit_allowed = !need_log_sync;
-        pg.running.store(static_cast<uint32_t>(write_group.size()),
+        pg.running.store(static_cast<uint32_t>(write_batch_group.size()),
                          std::memory_order_relaxed);
         write_thread_.LaunchParallelFollowers(&pg, current_sequence);
 
-        if (!w.CallbackFailed()) {
-          // do leader write
-          ColumnFamilyMemTablesImpl column_family_memtables(
-              versions_->GetColumnFamilySet());
-          assert(w.sequence == current_sequence);
-          WriteBatchInternal::SetSequence(w.batch, w.sequence);
-          w.status = WriteBatchInternal::InsertInto(
-              w.batch, &column_family_memtables, &flush_scheduler_,
-              write_options.ignore_missing_column_families, 0 /*log_number*/,
-              this, true /*dont_filter_deletes*/,
-              true /*concurrent_memtable_writes*/);
-        }
+        ColumnFamilyMemTablesImpl column_family_memtables(
+            versions_->GetColumnFamilySet());
+        assert(w.sequence == current_sequence);
+        WriteBatchInternal::SetSequence(w.batch, w.sequence);
+        w.status = WriteBatchInternal::InsertInto(
+            w.batch, &column_family_memtables, &flush_scheduler_,
+            write_options.ignore_missing_column_families, 0 /*log_number*/,
+            this, true /*dont_filter_deletes*/,
+            true /*concurrent_memtable_writes*/);
 
+        assert(last_writer->sequence +
+                   WriteBatchInternal::Count(last_writer->batch) - 1 ==
+               last_sequence);
         // CompleteParallelWorker returns true if this thread should
         // handle exit, false means somebody else did
         exit_completed_early = !write_thread_.CompleteParallelWorker(&w);
-        status = w.FinalStatus();
+        status = w.status;
+        assert(status.ok() || !exit_completed_early);
       }
 
-      if (!exit_completed_early && w.status.ok()) {
+      if (status.ok() && !exit_completed_early) {
         SetTickerCount(stats_, SEQUENCE_NUMBER, last_sequence);
         versions_->SetLastSequence(last_sequence);
         if (!need_log_sync) {
-          write_thread_.ExitAsBatchGroupLeader(&w, last_writer, w.status);
+          write_thread_.ExitAsBatchGroupLeader(&w, last_writer, status);
           exit_completed_early = true;
         }
       }
@@ -4579,14 +4497,14 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
       //
       // Is setting bg_error_ enough here?  This will at least stop
       // compaction and fail any further writes.
-      if (!status.ok() && bg_error_.ok() && !w.CallbackFailed()) {
+      if (!status.ok() && bg_error_.ok()) {
         bg_error_ = status;
       }
     }
   }
   PERF_TIMER_START(write_pre_and_post_process_time);
 
-  if (db_options_.paranoid_checks && !status.ok() && !w.CallbackFailed() &&
+  if (db_options_.paranoid_checks && !status.ok() && !callback_failed &&
       !status.IsBusy()) {
     mutex_.Lock();
     if (bg_error_.ok()) {
@@ -4602,7 +4520,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   }
 
   if (!exit_completed_early) {
-    write_thread_.ExitAsBatchGroupLeader(&w, last_writer, w.status);
+    write_thread_.ExitAsBatchGroupLeader(&w, last_writer, status);
   }
 
   return status;

db/db_impl.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -494,11 +494,10 @@ class DBImpl : public DB {
 
   // Background process needs to call
   //     auto x = CaptureCurrentFileNumberInPendingOutputs()
-  //     auto file_num = versions_->NewFileNumber();
   //     <do something>
   //     ReleaseFileNumberFromPendingOutputs(x)
-  // This will protect any file with number `file_num` or greater from being
-  // deleted while <do something> is running.
+  // This will protect any temporary files created while <do something> is
+  // executing from being deleted.
   // -----------
   // This function will capture current file number and append it to
   // pending_outputs_. This will prevent any background process to delete any
@@ -523,7 +522,7 @@ class DBImpl : public DB {
                          SequenceNumber* max_sequence, bool read_only);
 
   // The following two methods are used to flush a memtable to
-  // storage. The first one is used at database RecoveryTime (when the
+  // storage. The first one is used atdatabase RecoveryTime (when the
   // database is opened) and is heavyweight because it holds the mutex
   // for the entire period. The second method WriteLevel0Table supports
   // concurrent flush memtables to storage.
@@ -844,12 +843,9 @@ class DBImpl : public DB {
   // Unified interface for logging events
   EventLogger event_logger_;
 
-  // A value of > 0 temporarily disables scheduling of background work
+  // A value of >0 temporarily disables scheduling of background work
   int bg_work_paused_;
 
-  // A value of > 0 temporarily disables scheduling of background compaction
-  int bg_compaction_paused_;
-
   // Guard against multiple concurrent refitting
   bool refitting_level_;
 

db/db_impl_debug.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_impl_experimental.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_impl_readonly.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -54,11 +54,10 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
   auto db_iter = NewArenaWrappedDbIterator(
       env_, *cfd->ioptions(), cfd->user_comparator(),
       (read_options.snapshot != nullptr
-           ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
-                 ->number_
+           ? reinterpret_cast<const SnapshotImpl*>(
+                read_options.snapshot)->number_
            : latest_snapshot),
-      super_version->mutable_cf_options.max_sequential_skip_in_iterations,
-      super_version->version_number);
+      super_version->mutable_cf_options.max_sequential_skip_in_iterations);
   auto internal_iter = NewInternalIterator(
       read_options, cfd, super_version, db_iter->GetArena());
   db_iter->SetIterUnderDBIter(internal_iter);
@@ -82,11 +81,10 @@ Status DBImplReadOnly::NewIterators(
     auto* db_iter = NewArenaWrappedDbIterator(
         env_, *cfd->ioptions(), cfd->user_comparator(),
         (read_options.snapshot != nullptr
-             ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
-                   ->number_
-             : latest_snapshot),
-        sv->mutable_cf_options.max_sequential_skip_in_iterations,
-        sv->version_number);
+            ? reinterpret_cast<const SnapshotImpl*>(
+                  read_options.snapshot)->number_
+            : latest_snapshot),
+        sv->mutable_cf_options.max_sequential_skip_in_iterations);
     auto* internal_iter = NewInternalIterator(
         read_options, cfd, sv, db_iter->GetArena());
     db_iter->SetIterUnderDBIter(internal_iter);

db/db_impl_readonly.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_info_dumper.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_info_dumper.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_inplace_update_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -18,11 +18,12 @@ class DBTestInPlaceUpdate : public DBTestBase {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
     options.env = env_;
     options.write_buffer_size = 100000;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Update key with values of smaller size
@@ -40,11 +41,12 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
     options.env = env_;
     options.write_buffer_size = 100000;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Update key with values of larger size
@@ -62,7 +64,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
 
@@ -70,6 +72,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
     options.write_buffer_size = 100000;
     options.inplace_callback =
       rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerSize;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Update key with values of smaller size
@@ -89,7 +92,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
 
@@ -97,6 +100,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
     options.write_buffer_size = 100000;
     options.inplace_callback =
       rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerVarintSize;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Update key with values of smaller varint size
@@ -116,7 +120,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
 
@@ -124,6 +128,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
     options.write_buffer_size = 100000;
     options.inplace_callback =
       rocksdb::DBTestInPlaceUpdate::updateInPlaceLargerSize;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Update key with values of larger size
@@ -141,7 +146,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
 
@@ -149,6 +154,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
     options.write_buffer_size = 100000;
     options.inplace_callback =
       rocksdb::DBTestInPlaceUpdate::updateInPlaceNoAction;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Callback function requests no actions from db

db/db_iter.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -13,20 +13,18 @@
 #include <string>
 #include <limits>
 
-#include "db/dbformat.h"
 #include "db/filename.h"
-#include "db/merge_context.h"
+#include "db/dbformat.h"
 #include "port/port.h"
 #include "rocksdb/env.h"
+#include "rocksdb/options.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/merge_operator.h"
-#include "rocksdb/options.h"
 #include "table/internal_iterator.h"
 #include "util/arena.h"
 #include "util/logging.h"
 #include "util/mutexlock.h"
 #include "util/perf_context_imp.h"
-#include "util/string_util.h"
 
 namespace rocksdb {
 
@@ -61,47 +59,9 @@ class DBIter: public Iterator {
     kReverse
   };
 
-  // LocalStatistics contain Statistics counters that will be aggregated per
-  // each iterator instance and then will be sent to the global statistics when
-  // the iterator is destroyed.
-  //
-  // The purpose of this approach is to avoid perf regression happening
-  // when multiple threads bump the atomic counters from a DBIter::Next().
-  struct LocalStatistics {
-    explicit LocalStatistics() { ResetCounters(); }
-
-    void ResetCounters() {
-      next_count_ = 0;
-      next_found_count_ = 0;
-      prev_count_ = 0;
-      prev_found_count_ = 0;
-      bytes_read_ = 0;
-    }
-
-    void BumpGlobalStatistics(Statistics* global_statistics) {
-      RecordTick(global_statistics, NUMBER_DB_NEXT, next_count_);
-      RecordTick(global_statistics, NUMBER_DB_NEXT_FOUND, next_found_count_);
-      RecordTick(global_statistics, NUMBER_DB_PREV, prev_count_);
-      RecordTick(global_statistics, NUMBER_DB_PREV_FOUND, prev_found_count_);
-      RecordTick(global_statistics, ITER_BYTES_READ, bytes_read_);
-      ResetCounters();
-    }
-
-    // Map to Tickers::NUMBER_DB_NEXT
-    uint64_t next_count_;
-    // Map to Tickers::NUMBER_DB_NEXT_FOUND
-    uint64_t next_found_count_;
-    // Map to Tickers::NUMBER_DB_PREV
-    uint64_t prev_count_;
-    // Map to Tickers::NUMBER_DB_PREV_FOUND
-    uint64_t prev_found_count_;
-    // Map to Tickers::ITER_BYTES_READ
-    uint64_t bytes_read_;
-  };
-
   DBIter(Env* env, const ImmutableCFOptions& ioptions, const Comparator* cmp,
          InternalIterator* iter, SequenceNumber s, bool arena_mode,
-         uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+         uint64_t max_sequential_skip_in_iterations,
          const Slice* iterate_upper_bound = nullptr,
          bool prefix_same_as_start = false)
       : arena_mode_(arena_mode),
@@ -115,7 +75,6 @@ class DBIter: public Iterator {
         valid_(false),
         current_entry_is_merged_(false),
         statistics_(ioptions.statistics),
-        version_number_(version_number),
         iterate_upper_bound_(iterate_upper_bound),
         prefix_same_as_start_(prefix_same_as_start),
         iter_pinned_(false) {
@@ -125,7 +84,6 @@ class DBIter: public Iterator {
   }
   virtual ~DBIter() {
     RecordTick(statistics_, NO_ITERATORS, -1);
-    local_stats_.BumpGlobalStatistics(statistics_);
     if (!arena_mode_) {
       delete iter_;
     } else {
@@ -178,27 +136,9 @@ class DBIter: public Iterator {
     }
     return s;
   }
-
-  virtual Status GetProperty(std::string prop_name,
-                             std::string* prop) override {
-    if (prop == nullptr) {
-      return Status::InvalidArgument("prop is nullptr");
-    }
-    if (prop_name == "rocksdb.iterator.super-version-number") {
-      // First try to pass the value returned from inner iterator.
-      if (!iter_->GetProperty(prop_name, prop).ok()) {
-        *prop = ToString(version_number_);
-      }
-      return Status::OK();
-    } else if (prop_name == "rocksdb.iterator.is-key-pinned") {
-      if (valid_) {
-        *prop = (iter_pinned_ && saved_key_.IsKeyPinned()) ? "1" : "0";
-      } else {
-        *prop = "Iterator is not valid.";
-      }
-      return Status::OK();
-    }
-    return Status::InvalidArgument("Undentified property.");
+  virtual bool IsKeyPinned() const override {
+    assert(valid_);
+    return iter_pinned_ && saved_key_.IsKeyPinned();
   }
 
   virtual void Next() override;
@@ -246,14 +186,12 @@ class DBIter: public Iterator {
   bool current_entry_is_merged_;
   Statistics* statistics_;
   uint64_t max_skip_;
-  uint64_t version_number_;
   const Slice* iterate_upper_bound_;
   IterKey prefix_start_;
   bool prefix_same_as_start_;
   bool iter_pinned_;
   // List of operands for merge operator.
-  MergeContext merge_context_;
-  LocalStatistics local_stats_;
+  std::deque<std::string> merge_operands_;
 
   // No copying allowed
   DBIter(const DBIter&);
@@ -291,9 +229,6 @@ void DBIter::Next() {
     PERF_COUNTER_ADD(internal_key_skipped_count, 1);
   }
 
-  if (statistics_ != nullptr) {
-    local_stats_.next_count_++;
-  }
   // Now we point to the next internal position, for both of merge and
   // not merge cases.
   if (!iter_->Valid()) {
@@ -301,15 +236,18 @@ void DBIter::Next() {
     return;
   }
   FindNextUserEntry(true /* skipping the current user key */);
+  if (statistics_ != nullptr) {
+    RecordTick(statistics_, NUMBER_DB_NEXT);
+    if (valid_) {
+      RecordTick(statistics_, NUMBER_DB_NEXT_FOUND);
+      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
+    }
+  }
   if (valid_ && prefix_extractor_ && prefix_same_as_start_ &&
       prefix_extractor_->Transform(saved_key_.GetKey())
               .compare(prefix_start_.GetKey()) != 0) {
     valid_ = false;
   }
-  if (statistics_ != nullptr && valid_) {
-    local_stats_.next_found_count_++;
-    local_stats_.bytes_read_ += (key().size() + value().size());
-  }
 }
 
 // PRE: saved_key_ has the current user key if skipping
@@ -337,7 +275,7 @@ void DBIter::FindNextUserEntryInternal(bool skipping) {
 
     if (ParseKey(&ikey)) {
       if (iterate_upper_bound_ != nullptr &&
-          user_comparator_->Compare(ikey.user_key, *iterate_upper_bound_) >= 0) {
+          ikey.user_key.compare(*iterate_upper_bound_) >= 0) {
         break;
       }
 
@@ -412,9 +350,9 @@ void DBIter::MergeValuesNewToOld() {
     return;
   }
 
-  merge_context_.Clear();
   // Start the merge process by pushing the first operand
-  merge_context_.PushOperand(iter_->value());
+  std::deque<std::string> operands;
+  operands.push_front(iter_->value().ToString());
 
   ParsedInternalKey ikey;
   for (iter_->Next(); iter_->Valid(); iter_->Next()) {
@@ -439,8 +377,7 @@ void DBIter::MergeValuesNewToOld() {
       {
         StopWatchNano timer(env_, statistics_ != nullptr);
         PERF_TIMER_GUARD(merge_operator_time_nanos);
-        user_merge_operator_->FullMerge(ikey.user_key, &val,
-                                        merge_context_.GetOperands(),
+        user_merge_operator_->FullMerge(ikey.user_key, &val, operands,
                                         &saved_value_, logger_);
         RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                    timer.ElapsedNanos());
@@ -452,7 +389,7 @@ void DBIter::MergeValuesNewToOld() {
       // hit a merge, add the value as an operand and run associative merge.
       // when complete, add result to operands and continue.
       const Slice& val = iter_->value();
-      merge_context_.PushOperand(val);
+      operands.push_front(val.ToString());
     } else {
       assert(false);
     }
@@ -465,9 +402,8 @@ void DBIter::MergeValuesNewToOld() {
     // a deletion marker.
     // feed null as the existing value to the merge operator, such that
     // client can differentiate this scenario and do things accordingly.
-    user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
-                                    merge_context_.GetOperands(), &saved_value_,
-                                    logger_);
+    user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
+                                    &saved_value_, logger_);
     RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
   }
 }
@@ -479,10 +415,10 @@ void DBIter::Prev() {
   }
   PrevInternal();
   if (statistics_ != nullptr) {
-    local_stats_.prev_count_++;
+    RecordTick(statistics_, NUMBER_DB_PREV);
     if (valid_) {
-      local_stats_.prev_found_count_++;
-      local_stats_.bytes_read_ += (key().size() + value().size());
+      RecordTick(statistics_, NUMBER_DB_PREV_FOUND);
+      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
     }
   }
   if (valid_ && prefix_extractor_ && prefix_same_as_start_ &&
@@ -559,7 +495,7 @@ void DBIter::PrevInternal() {
 // saved_value_
 bool DBIter::FindValueForCurrentKey() {
   assert(iter_->Valid());
-  merge_context_.Clear();
+  merge_operands_.clear();
   // last entry before merge (could be kTypeDeletion, kTypeSingleDeletion or
   // kTypeValue)
   ValueType last_not_merge_type = kTypeDeletion;
@@ -579,19 +515,19 @@ bool DBIter::FindValueForCurrentKey() {
     last_key_entry_type = ikey.type;
     switch (last_key_entry_type) {
       case kTypeValue:
-        merge_context_.Clear();
+        merge_operands_.clear();
         saved_value_ = iter_->value().ToString();
         last_not_merge_type = kTypeValue;
         break;
       case kTypeDeletion:
       case kTypeSingleDeletion:
-        merge_context_.Clear();
+        merge_operands_.clear();
         last_not_merge_type = last_key_entry_type;
         PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
         break;
       case kTypeMerge:
         assert(user_merge_operator_ != nullptr);
-        merge_context_.PushOperandBack(iter_->value());
+        merge_operands_.push_back(iter_->value().ToString());
         break;
       default:
         assert(false);
@@ -614,8 +550,8 @@ bool DBIter::FindValueForCurrentKey() {
         StopWatchNano timer(env_, statistics_ != nullptr);
         PERF_TIMER_GUARD(merge_operator_time_nanos);
         user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
-                                        merge_context_.GetOperands(),
-                                        &saved_value_, logger_);
+                                        merge_operands_, &saved_value_,
+                                        logger_);
         RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                    timer.ElapsedNanos());
       } else {
@@ -626,8 +562,8 @@ bool DBIter::FindValueForCurrentKey() {
           StopWatchNano timer(env_, statistics_ != nullptr);
           PERF_TIMER_GUARD(merge_operator_time_nanos);
           user_merge_operator_->FullMerge(saved_key_.GetKey(), &temp_slice,
-                                          merge_context_.GetOperands(),
-                                          &saved_value_, logger_);
+                                          merge_operands_, &saved_value_,
+                                          logger_);
           RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                      timer.ElapsedNanos());
         }
@@ -670,11 +606,11 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
 
   // kTypeMerge. We need to collect all kTypeMerge values and save them
   // in operands
-  merge_context_.Clear();
+  std::deque<std::string> operands;
   while (iter_->Valid() &&
          user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) &&
          ikey.type == kTypeMerge) {
-    merge_context_.PushOperand(iter_->value());
+    operands.push_front(iter_->value().ToString());
     iter_->Next();
     FindParseableKey(&ikey, kForward);
   }
@@ -685,8 +621,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
     {
       StopWatchNano timer(env_, statistics_ != nullptr);
       PERF_TIMER_GUARD(merge_operator_time_nanos);
-      user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
-                                      merge_context_.GetOperands(),
+      user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
                                       &saved_value_, logger_);
       RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
     }
@@ -704,9 +639,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   {
     StopWatchNano timer(env_, statistics_ != nullptr);
     PERF_TIMER_GUARD(merge_operator_time_nanos);
-    user_merge_operator_->FullMerge(saved_key_.GetKey(), &val,
-                                    merge_context_.GetOperands(), &saved_value_,
-                                    logger_);
+    user_merge_operator_->FullMerge(saved_key_.GetKey(), &val, operands,
+                                    &saved_value_, logger_);
     RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
   }
   valid_ = true;
@@ -884,13 +818,12 @@ Iterator* NewDBIterator(Env* env, const ImmutableCFOptions& ioptions,
                         InternalIterator* internal_iter,
                         const SequenceNumber& sequence,
                         uint64_t max_sequential_skip_in_iterations,
-                        uint64_t version_number,
                         const Slice* iterate_upper_bound,
                         bool prefix_same_as_start, bool pin_data) {
   DBIter* db_iter =
       new DBIter(env, ioptions, user_key_comparator, internal_iter, sequence,
-                 false, max_sequential_skip_in_iterations, version_number,
-                 iterate_upper_bound, prefix_same_as_start);
+                 false, max_sequential_skip_in_iterations, iterate_upper_bound,
+                 prefix_same_as_start);
   if (pin_data) {
     db_iter->PinData();
   }
@@ -917,13 +850,12 @@ inline Slice ArenaWrappedDBIter::key() const { return db_iter_->key(); }
 inline Slice ArenaWrappedDBIter::value() const { return db_iter_->value(); }
 inline Status ArenaWrappedDBIter::status() const { return db_iter_->status(); }
 inline Status ArenaWrappedDBIter::PinData() { return db_iter_->PinData(); }
-inline Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
-                                              std::string* prop) {
-  return db_iter_->GetProperty(prop_name, prop);
-}
 inline Status ArenaWrappedDBIter::ReleasePinnedData() {
   return db_iter_->ReleasePinnedData();
 }
+inline bool ArenaWrappedDBIter::IsKeyPinned() const {
+  return db_iter_->IsKeyPinned();
+}
 void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1,
                                          void* arg2) {
   db_iter_->RegisterCleanup(function, arg1, arg2);
@@ -932,7 +864,7 @@ void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1,
 ArenaWrappedDBIter* NewArenaWrappedDbIterator(
     Env* env, const ImmutableCFOptions& ioptions,
     const Comparator* user_key_comparator, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+    uint64_t max_sequential_skip_in_iterations,
     const Slice* iterate_upper_bound, bool prefix_same_as_start,
     bool pin_data) {
   ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
@@ -940,7 +872,7 @@ ArenaWrappedDBIter* NewArenaWrappedDbIterator(
   auto mem = arena->AllocateAligned(sizeof(DBIter));
   DBIter* db_iter =
       new (mem) DBIter(env, ioptions, user_key_comparator, nullptr, sequence,
-                       true, max_sequential_skip_in_iterations, version_number,
+                       true, max_sequential_skip_in_iterations,
                        iterate_upper_bound, prefix_same_as_start);
 
   iter->SetDBIter(db_iter);

db/db_iter.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -9,7 +9,6 @@
 
 #pragma once
 #include <stdint.h>
-#include <string>
 #include "rocksdb/db.h"
 #include "rocksdb/iterator.h"
 #include "db/dbformat.h"
@@ -25,12 +24,14 @@ class InternalIterator;
 // Return a new iterator that converts internal keys (yielded by
 // "*internal_iter") that were live at the specified "sequence" number
 // into appropriate user keys.
-extern Iterator* NewDBIterator(
-    Env* env, const ImmutableCFOptions& options,
-    const Comparator* user_key_comparator, InternalIterator* internal_iter,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
-    uint64_t version_number, const Slice* iterate_upper_bound = nullptr,
-    bool prefix_same_as_start = false, bool pin_data = false);
+extern Iterator* NewDBIterator(Env* env, const ImmutableCFOptions& options,
+                               const Comparator* user_key_comparator,
+                               InternalIterator* internal_iter,
+                               const SequenceNumber& sequence,
+                               uint64_t max_sequential_skip_in_iterations,
+                               const Slice* iterate_upper_bound = nullptr,
+                               bool prefix_same_as_start = false,
+                               bool pin_data = false);
 
 // A wrapper iterator which wraps DB Iterator and the arena, with which the DB
 // iterator is supposed be allocated. This class is used as an entry point of
@@ -65,7 +66,7 @@ class ArenaWrappedDBIter : public Iterator {
   void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2);
   virtual Status PinData();
   virtual Status ReleasePinnedData();
-  virtual Status GetProperty(std::string prop_name, std::string* prop) override;
+  virtual bool IsKeyPinned() const override;
 
  private:
   DBIter* db_iter_;
@@ -76,7 +77,7 @@ class ArenaWrappedDBIter : public Iterator {
 extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
     Env* env, const ImmutableCFOptions& options,
     const Comparator* user_key_comparator, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+    uint64_t max_sequential_skip_in_iterations,
     const Slice* iterate_upper_bound = nullptr,
     bool prefix_same_as_start = false, bool pin_data = false);
 

db/db_iter_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -181,9 +181,10 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     internal_iter->AddPut("b", "val_b");
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 10,
+                      options.max_sequential_skip_in_iterations));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -214,7 +215,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0));
+        10, options.max_sequential_skip_in_iterations));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -240,8 +241,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -273,8 +273,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -309,8 +308,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
@@ -339,8 +337,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        7, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        7, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     SetPerfLevel(kEnableCount);
     ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -377,8 +374,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        4, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -403,8 +399,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
@@ -426,8 +421,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -462,8 +456,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        7, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        7, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     SetPerfLevel(kEnableCount);
     ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -489,9 +482,10 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     internal_iter->AddPut("b", "val_b");
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 10,
+                      options.max_sequential_skip_in_iterations));
 
     db_iter->SeekToFirst();
     ASSERT_TRUE(db_iter->Valid());
@@ -530,9 +524,10 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     internal_iter->AddPut("b", "val_b");
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        2, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 2,
+                      options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -560,9 +555,10 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     internal_iter->AddPut("c", "val_c");
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 10,
+                      options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -587,9 +583,10 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {
     TestIterator* internal_iter = new TestIterator(BytewiseComparator());
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 0,
+                      options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
   }
@@ -598,9 +595,10 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {
     TestIterator* internal_iter = new TestIterator(BytewiseComparator());
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 0,
+                      options.max_sequential_skip_in_iterations));
     db_iter->SeekToFirst();
     ASSERT_TRUE(!db_iter->Valid());
   }
@@ -619,9 +617,10 @@ TEST_F(DBIteratorTest, DBIteratorUseSkipCountSkips) {
   }
   internal_iter->Finish();
 
-  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 2,
-      options.max_sequential_skip_in_iterations, 0));
+  std::unique_ptr<Iterator> db_iter(
+      NewDBIterator(env_, ImmutableCFOptions(options),
+                    BytewiseComparator(), internal_iter, 2,
+                    options.max_sequential_skip_in_iterations));
   db_iter->SeekToLast();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -660,8 +659,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
 
       options.statistics = rocksdb::CreateDBStatistics();
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
+          env_, ImmutableCFOptions(options),
+          BytewiseComparator(), internal_iter, i + 2,
+          options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -695,8 +695,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->Finish();
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
+          env_, ImmutableCFOptions(options),
+          BytewiseComparator(), internal_iter, i + 2,
+          options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -723,8 +724,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->Finish();
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, 202, options.max_sequential_skip_in_iterations, 0));
+          env_, ImmutableCFOptions(options),
+          BytewiseComparator(), internal_iter, 202,
+          options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -754,9 +756,10 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       }
       internal_iter->AddPut("c", "200");
       internal_iter->Finish();
-      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, i, options.max_sequential_skip_in_iterations, 0));
+      std::unique_ptr<Iterator> db_iter(
+          NewDBIterator(env_, ImmutableCFOptions(options),
+                        BytewiseComparator(), internal_iter, i,
+                        options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(!db_iter->Valid());
 
@@ -770,9 +773,10 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
     }
     internal_iter->AddPut("c", "200");
     internal_iter->Finish();
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        200, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 200,
+                      options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -805,8 +809,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->Finish();
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
+          env_, ImmutableCFOptions(options),
+          BytewiseComparator(), internal_iter, i + 2,
+          options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -839,8 +844,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->Finish();
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
+          env_, ImmutableCFOptions(options),
+          BytewiseComparator(), internal_iter, i + 2,
+          options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -881,7 +887,7 @@ TEST_F(DBIteratorTest, DBIterator1) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 1,
-      options.max_sequential_skip_in_iterations, 0));
+      options.max_sequential_skip_in_iterations));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -907,7 +913,7 @@ TEST_F(DBIteratorTest, DBIterator2) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 0,
-      options.max_sequential_skip_in_iterations, 0));
+      options.max_sequential_skip_in_iterations));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -930,7 +936,7 @@ TEST_F(DBIteratorTest, DBIterator3) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 2,
-      options.max_sequential_skip_in_iterations, 0));
+      options.max_sequential_skip_in_iterations));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -952,7 +958,7 @@ TEST_F(DBIteratorTest, DBIterator4) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 4,
-      options.max_sequential_skip_in_iterations, 0));
+      options.max_sequential_skip_in_iterations));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -981,7 +987,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations, 0));
+        0, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1003,7 +1009,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        1, options.max_sequential_skip_in_iterations, 0));
+        1, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1025,7 +1031,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        2, options.max_sequential_skip_in_iterations, 0));
+        2, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1047,7 +1053,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        3, options.max_sequential_skip_in_iterations, 0));
+        3, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1069,7 +1075,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations, 0));
+        4, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1091,7 +1097,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        5, options.max_sequential_skip_in_iterations, 0));
+        5, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1113,7 +1119,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        6, options.max_sequential_skip_in_iterations, 0));
+        6, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1139,7 +1145,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations, 0));
+        0, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1161,7 +1167,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        1, options.max_sequential_skip_in_iterations, 0));
+        1, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1183,7 +1189,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        2, options.max_sequential_skip_in_iterations, 0));
+        2, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1205,7 +1211,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        3, options.max_sequential_skip_in_iterations, 0));
+        3, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
   }
@@ -1223,7 +1229,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations, 0));
+        4, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1245,7 +1251,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        5, options.max_sequential_skip_in_iterations, 0));
+        5, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1267,7 +1273,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        6, options.max_sequential_skip_in_iterations, 0));
+        6, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1305,7 +1311,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations, 0));
+        0, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1339,7 +1345,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        2, options.max_sequential_skip_in_iterations, 0));
+        2, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1379,7 +1385,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations, 0));
+        4, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1419,7 +1425,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        5, options.max_sequential_skip_in_iterations, 0));
+        5, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1464,7 +1470,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        6, options.max_sequential_skip_in_iterations, 0));
+        6, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1510,7 +1516,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        7, options.max_sequential_skip_in_iterations, 0));
+        7, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1550,7 +1556,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        9, options.max_sequential_skip_in_iterations, 0));
+        9, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1596,7 +1602,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        13, options.max_sequential_skip_in_iterations, 0));
+        13, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1643,7 +1649,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        14, options.max_sequential_skip_in_iterations, 0));
+        14, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1672,7 +1678,7 @@ TEST_F(DBIteratorTest, DBIterator8) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-      10, options.max_sequential_skip_in_iterations, 0));
+      10, options.max_sequential_skip_in_iterations));
   db_iter->SeekToLast();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -1701,7 +1707,7 @@ TEST_F(DBIteratorTest, DBIterator9) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0));
+        10, options.max_sequential_skip_in_iterations));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -1748,7 +1754,7 @@ TEST_F(DBIteratorTest, DBIterator10) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-      10, options.max_sequential_skip_in_iterations, 0));
+      10, options.max_sequential_skip_in_iterations));
 
   db_iter->Seek("c");
   ASSERT_TRUE(db_iter->Valid());
@@ -1772,9 +1778,9 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {
   internal_iter->AddPut("b", "2");
   internal_iter->Finish();
 
-  std::unique_ptr<Iterator> db_iter(
-      NewDBIterator(env_, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0 /* force seek */, 0));
+  std::unique_ptr<Iterator> db_iter(NewDBIterator(
+      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
+      10, 0 /* force seek */));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1801,7 +1807,7 @@ TEST_F(DBIteratorTest, DBIterator11) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 1,
-      options.max_sequential_skip_in_iterations, 0));
+      options.max_sequential_skip_in_iterations));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1826,7 +1832,7 @@ TEST_F(DBIteratorTest, DBIterator12) {
 
   std::unique_ptr<Iterator> db_iter(
       NewDBIterator(env_, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0, 0));
+                    internal_iter, 10, 0));
   db_iter->SeekToLast();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -1868,7 +1874,7 @@ class DBIterWithMergeIterTest : public testing::Test {
     db_iter_.reset(NewDBIterator(env_, ImmutableCFOptions(options_),
                                  BytewiseComparator(), merge_iter,
                                  8 /* read data earlier than seqId 8 */,
-                                 3 /* max iterators before reseek */, 0));
+                                 3 /* max iterators before reseek */));
   }
 
   Env* env_;

db/db_log_iter_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_properties_test.cc

@@ -1,4 +1,4 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+// Copyright (c) 2016, Facebook, Inc.  All rights reserved.
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
@@ -90,21 +90,6 @@ TEST_F(DBPropertiesTest, Empty) {
   } while (ChangeOptions());
 }
 
-TEST_F(DBPropertiesTest, CurrentVersionNumber) {
-  uint64_t v1, v2, v3;
-  ASSERT_TRUE(
-      dbfull()->GetIntProperty("rocksdb.current-super-version-number", &v1));
-  Put("12345678", "");
-  ASSERT_TRUE(
-      dbfull()->GetIntProperty("rocksdb.current-super-version-number", &v2));
-  Flush();
-  ASSERT_TRUE(
-      dbfull()->GetIntProperty("rocksdb.current-super-version-number", &v3));
-
-  ASSERT_EQ(v1, v2);
-  ASSERT_GT(v3, v2);
-}
-
 TEST_F(DBPropertiesTest, GetAggregatedIntPropertyTest) {
   const int kKeySize = 100;
   const int kValueSize = 500;
@@ -253,35 +238,6 @@ TEST_F(DBPropertiesTest, ValidatePropertyInfo) {
   }
 }
 
-TEST_F(DBPropertiesTest, ValidateSampleNumber) {
-  // When "max_open_files" is -1, we read all the files for
-  // "rocksdb.estimate-num-keys" computation, which is the ground truth.
-  // Otherwise, we sample 20 newest files to make an estimation.
-  // Formula: lastest_20_files_active_key_ratio * total_files
-  Options options = CurrentOptions();
-  options.disable_auto_compactions = true;
-  options.level0_stop_writes_trigger = 1000;
-  DestroyAndReopen(options);
-  int key = 0;
-  for (int files = 20; files >= 10; files -= 10) {
-    for (int i = 0; i < files; i++) {
-      int rows = files / 10;
-      for (int j = 0; j < rows; j++) {
-        db_->Put(WriteOptions(), std::to_string(++key), "foo");
-      }
-      db_->Flush(FlushOptions());
-    }
-  }
-  std::string num;
-  Reopen(options);
-  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
-  ASSERT_EQ("45", num);
-  options.max_open_files = -1;
-  Reopen(options);
-  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
-  ASSERT_EQ("50", num);
-}
-
 TEST_F(DBPropertiesTest, AggregatedTableProperties) {
   for (int kTableCount = 40; kTableCount <= 100; kTableCount += 30) {
     const int kKeysPerTable = 100;

db/db_table_properties_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -215,34 +215,6 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfTablesInRange) {
     TestGetPropertiesOfTablesInRange(std::move(ranges));
   }
 }
-
-TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) {
-  std::string kExtraCfName = "pikachu";
-  CreateAndReopenWithCF({kExtraCfName}, CurrentOptions());
-
-  // Create one table per CF, then verify it was created with the column family
-  // name property.
-  for (int cf = 0; cf < 2; ++cf) {
-    Put(cf, "key", "val");
-    Flush(cf);
-
-    TablePropertiesCollection fname_to_props;
-    ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props));
-    ASSERT_EQ(1U, fname_to_props.size());
-
-    std::string expected_cf_name;
-    if (cf > 0) {
-      expected_cf_name = kExtraCfName;
-    } else {
-      expected_cf_name = kDefaultColumnFamilyName;
-    }
-    ASSERT_EQ(expected_cf_name,
-              fname_to_props.begin()->second->column_family_name);
-    ASSERT_EQ(cf, static_cast<uint32_t>(
-                      fname_to_props.begin()->second->column_family_id));
-  }
-}
-
 }  // namespace rocksdb
 
 #endif  // ROCKSDB_LITE

db/db_tailing_iter_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -654,51 +654,6 @@ TEST_F(DBTestTailingIterator, ManagedTailingIteratorSeekToSame) {
   ASSERT_EQ(found, iter->key().ToString());
 }
 
-TEST_F(DBTestTailingIterator, ForwardIteratorVersionProperty) {
-  Options options = CurrentOptions();
-  options.write_buffer_size = 1000;
-
-  ReadOptions read_options;
-  read_options.tailing = true;
-
-  Put("foo", "bar");
-
-  uint64_t v1, v2, v3, v4;
-  {
-    std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
-    iter->Seek("foo");
-    std::string prop_value;
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
-                                &prop_value));
-    v1 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-    Put("foo1", "bar1");
-    Flush();
-
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
-                                &prop_value));
-    v2 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-    iter->Seek("f");
-
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
-                                &prop_value));
-    v3 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-    ASSERT_EQ(v1, v2);
-    ASSERT_GT(v3, v2);
-  }
-
-  {
-    std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
-    iter->Seek("foo");
-    std::string prop_value;
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
-                                &prop_value));
-    v4 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-  }
-  ASSERT_EQ(v3, v4);
-}
 }  // namespace rocksdb
 
 #endif  // !defined(ROCKSDB_LITE)

db/db_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -228,17 +228,20 @@ TEST_F(DBTest, MemEnvTest) {
 #endif  // ROCKSDB_LITE
 
 TEST_F(DBTest, WriteEmptyBatch) {
-  Options options = CurrentOptions();
+  Options options;
   options.env = env_;
   options.write_buffer_size = 100000;
+  options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
   ASSERT_OK(Put(1, "foo", "bar"));
+  env_->sync_counter_.store(0);
   WriteOptions wo;
   wo.sync = true;
   wo.disableWAL = false;
   WriteBatch empty_batch;
   ASSERT_OK(dbfull()->Write(wo, &empty_batch));
+  ASSERT_GE(env_->sync_counter_.load(), 1);
 
   // make sure we can re-open it.
   ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options));
@@ -280,12 +283,13 @@ TEST_F(DBTest, ReadOnlyDB) {
 
 TEST_F(DBTest, CompactedDB) {
   const uint64_t kFileSize = 1 << 20;
-  Options options = CurrentOptions();
+  Options options;
   options.disable_auto_compactions = true;
   options.write_buffer_size = kFileSize;
   options.target_file_size_base = kFileSize;
   options.max_bytes_for_level_base = 1 << 30;
   options.compression = kNoCompression;
+  options = CurrentOptions(options);
   Reopen(options);
   // 1 L0 file, use CompactedDB if max_open_files = -1
   ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, '1')));
@@ -630,27 +634,6 @@ TEST_F(DBTest, ReadFromPersistedTier) {
   } while (ChangeOptions(kSkipHashCuckoo));
 }
 
-TEST_F(DBTest, IteratorProperty) {
-  // The test needs to be changed if kPersistedTier is supported in iterator.
-  Options options = CurrentOptions();
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Put(1, "1", "2");
-  ReadOptions ropt;
-  ropt.pin_data = false;
-  {
-    unique_ptr<Iterator> iter(db_->NewIterator(ropt, handles_[1]));
-    iter->SeekToFirst();
-    std::string prop_value;
-    ASSERT_NOK(iter->GetProperty("non_existing.value", &prop_value));
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
-    ASSERT_EQ("0", prop_value);
-    iter->Next();
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
-    ASSERT_EQ("Iterator is not valid.", prop_value);
-  }
-  Close();
-}
-
 TEST_F(DBTest, PersistedTierOnIterator) {
   // The test needs to be changed if kPersistedTier is supported in iterator.
   Options options = CurrentOptions();
@@ -762,10 +745,11 @@ TEST_F(DBTest, DISABLED_VeryLargeValue) {
   std::string key1(kKeySize, 'c');
   std::string key2(kKeySize, 'd');
 
-  Options options = CurrentOptions();
+  Options options;
   options.env = env_;
   options.write_buffer_size = 100000;  // Small write buffer
   options.paranoid_checks = true;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   ASSERT_OK(Put("boo", "v1"));
@@ -808,8 +792,10 @@ TEST_F(DBTest, DISABLED_VeryLargeValue) {
 
 TEST_F(DBTest, GetFromImmutableLayer) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.env = env_;
+    options.write_buffer_size = 100000;  // Small write buffer
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     ASSERT_OK(Put(1, "foo", "v1"));
@@ -1911,9 +1897,11 @@ TEST_F(DBTest, Recover) {
 
 TEST_F(DBTest, RecoverWithTableHandle) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
+    options.write_buffer_size = 100;
     options.disable_auto_compactions = true;
+    options = CurrentOptions(options);
     DestroyAndReopen(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -2361,9 +2349,10 @@ TEST_F(DBTest, CompressedCache) {
   // Iteration 4: both block cache and compressed cache, but DB is not
   // compressed
   for (int iter = 0; iter < 4; iter++) {
-    Options options = CurrentOptions();
+    Options options;
     options.write_buffer_size = 64*1024;        // small write buffer
     options.statistics = rocksdb::CreateDBStatistics();
+    options = CurrentOptions(options);
 
     BlockBasedTableOptions table_options;
     switch (iter) {
@@ -2628,9 +2617,10 @@ TEST_F(DBTest, MinLevelToCompress2) {
 
 TEST_F(DBTest, RepeatedWritesToSameKey) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.env = env_;
     options.write_buffer_size = 100000;  // Small write buffer
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // We must have at most one file per level except for level-0,
@@ -2707,10 +2697,11 @@ static bool Between(uint64_t val, uint64_t low, uint64_t high) {
 }
 
 TEST_F(DBTest, ApproximateSizesMemTable) {
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 100000000;  // Large write buffer
   options.compression = kNoCompression;
   options.create_if_missing = true;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   const int N = 128;
@@ -2813,10 +2804,11 @@ TEST_F(DBTest, ApproximateSizesMemTable) {
 
 TEST_F(DBTest, ApproximateSizes) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.write_buffer_size = 100000000;        // Large write buffer
     options.compression = kNoCompression;
     options.create_if_missing = true;
+    options = CurrentOptions(options);
     DestroyAndReopen(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -7832,7 +7824,8 @@ TEST_F(DBTest, MergeTestTime) {
   this->env_->addon_time_.store(0);
   this->env_->time_elapse_only_sleep_ = true;
   this->env_->no_sleep_ = true;
-  Options options = CurrentOptions();
+  Options options;
+  options = CurrentOptions(options);
   options.statistics = rocksdb::CreateDBStatistics();
   options.merge_operator.reset(new DelayedMergeOperator(this));
   DestroyAndReopen(options);
@@ -7872,7 +7865,8 @@ TEST_F(DBTest, MergeTestTime) {
 #ifndef ROCKSDB_LITE
 TEST_P(DBTestWithParam, MergeCompactionTimeTest) {
   SetPerfLevel(kEnableTime);
-  Options options = CurrentOptions();
+  Options options;
+  options = CurrentOptions(options);
   options.compaction_filter_factory = std::make_shared<KeepFilterFactory>();
   options.statistics = rocksdb::CreateDBStatistics();
   options.merge_operator.reset(new DelayedMergeOperator(this));
@@ -7891,13 +7885,14 @@ TEST_P(DBTestWithParam, MergeCompactionTimeTest) {
 }
 
 TEST_P(DBTestWithParam, FilterCompactionTimeTest) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_filter_factory =
       std::make_shared<DelayFilterFactory>(this);
   options.disable_auto_compactions = true;
   options.create_if_missing = true;
   options.statistics = rocksdb::CreateDBStatistics();
   options.max_subcompactions = max_subcompactions_;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   // put some data
@@ -7939,8 +7934,9 @@ TEST_F(DBTest, TestLogCleanup) {
 
 #ifndef ROCKSDB_LITE
 TEST_F(DBTest, EmptyCompactedDB) {
-  Options options = CurrentOptions();
+  Options options;
   options.max_open_files = -1;
+  options = CurrentOptions(options);
   Close();
   ASSERT_OK(ReadOnlyReopen(options));
   Status s = Put("new", "value");
@@ -8176,8 +8172,9 @@ TEST_F(DBTest, AutomaticConflictsWithManualCompaction) {
 // Github issue #595
 // Large write batch with column families
 TEST_F(DBTest, LargeBatchWithColumnFamilies) {
-  Options options = CurrentOptions();
+  Options options;
   options.env = env_;
+  options = CurrentOptions(options);
   options.write_buffer_size = 100000;  // Small write buffer
   CreateAndReopenWithCF({"pikachu"}, options);
   int64_t j = 0;
@@ -8271,10 +8268,11 @@ TEST_F(DBTest, DelayedWriteRate) {
   const int kEntriesPerMemTable = 100;
   const int kTotalFlushes = 20;
 
-  Options options = CurrentOptions();
+  Options options;
   env_->SetBackgroundThreads(1, Env::LOW);
   options.env = env_;
   env_->no_sleep_ = true;
+  options = CurrentOptions(options);
   options.write_buffer_size = 100000000;
   options.max_write_buffer_number = 256;
   options.max_background_compactions = 1;
@@ -8335,9 +8333,10 @@ TEST_F(DBTest, DelayedWriteRate) {
 }
 
 TEST_F(DBTest, HardLimit) {
-  Options options = CurrentOptions();
+  Options options;
   options.env = env_;
   env_->SetBackgroundThreads(1, Env::LOW);
+  options = CurrentOptions(options);
   options.max_write_buffer_number = 256;
   options.write_buffer_size = 110 << 10;  // 110KB
   options.arena_block_size = 4 * 1024;
@@ -8385,8 +8384,9 @@ TEST_F(DBTest, HardLimit) {
 
 #ifndef ROCKSDB_LITE
 TEST_F(DBTest, SoftLimit) {
-  Options options = CurrentOptions();
+  Options options;
   options.env = env_;
+  options = CurrentOptions(options);
   options.write_buffer_size = 100000;  // Small write buffer
   options.max_write_buffer_number = 256;
   options.level0_file_num_compaction_trigger = 1;
@@ -8502,8 +8502,9 @@ TEST_F(DBTest, SoftLimit) {
 }
 
 TEST_F(DBTest, LastWriteBufferDelay) {
-  Options options = CurrentOptions();
+  Options options;
   options.env = env_;
+  options = CurrentOptions(options);
   options.write_buffer_size = 100000;
   options.max_write_buffer_number = 4;
   options.delayed_write_rate = 20000;
@@ -8637,6 +8638,7 @@ TEST_F(DBTest, DeletingOldWalAfterDrop) {
   EXPECT_GT(lognum2, lognum1);
 }
 
+#ifndef ROCKSDB_LITE
 TEST_F(DBTest, DBWithSstFileManager) {
   std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_));
   auto sfm = static_cast<SstFileManagerImpl*>(sst_file_manager.get());
@@ -8701,7 +8703,6 @@ TEST_F(DBTest, DBWithSstFileManager) {
   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
 
-#ifndef ROCKSDB_LITE
 TEST_F(DBTest, RateLimitedDelete) {
   rocksdb::SyncPoint::GetInstance()->LoadDependency({
       {"DBTest::RateLimitedDelete:1", "DeleteScheduler::BackgroundEmptyTrash"},
@@ -8874,102 +8875,6 @@ TEST_F(DBTest, DestroyDBWithRateLimitedDelete) {
 }
 #endif  // ROCKSDB_LITE
 
-TEST_F(DBTest, DBWithMaxSpaceAllowed) {
-  std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_));
-  auto sfm = static_cast<SstFileManagerImpl*>(sst_file_manager.get());
-
-  Options options = CurrentOptions();
-  options.sst_file_manager = sst_file_manager;
-  options.disable_auto_compactions = true;
-  DestroyAndReopen(options);
-
-  Random rnd(301);
-
-  // Generate a file containing 100 keys.
-  for (int i = 0; i < 100; i++) {
-    ASSERT_OK(Put(Key(i), RandomString(&rnd, 50)));
-  }
-  ASSERT_OK(Flush());
-
-  uint64_t first_file_size = 0;
-  auto files_in_db = GetAllSSTFiles(&first_file_size);
-  ASSERT_EQ(sfm->GetTotalSize(), first_file_size);
-
-  // Set the maximum allowed space usage to the current total size
-  sfm->SetMaxAllowedSpaceUsage(first_file_size + 1);
-
-  ASSERT_OK(Put("key1", "val1"));
-  // This flush will cause bg_error_ and will fail
-  ASSERT_NOK(Flush());
-}
-
-TEST_F(DBTest, DBWithMaxSpaceAllowedRandomized) {
-  // This test will set a maximum allowed space for the DB, then it will
-  // keep filling the DB until the limit is reached and bg_error_ is set.
-  // When bg_error_ is set we will verify that the DB size is greater
-  // than the limit.
-
-  std::vector<int> max_space_limits_mbs = {1, 2, 4, 8, 10};
-
-  bool bg_error_set = false;
-  uint64_t total_sst_files_size = 0;
-
-  int reached_max_space_on_flush = 0;
-  int reached_max_space_on_compaction = 0;
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached",
-      [&](void* arg) {
-        bg_error_set = true;
-        GetAllSSTFiles(&total_sst_files_size);
-        reached_max_space_on_flush++;
-      });
-
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "CompactionJob::FinishCompactionOutputFile:MaxAllowedSpaceReached",
-      [&](void* arg) {
-        bg_error_set = true;
-        GetAllSSTFiles(&total_sst_files_size);
-        reached_max_space_on_compaction++;
-      });
-
-  for (auto limit_mb : max_space_limits_mbs) {
-    bg_error_set = false;
-    total_sst_files_size = 0;
-    rocksdb::SyncPoint::GetInstance()->ClearTrace();
-    rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-    std::shared_ptr<SstFileManager> sst_file_manager(NewSstFileManager(env_));
-    auto sfm = static_cast<SstFileManagerImpl*>(sst_file_manager.get());
-
-    Options options = CurrentOptions();
-    options.sst_file_manager = sst_file_manager;
-    options.write_buffer_size = 1024 * 512;  // 512 Kb
-    DestroyAndReopen(options);
-    Random rnd(301);
-
-    sfm->SetMaxAllowedSpaceUsage(limit_mb * 1024 * 1024);
-
-    int keys_written = 0;
-    uint64_t estimated_db_size = 0;
-    while (true) {
-      auto s = Put(RandomString(&rnd, 10), RandomString(&rnd, 50));
-      if (!s.ok()) {
-        break;
-      }
-      keys_written++;
-      // Check the estimated db size vs the db limit just to make sure we
-      // dont run into an infinite loop
-      estimated_db_size = keys_written * 60;  // ~60 bytes per key
-      ASSERT_LT(estimated_db_size, limit_mb * 1024 * 1024 * 2);
-    }
-    ASSERT_TRUE(bg_error_set);
-    ASSERT_GE(total_sst_files_size, limit_mb * 1024 * 1024);
-    rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  }
-
-  ASSERT_GT(reached_max_space_on_flush, 0);
-  ASSERT_GT(reached_max_space_on_compaction, 0);
-}
-
 TEST_F(DBTest, UnsupportedManualSync) {
   DestroyAndReopen(CurrentOptions());
   env_->is_wal_sync_thread_safe_.store(false);
@@ -9389,58 +9294,6 @@ TEST_F(DBTest, AddExternalSstFile) {
                          kSkipFIFOCompaction));
 }
 
-// This test reporduce a bug that can happen in some cases if the DB started
-// purging obsolete files when we are adding an external sst file.
-// This situation may result in deleting the file while it's being added.
-TEST_F(DBTest, AddExternalSstFilePurgeObsoleteFilesBug) {
-  std::string sst_files_folder = test::TmpDir(env_) + "/sst_files/";
-  env_->CreateDir(sst_files_folder);
-  Options options = CurrentOptions();
-  options.env = env_;
-  const ImmutableCFOptions ioptions(options);
-
-  SstFileWriter sst_file_writer(EnvOptions(), ioptions, options.comparator);
-
-  // file1.sst (0 => 500)
-  std::string sst_file_path = sst_files_folder + "file1.sst";
-  Status s = sst_file_writer.Open(sst_file_path);
-  ASSERT_OK(s);
-  for (int i = 0; i < 500; i++) {
-    std::string k = Key(i);
-    s = sst_file_writer.Add(k, k + "_val");
-    ASSERT_OK(s);
-  }
-
-  ExternalSstFileInfo sst_file_info;
-  s = sst_file_writer.Finish(&sst_file_info);
-  ASSERT_OK(s);
-
-  options.delete_obsolete_files_period_micros = 0;
-  options.disable_auto_compactions = true;
-  DestroyAndReopen(options);
-
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::AddFile:FileCopied", [&](void* arg) {
-        ASSERT_OK(Put("aaa", "bbb"));
-        ASSERT_OK(Flush());
-        ASSERT_OK(Put("aaa", "xxx"));
-        ASSERT_OK(Flush());
-        db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  s = db_->AddFile(sst_file_path);
-  ASSERT_OK(s);
-
-  for (int i = 0; i < 500; i++) {
-    std::string k = Key(i);
-    std::string v = k + "_val";
-    ASSERT_EQ(Get(k), v);
-  }
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-}
-
 TEST_F(DBTest, AddExternalSstFileNoCopy) {
   std::string sst_files_folder = test::TmpDir(env_) + "/sst_files/";
   env_->CreateDir(sst_files_folder);
@@ -9842,10 +9695,7 @@ TEST_F(DBTest, PinnedDataIteratorRandomized) {
           ASSERT_EQ(true_data.lower_bound(k), true_data.end());
           continue;
         }
-        std::string prop_value;
-        ASSERT_OK(
-            iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
-        ASSERT_EQ("1", prop_value);
+        ASSERT_TRUE(iter->IsKeyPinned());
         keys_slices.push_back(iter->key());
         true_keys.push_back(true_data.lower_bound(k)->first);
       }
@@ -9860,10 +9710,7 @@ TEST_F(DBTest, PinnedDataIteratorRandomized) {
       printf("Testing iterating forward on all keys\n");
       std::vector<Slice> all_keys;
       for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-        std::string prop_value;
-        ASSERT_OK(
-            iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
-        ASSERT_EQ("1", prop_value);
+        ASSERT_TRUE(iter->IsKeyPinned());
         all_keys.push_back(iter->key());
       }
       ASSERT_EQ(all_keys.size(), true_data.size());
@@ -9881,10 +9728,7 @@ TEST_F(DBTest, PinnedDataIteratorRandomized) {
       printf("Testing iterating backward on all keys\n");
       std::vector<Slice> all_keys;
       for (iter->SeekToLast(); iter->Valid(); iter->Prev()) {
-        std::string prop_value;
-        ASSERT_OK(
-            iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
-        ASSERT_EQ("1", prop_value);
+        ASSERT_TRUE(iter->IsKeyPinned());
         all_keys.push_back(iter->key());
       }
       ASSERT_EQ(all_keys.size(), true_data.size());
@@ -9955,9 +9799,7 @@ TEST_F(DBTest, PinnedDataIteratorMultipleFiles) {
 
   std::vector<std::pair<Slice, std::string>> results;
   for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    std::string prop_value;
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
-    ASSERT_EQ("1", prop_value);
+    ASSERT_TRUE(iter->IsKeyPinned());
     results.emplace_back(iter->key(), iter->value().ToString());
   }
 
@@ -10010,9 +9852,7 @@ TEST_F(DBTest, PinnedDataIteratorMergeOperator) {
 
   std::vector<std::pair<Slice, std::string>> results;
   for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    std::string prop_value;
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
-    ASSERT_EQ("1", prop_value);
+    ASSERT_TRUE(iter->IsKeyPinned());
     results.emplace_back(iter->key(), iter->value().ToString());
   }
 
@@ -10067,9 +9907,7 @@ TEST_F(DBTest, PinnedDataIteratorReadAfterUpdate) {
 
   std::vector<std::pair<Slice, std::string>> results;
   for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    std::string prop_value;
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
-    ASSERT_EQ("1", prop_value);
+    ASSERT_TRUE(iter->IsKeyPinned());
     results.emplace_back(iter->key(), iter->value().ToString());
   }
 
@@ -10088,8 +9926,9 @@ INSTANTIATE_TEST_CASE_P(DBTestWithParam, DBTestWithParam,
                                            ::testing::Bool()));
 
 TEST_F(DBTest, PauseBackgroundWorkTest) {
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 100000;  // Small write buffer
+  options = CurrentOptions(options);
   Reopen(options);
 
   std::vector<std::thread> threads;
@@ -10113,6 +9952,403 @@ TEST_F(DBTest, PauseBackgroundWorkTest) {
   ASSERT_EQ(true, done.load());
 }
 
+#ifndef ROCKSDB_LITE
+namespace {
+void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
+                          const std::vector<Slice>& keys_must_not_exist) {
+  // Ensure that expected keys exist
+  std::vector<std::string> values;
+  if (keys_must_exist.size() > 0) {
+    std::vector<Status> status_list =
+        db->MultiGet(ReadOptions(), keys_must_exist, &values);
+    for (size_t i = 0; i < keys_must_exist.size(); i++) {
+      ASSERT_OK(status_list[i]);
+    }
+  }
+
+  // Ensure that given keys don't exist
+  if (keys_must_not_exist.size() > 0) {
+    std::vector<Status> status_list =
+        db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
+    for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
+      ASSERT_TRUE(status_list[i].IsNotFound());
+    }
+  }
+}
+
+}  // namespace
+
+TEST_F(DBTest, WalFilterTest) {
+  class TestWalFilter : public WalFilter {
+   private:
+    // Processing option that is requested to be applied at the given index
+    WalFilter::WalProcessingOption wal_processing_option_;
+    // Index at which to apply wal_processing_option_
+    // At other indexes default wal_processing_option::kContinueProcessing is
+    // returned.
+    size_t apply_option_at_record_index_;
+    // Current record index, incremented with each record encountered.
+    size_t current_record_index_;
+
+   public:
+    TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
+                  size_t apply_option_for_record_index)
+        : wal_processing_option_(wal_processing_option),
+          apply_option_at_record_index_(apply_option_for_record_index),
+          current_record_index_(0) {}
+
+    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
+                                          WriteBatch* new_batch,
+                                          bool* batch_changed) const override {
+      WalFilter::WalProcessingOption option_to_return;
+
+      if (current_record_index_ == apply_option_at_record_index_) {
+        option_to_return = wal_processing_option_;
+      } else {
+        option_to_return = WalProcessingOption::kContinueProcessing;
+      }
+
+      // Filter is passed as a const object for RocksDB to not modify the
+      // object, however we modify it for our own purpose here and hence
+      // cast the constness away.
+      (const_cast<TestWalFilter*>(this)->current_record_index_)++;
+
+      return option_to_return;
+    }
+
+    virtual const char* Name() const override { return "TestWalFilter"; }
+  };
+
+  // Create 3 batches with two keys each
+  std::vector<std::vector<std::string>> batch_keys(3);
+
+  batch_keys[0].push_back("key1");
+  batch_keys[0].push_back("key2");
+  batch_keys[1].push_back("key3");
+  batch_keys[1].push_back("key4");
+  batch_keys[2].push_back("key5");
+  batch_keys[2].push_back("key6");
+
+  // Test with all WAL processing options
+  for (int option = 0;
+       option < static_cast<int>(
+                    WalFilter::WalProcessingOption::kWalProcessingOptionMax);
+       option++) {
+    Options options = OptionsForLogIterTest();
+    DestroyAndReopen(options);
+    CreateAndReopenWithCF({"pikachu"}, options);
+
+    // Write given keys in given batches
+    for (size_t i = 0; i < batch_keys.size(); i++) {
+      WriteBatch batch;
+      for (size_t j = 0; j < batch_keys[i].size(); j++) {
+        batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
+      }
+      dbfull()->Write(WriteOptions(), &batch);
+    }
+
+    WalFilter::WalProcessingOption wal_processing_option =
+        static_cast<WalFilter::WalProcessingOption>(option);
+
+    // Create a test filter that would apply wal_processing_option at the first
+    // record
+    size_t apply_option_for_record_index = 1;
+    TestWalFilter test_wal_filter(wal_processing_option,
+                                  apply_option_for_record_index);
+
+    // Reopen database with option to use WAL filter
+    options = OptionsForLogIterTest();
+    options.wal_filter = &test_wal_filter;
+    Status status =
+        TryReopenWithColumnFamilies({"default", "pikachu"}, options);
+    if (wal_processing_option ==
+        WalFilter::WalProcessingOption::kCorruptedRecord) {
+      assert(!status.ok());
+      // In case of corruption we can turn off paranoid_checks to reopen
+      // databse
+      options.paranoid_checks = false;
+      ReopenWithColumnFamilies({"default", "pikachu"}, options);
+    } else {
+      assert(status.ok());
+    }
+
+    // Compute which keys we expect to be found
+    // and which we expect not to be found after recovery.
+    std::vector<Slice> keys_must_exist;
+    std::vector<Slice> keys_must_not_exist;
+    switch (wal_processing_option) {
+      case WalFilter::WalProcessingOption::kCorruptedRecord:
+      case WalFilter::WalProcessingOption::kContinueProcessing: {
+        fprintf(stderr, "Testing with complete WAL processing\n");
+        // we expect all records to be processed
+        for (size_t i = 0; i < batch_keys.size(); i++) {
+          for (size_t j = 0; j < batch_keys[i].size(); j++) {
+            keys_must_exist.push_back(Slice(batch_keys[i][j]));
+          }
+        }
+        break;
+      }
+      case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
+        fprintf(stderr,
+                "Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
+                apply_option_for_record_index);
+        // We expect the record with apply_option_for_record_index to be not
+        // found.
+        for (size_t i = 0; i < batch_keys.size(); i++) {
+          for (size_t j = 0; j < batch_keys[i].size(); j++) {
+            if (i == apply_option_for_record_index) {
+              keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
+            } else {
+              keys_must_exist.push_back(Slice(batch_keys[i][j]));
+            }
+          }
+        }
+        break;
+      }
+      case WalFilter::WalProcessingOption::kStopReplay: {
+        fprintf(stderr,
+                "Testing with stopping replay from record %" ROCKSDB_PRIszt
+                "\n",
+                apply_option_for_record_index);
+        // We expect records beyond apply_option_for_record_index to be not
+        // found.
+        for (size_t i = 0; i < batch_keys.size(); i++) {
+          for (size_t j = 0; j < batch_keys[i].size(); j++) {
+            if (i >= apply_option_for_record_index) {
+              keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
+            } else {
+              keys_must_exist.push_back(Slice(batch_keys[i][j]));
+            }
+          }
+        }
+        break;
+      }
+      default:
+        assert(false);  // unhandled case
+    }
+
+    bool checked_after_reopen = false;
+
+    while (true) {
+      // Ensure that expected keys exists
+      // and not expected keys don't exist after recovery
+      ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
+
+      if (checked_after_reopen) {
+        break;
+      }
+
+      // reopen database again to make sure previous log(s) are not used
+      //(even if they were skipped)
+      // reopn database with option to use WAL filter
+      options = OptionsForLogIterTest();
+      ReopenWithColumnFamilies({"default", "pikachu"}, options);
+
+      checked_after_reopen = true;
+    }
+  }
+}
+
+TEST_F(DBTest, WalFilterTestWithChangeBatch) {
+  class ChangeBatchHandler : public WriteBatch::Handler {
+   private:
+    // Batch to insert keys in
+    WriteBatch* new_write_batch_;
+    // Number of keys to add in the new batch
+    size_t num_keys_to_add_in_new_batch_;
+    // Number of keys added to new batch
+    size_t num_keys_added_;
+
+   public:
+    ChangeBatchHandler(WriteBatch* new_write_batch,
+                       size_t num_keys_to_add_in_new_batch)
+        : new_write_batch_(new_write_batch),
+          num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
+          num_keys_added_(0) {}
+    virtual void Put(const Slice& key, const Slice& value) override {
+      if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
+        new_write_batch_->Put(key, value);
+        ++num_keys_added_;
+      }
+    }
+  };
+
+  class TestWalFilterWithChangeBatch : public WalFilter {
+   private:
+    // Index at which to start changing records
+    size_t change_records_from_index_;
+    // Number of keys to add in the new batch
+    size_t num_keys_to_add_in_new_batch_;
+    // Current record index, incremented with each record encountered.
+    size_t current_record_index_;
+
+   public:
+    TestWalFilterWithChangeBatch(size_t change_records_from_index,
+                                 size_t num_keys_to_add_in_new_batch)
+        : change_records_from_index_(change_records_from_index),
+          num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
+          current_record_index_(0) {}
+
+    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
+                                          WriteBatch* new_batch,
+                                          bool* batch_changed) const override {
+      if (current_record_index_ >= change_records_from_index_) {
+        ChangeBatchHandler handler(new_batch, num_keys_to_add_in_new_batch_);
+        batch.Iterate(&handler);
+        *batch_changed = true;
+      }
+
+      // Filter is passed as a const object for RocksDB to not modify the
+      // object, however we modify it for our own purpose here and hence
+      // cast the constness away.
+      (const_cast<TestWalFilterWithChangeBatch*>(this)
+           ->current_record_index_)++;
+
+      return WalProcessingOption::kContinueProcessing;
+    }
+
+    virtual const char* Name() const override {
+      return "TestWalFilterWithChangeBatch";
+    }
+  };
+
+  std::vector<std::vector<std::string>> batch_keys(3);
+
+  batch_keys[0].push_back("key1");
+  batch_keys[0].push_back("key2");
+  batch_keys[1].push_back("key3");
+  batch_keys[1].push_back("key4");
+  batch_keys[2].push_back("key5");
+  batch_keys[2].push_back("key6");
+
+  Options options = OptionsForLogIterTest();
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  // Write given keys in given batches
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    WriteBatch batch;
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
+    }
+    dbfull()->Write(WriteOptions(), &batch);
+  }
+
+  // Create a test filter that would apply wal_processing_option at the first
+  // record
+  size_t change_records_from_index = 1;
+  size_t num_keys_to_add_in_new_batch = 1;
+  TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
+      change_records_from_index, num_keys_to_add_in_new_batch);
+
+  // Reopen database with option to use WAL filter
+  options = OptionsForLogIterTest();
+  options.wal_filter = &test_wal_filter_with_change_batch;
+  ReopenWithColumnFamilies({"default", "pikachu"}, options);
+
+  // Ensure that all keys exist before change_records_from_index_
+  // And after that index only single key exists
+  // as our filter adds only single key for each batch
+  std::vector<Slice> keys_must_exist;
+  std::vector<Slice> keys_must_not_exist;
+
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      if (i >= change_records_from_index && j >= num_keys_to_add_in_new_batch) {
+        keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
+      } else {
+        keys_must_exist.push_back(Slice(batch_keys[i][j]));
+      }
+    }
+  }
+
+  bool checked_after_reopen = false;
+
+  while (true) {
+    // Ensure that expected keys exists
+    // and not expected keys don't exist after recovery
+    ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
+
+    if (checked_after_reopen) {
+      break;
+    }
+
+    // reopen database again to make sure previous log(s) are not used
+    //(even if they were skipped)
+    // reopn database with option to use WAL filter
+    options = OptionsForLogIterTest();
+    ReopenWithColumnFamilies({"default", "pikachu"}, options);
+
+    checked_after_reopen = true;
+  }
+}
+
+TEST_F(DBTest, WalFilterTestWithChangeBatchExtraKeys) {
+  class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
+   public:
+    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
+                                          WriteBatch* new_batch,
+                                          bool* batch_changed) const override {
+      *new_batch = batch;
+      new_batch->Put("key_extra", "value_extra");
+      *batch_changed = true;
+      return WalProcessingOption::kContinueProcessing;
+    }
+
+    virtual const char* Name() const override {
+      return "WalFilterTestWithChangeBatchExtraKeys";
+    }
+  };
+
+  std::vector<std::vector<std::string>> batch_keys(3);
+
+  batch_keys[0].push_back("key1");
+  batch_keys[0].push_back("key2");
+  batch_keys[1].push_back("key3");
+  batch_keys[1].push_back("key4");
+  batch_keys[2].push_back("key5");
+  batch_keys[2].push_back("key6");
+
+  Options options = OptionsForLogIterTest();
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  // Write given keys in given batches
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    WriteBatch batch;
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
+    }
+    dbfull()->Write(WriteOptions(), &batch);
+  }
+
+  // Create a test filter that would add extra keys
+  TestWalFilterWithChangeBatchAddExtraKeys test_wal_filter_extra_keys;
+
+  // Reopen database with option to use WAL filter
+  options = OptionsForLogIterTest();
+  options.wal_filter = &test_wal_filter_extra_keys;
+  Status status = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
+  ASSERT_TRUE(status.IsNotSupported());
+
+  // Reopen without filter, now reopen should succeed - previous
+  // attempt to open must not have altered the db.
+  options = OptionsForLogIterTest();
+  ReopenWithColumnFamilies({"default", "pikachu"}, options);
+
+  std::vector<Slice> keys_must_exist;
+  std::vector<Slice> keys_must_not_exist;  // empty vector
+
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      keys_must_exist.push_back(Slice(batch_keys[i][j]));
+    }
+  }
+
+  ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
+}
+#endif  // ROCKSDB_LITE
+
 class SliceTransformLimitedDomain : public SliceTransform {
   const char* Name() const override { return "SliceTransformLimitedDomain"; }
 
@@ -10193,88 +10429,6 @@ TEST_F(DBTest, PrefixExtractorBlockFilter) {
   delete iter;
 }
 
-TEST_F(DBTest, IteratorWithLocalStatistics) {
-  Options options = CurrentOptions();
-  options.statistics = rocksdb::CreateDBStatistics();
-  DestroyAndReopen(options);
-
-  Random rnd(301);
-  for (int i = 0; i < 1000; i++) {
-    // Key 10 bytes / Value 10 bytes
-    ASSERT_OK(Put(RandomString(&rnd, 10), RandomString(&rnd, 10)));
-  }
-
-  std::atomic<uint64_t> total_next(0);
-  std::atomic<uint64_t> total_next_found(0);
-  std::atomic<uint64_t> total_prev(0);
-  std::atomic<uint64_t> total_prev_found(0);
-  std::atomic<uint64_t> total_bytes(0);
-
-  std::vector<std::thread> threads;
-  std::function<void()> reader_func_next = [&]() {
-    Iterator* iter = db_->NewIterator(ReadOptions());
-
-    iter->SeekToFirst();
-    // Seek will bump ITER_BYTES_READ
-    total_bytes += iter->key().size();
-    total_bytes += iter->value().size();
-    while (true) {
-      iter->Next();
-      total_next++;
-
-      if (!iter->Valid()) {
-        break;
-      }
-      total_next_found++;
-      total_bytes += iter->key().size();
-      total_bytes += iter->value().size();
-    }
-
-    delete iter;
-  };
-
-  std::function<void()> reader_func_prev = [&]() {
-    Iterator* iter = db_->NewIterator(ReadOptions());
-
-    iter->SeekToLast();
-    // Seek will bump ITER_BYTES_READ
-    total_bytes += iter->key().size();
-    total_bytes += iter->value().size();
-    while (true) {
-      iter->Prev();
-      total_prev++;
-
-      if (!iter->Valid()) {
-        break;
-      }
-      total_prev_found++;
-      total_bytes += iter->key().size();
-      total_bytes += iter->value().size();
-    }
-
-    delete iter;
-  };
-
-  for (int i = 0; i < 10; i++) {
-    threads.emplace_back(reader_func_next);
-  }
-  for (int i = 0; i < 15; i++) {
-    threads.emplace_back(reader_func_prev);
-  }
-
-  for (auto& t : threads) {
-    t.join();
-  }
-
-  ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_NEXT), total_next);
-  ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_NEXT_FOUND),
-            total_next_found);
-  ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV), total_prev);
-  ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV_FOUND),
-            total_prev_found);
-  ASSERT_EQ(TestGetTickerCount(options, ITER_BYTES_READ), total_bytes);
-}
-
 #ifndef ROCKSDB_LITE
 class BloomStatsTestWithParam
     : public DBTest,

db/db_test2.cc

@@ -1,864 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include <cstdlib>
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-#include "rocksdb/wal_filter.h"
-
-namespace rocksdb {
-
-static uint64_t TestGetTickerCount(const Options& options,
-                                   Tickers ticker_type) {
-  return options.statistics->getTickerCount(ticker_type);
-}
-
-class DBTest2 : public DBTestBase {
- public:
-  DBTest2() : DBTestBase("/db_test2") {}
-};
-
-class PrefixFullBloomWithReverseComparator
-    : public DBTestBase,
-      public ::testing::WithParamInterface<bool> {
- public:
-  PrefixFullBloomWithReverseComparator()
-      : DBTestBase("/prefix_bloom_reverse") {}
-  virtual void SetUp() override { if_cache_filter_ = GetParam(); }
-  bool if_cache_filter_;
-};
-
-TEST_P(PrefixFullBloomWithReverseComparator,
-       PrefixFullBloomWithReverseComparator) {
-  Options options = last_options_;
-  options.comparator = ReverseBytewiseComparator();
-  options.prefix_extractor.reset(NewCappedPrefixTransform(3));
-  options.statistics = rocksdb::CreateDBStatistics();
-  BlockBasedTableOptions bbto;
-  if (if_cache_filter_) {
-    bbto.no_block_cache = false;
-    bbto.cache_index_and_filter_blocks = true;
-    bbto.block_cache = NewLRUCache(1);
-  }
-  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
-  bbto.whole_key_filtering = false;
-  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-  DestroyAndReopen(options);
-
-  ASSERT_OK(dbfull()->Put(WriteOptions(), "bar123", "foo"));
-  ASSERT_OK(dbfull()->Put(WriteOptions(), "bar234", "foo2"));
-  ASSERT_OK(dbfull()->Put(WriteOptions(), "foo123", "foo3"));
-
-  dbfull()->Flush(FlushOptions());
-
-  if (bbto.block_cache) {
-    bbto.block_cache->EraseUnRefEntries();
-  }
-
-  unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
-  iter->Seek("bar345");
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ("bar234", iter->key().ToString());
-  ASSERT_EQ("foo2", iter->value().ToString());
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ("bar123", iter->key().ToString());
-  ASSERT_EQ("foo", iter->value().ToString());
-
-  iter->Seek("foo234");
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ("foo123", iter->key().ToString());
-  ASSERT_EQ("foo3", iter->value().ToString());
-
-  iter->Seek("bar");
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(!iter->Valid());
-}
-
-INSTANTIATE_TEST_CASE_P(PrefixFullBloomWithReverseComparator,
-                        PrefixFullBloomWithReverseComparator, testing::Bool());
-
-TEST_F(DBTest2, IteratorPropertyVersionNumber) {
-  Put("", "");
-  Iterator* iter1 = db_->NewIterator(ReadOptions());
-  std::string prop_value;
-  ASSERT_OK(
-      iter1->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
-  uint64_t version_number1 =
-      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-  Put("", "");
-  Flush();
-
-  Iterator* iter2 = db_->NewIterator(ReadOptions());
-  ASSERT_OK(
-      iter2->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
-  uint64_t version_number2 =
-      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-  ASSERT_GT(version_number2, version_number1);
-
-  Put("", "");
-
-  Iterator* iter3 = db_->NewIterator(ReadOptions());
-  ASSERT_OK(
-      iter3->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
-  uint64_t version_number3 =
-      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-  ASSERT_EQ(version_number2, version_number3);
-
-  iter1->SeekToFirst();
-  ASSERT_OK(
-      iter1->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
-  uint64_t version_number1_new =
-      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-  ASSERT_EQ(version_number1, version_number1_new);
-
-  delete iter1;
-  delete iter2;
-  delete iter3;
-}
-
-TEST_F(DBTest2, CacheIndexAndFilterWithDBRestart) {
-  Options options = CurrentOptions();
-  options.create_if_missing = true;
-  options.statistics = rocksdb::CreateDBStatistics();
-  BlockBasedTableOptions table_options;
-  table_options.cache_index_and_filter_blocks = true;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  Put(1, "a", "begin");
-  Put(1, "z", "end");
-  ASSERT_OK(Flush(1));
-  TryReopenWithColumnFamilies({"default", "pikachu"}, options);
-
-  std::string value;
-  value = Get(1, "a");
-}
-
-#ifndef ROCKSDB_LITE
-namespace {
-  void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
-    const std::vector<Slice>& keys_must_not_exist) {
-    // Ensure that expected keys exist
-    std::vector<std::string> values;
-    if (keys_must_exist.size() > 0) {
-      std::vector<Status> status_list =
-        db->MultiGet(ReadOptions(), keys_must_exist, &values);
-      for (size_t i = 0; i < keys_must_exist.size(); i++) {
-        ASSERT_OK(status_list[i]);
-      }
-    }
-
-    // Ensure that given keys don't exist
-    if (keys_must_not_exist.size() > 0) {
-      std::vector<Status> status_list =
-        db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
-      for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
-        ASSERT_TRUE(status_list[i].IsNotFound());
-      }
-    }
-  }
-
-}  // namespace
-
-TEST_F(DBTest2, WalFilterTest) {
-  class TestWalFilter : public WalFilter {
-  private:
-    // Processing option that is requested to be applied at the given index
-    WalFilter::WalProcessingOption wal_processing_option_;
-    // Index at which to apply wal_processing_option_
-    // At other indexes default wal_processing_option::kContinueProcessing is
-    // returned.
-    size_t apply_option_at_record_index_;
-    // Current record index, incremented with each record encountered.
-    size_t current_record_index_;
-
-  public:
-    TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
-      size_t apply_option_for_record_index)
-      : wal_processing_option_(wal_processing_option),
-      apply_option_at_record_index_(apply_option_for_record_index),
-      current_record_index_(0) {}
-
-    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
-      WriteBatch* new_batch,
-      bool* batch_changed) const override {
-      WalFilter::WalProcessingOption option_to_return;
-
-      if (current_record_index_ == apply_option_at_record_index_) {
-        option_to_return = wal_processing_option_;
-      }
-      else {
-        option_to_return = WalProcessingOption::kContinueProcessing;
-      }
-
-      // Filter is passed as a const object for RocksDB to not modify the
-      // object, however we modify it for our own purpose here and hence
-      // cast the constness away.
-      (const_cast<TestWalFilter*>(this)->current_record_index_)++;
-
-      return option_to_return;
-    }
-
-    virtual const char* Name() const override { return "TestWalFilter"; }
-  };
-
-  // Create 3 batches with two keys each
-  std::vector<std::vector<std::string>> batch_keys(3);
-
-  batch_keys[0].push_back("key1");
-  batch_keys[0].push_back("key2");
-  batch_keys[1].push_back("key3");
-  batch_keys[1].push_back("key4");
-  batch_keys[2].push_back("key5");
-  batch_keys[2].push_back("key6");
-
-  // Test with all WAL processing options
-  for (int option = 0;
-    option < static_cast<int>(
-    WalFilter::WalProcessingOption::kWalProcessingOptionMax);
-  option++) {
-    Options options = OptionsForLogIterTest();
-    DestroyAndReopen(options);
-    CreateAndReopenWithCF({ "pikachu" }, options);
-
-    // Write given keys in given batches
-    for (size_t i = 0; i < batch_keys.size(); i++) {
-      WriteBatch batch;
-      for (size_t j = 0; j < batch_keys[i].size(); j++) {
-        batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
-      }
-      dbfull()->Write(WriteOptions(), &batch);
-    }
-
-    WalFilter::WalProcessingOption wal_processing_option =
-      static_cast<WalFilter::WalProcessingOption>(option);
-
-    // Create a test filter that would apply wal_processing_option at the first
-    // record
-    size_t apply_option_for_record_index = 1;
-    TestWalFilter test_wal_filter(wal_processing_option,
-      apply_option_for_record_index);
-
-    // Reopen database with option to use WAL filter
-    options = OptionsForLogIterTest();
-    options.wal_filter = &test_wal_filter;
-    Status status =
-      TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
-    if (wal_processing_option ==
-      WalFilter::WalProcessingOption::kCorruptedRecord) {
-      assert(!status.ok());
-      // In case of corruption we can turn off paranoid_checks to reopen
-      // databse
-      options.paranoid_checks = false;
-      ReopenWithColumnFamilies({ "default", "pikachu" }, options);
-    }
-    else {
-      assert(status.ok());
-    }
-
-    // Compute which keys we expect to be found
-    // and which we expect not to be found after recovery.
-    std::vector<Slice> keys_must_exist;
-    std::vector<Slice> keys_must_not_exist;
-    switch (wal_processing_option) {
-    case WalFilter::WalProcessingOption::kCorruptedRecord:
-    case WalFilter::WalProcessingOption::kContinueProcessing: {
-      fprintf(stderr, "Testing with complete WAL processing\n");
-      // we expect all records to be processed
-      for (size_t i = 0; i < batch_keys.size(); i++) {
-        for (size_t j = 0; j < batch_keys[i].size(); j++) {
-          keys_must_exist.push_back(Slice(batch_keys[i][j]));
-        }
-      }
-      break;
-    }
-    case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
-      fprintf(stderr,
-        "Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
-        apply_option_for_record_index);
-      // We expect the record with apply_option_for_record_index to be not
-      // found.
-      for (size_t i = 0; i < batch_keys.size(); i++) {
-        for (size_t j = 0; j < batch_keys[i].size(); j++) {
-          if (i == apply_option_for_record_index) {
-            keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
-          }
-          else {
-            keys_must_exist.push_back(Slice(batch_keys[i][j]));
-          }
-        }
-      }
-      break;
-    }
-    case WalFilter::WalProcessingOption::kStopReplay: {
-      fprintf(stderr,
-        "Testing with stopping replay from record %" ROCKSDB_PRIszt
-        "\n",
-        apply_option_for_record_index);
-      // We expect records beyond apply_option_for_record_index to be not
-      // found.
-      for (size_t i = 0; i < batch_keys.size(); i++) {
-        for (size_t j = 0; j < batch_keys[i].size(); j++) {
-          if (i >= apply_option_for_record_index) {
-            keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
-          }
-          else {
-            keys_must_exist.push_back(Slice(batch_keys[i][j]));
-          }
-        }
-      }
-      break;
-    }
-    default:
-      assert(false);  // unhandled case
-    }
-
-    bool checked_after_reopen = false;
-
-    while (true) {
-      // Ensure that expected keys exists
-      // and not expected keys don't exist after recovery
-      ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
-
-      if (checked_after_reopen) {
-        break;
-      }
-
-      // reopen database again to make sure previous log(s) are not used
-      //(even if they were skipped)
-      // reopn database with option to use WAL filter
-      options = OptionsForLogIterTest();
-      ReopenWithColumnFamilies({ "default", "pikachu" }, options);
-
-      checked_after_reopen = true;
-    }
-  }
-}
-
-TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
-  class ChangeBatchHandler : public WriteBatch::Handler {
-  private:
-    // Batch to insert keys in
-    WriteBatch* new_write_batch_;
-    // Number of keys to add in the new batch
-    size_t num_keys_to_add_in_new_batch_;
-    // Number of keys added to new batch
-    size_t num_keys_added_;
-
-  public:
-    ChangeBatchHandler(WriteBatch* new_write_batch,
-      size_t num_keys_to_add_in_new_batch)
-      : new_write_batch_(new_write_batch),
-      num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
-      num_keys_added_(0) {}
-    virtual void Put(const Slice& key, const Slice& value) override {
-      if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
-        new_write_batch_->Put(key, value);
-        ++num_keys_added_;
-      }
-    }
-  };
-
-  class TestWalFilterWithChangeBatch : public WalFilter {
-  private:
-    // Index at which to start changing records
-    size_t change_records_from_index_;
-    // Number of keys to add in the new batch
-    size_t num_keys_to_add_in_new_batch_;
-    // Current record index, incremented with each record encountered.
-    size_t current_record_index_;
-
-  public:
-    TestWalFilterWithChangeBatch(size_t change_records_from_index,
-      size_t num_keys_to_add_in_new_batch)
-      : change_records_from_index_(change_records_from_index),
-      num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
-      current_record_index_(0) {}
-
-    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
-      WriteBatch* new_batch,
-      bool* batch_changed) const override {
-      if (current_record_index_ >= change_records_from_index_) {
-        ChangeBatchHandler handler(new_batch, num_keys_to_add_in_new_batch_);
-        batch.Iterate(&handler);
-        *batch_changed = true;
-      }
-
-      // Filter is passed as a const object for RocksDB to not modify the
-      // object, however we modify it for our own purpose here and hence
-      // cast the constness away.
-      (const_cast<TestWalFilterWithChangeBatch*>(this)
-        ->current_record_index_)++;
-
-      return WalProcessingOption::kContinueProcessing;
-    }
-
-    virtual const char* Name() const override {
-      return "TestWalFilterWithChangeBatch";
-    }
-  };
-
-  std::vector<std::vector<std::string>> batch_keys(3);
-
-  batch_keys[0].push_back("key1");
-  batch_keys[0].push_back("key2");
-  batch_keys[1].push_back("key3");
-  batch_keys[1].push_back("key4");
-  batch_keys[2].push_back("key5");
-  batch_keys[2].push_back("key6");
-
-  Options options = OptionsForLogIterTest();
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({ "pikachu" }, options);
-
-  // Write given keys in given batches
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    WriteBatch batch;
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
-    }
-    dbfull()->Write(WriteOptions(), &batch);
-  }
-
-  // Create a test filter that would apply wal_processing_option at the first
-  // record
-  size_t change_records_from_index = 1;
-  size_t num_keys_to_add_in_new_batch = 1;
-  TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
-    change_records_from_index, num_keys_to_add_in_new_batch);
-
-  // Reopen database with option to use WAL filter
-  options = OptionsForLogIterTest();
-  options.wal_filter = &test_wal_filter_with_change_batch;
-  ReopenWithColumnFamilies({ "default", "pikachu" }, options);
-
-  // Ensure that all keys exist before change_records_from_index_
-  // And after that index only single key exists
-  // as our filter adds only single key for each batch
-  std::vector<Slice> keys_must_exist;
-  std::vector<Slice> keys_must_not_exist;
-
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      if (i >= change_records_from_index && j >= 
-        num_keys_to_add_in_new_batch) {
-        keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
-      }
-      else {
-        keys_must_exist.push_back(Slice(batch_keys[i][j]));
-      }
-    }
-  }
-
-  bool checked_after_reopen = false;
-
-  while (true) {
-    // Ensure that expected keys exists
-    // and not expected keys don't exist after recovery
-    ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
-
-    if (checked_after_reopen) {
-      break;
-    }
-
-    // reopen database again to make sure previous log(s) are not used
-    //(even if they were skipped)
-    // reopn database with option to use WAL filter
-    options = OptionsForLogIterTest();
-    ReopenWithColumnFamilies({ "default", "pikachu" }, options);
-
-    checked_after_reopen = true;
-  }
-}
-
-TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
-  class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
-  public:
-    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
-      WriteBatch* new_batch,
-      bool* batch_changed) const override {
-      *new_batch = batch;
-      new_batch->Put("key_extra", "value_extra");
-      *batch_changed = true;
-      return WalProcessingOption::kContinueProcessing;
-    }
-
-    virtual const char* Name() const override {
-      return "WalFilterTestWithChangeBatchExtraKeys";
-    }
-  };
-
-  std::vector<std::vector<std::string>> batch_keys(3);
-
-  batch_keys[0].push_back("key1");
-  batch_keys[0].push_back("key2");
-  batch_keys[1].push_back("key3");
-  batch_keys[1].push_back("key4");
-  batch_keys[2].push_back("key5");
-  batch_keys[2].push_back("key6");
-
-  Options options = OptionsForLogIterTest();
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({ "pikachu" }, options);
-
-  // Write given keys in given batches
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    WriteBatch batch;
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
-    }
-    dbfull()->Write(WriteOptions(), &batch);
-  }
-
-  // Create a test filter that would add extra keys
-  TestWalFilterWithChangeBatchAddExtraKeys test_wal_filter_extra_keys;
-
-  // Reopen database with option to use WAL filter
-  options = OptionsForLogIterTest();
-  options.wal_filter = &test_wal_filter_extra_keys;
-  Status status = TryReopenWithColumnFamilies({ "default", "pikachu" }, 
-    options);
-  ASSERT_TRUE(status.IsNotSupported());
-
-  // Reopen without filter, now reopen should succeed - previous
-  // attempt to open must not have altered the db.
-  options = OptionsForLogIterTest();
-  ReopenWithColumnFamilies({ "default", "pikachu" }, options);
-
-  std::vector<Slice> keys_must_exist;
-  std::vector<Slice> keys_must_not_exist;  // empty vector
-
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      keys_must_exist.push_back(Slice(batch_keys[i][j]));
-    }
-  }
-
-  ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
-}
-
-TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
-  class TestWalFilterWithColumnFamilies : public WalFilter {
-  private:
-    // column_family_id -> log_number map (provided to WALFilter)
-    std::map<uint32_t, uint64_t> cf_log_number_map_;
-    // column_family_name -> column_family_id map (provided to WALFilter)
-    std::map<std::string, uint32_t> cf_name_id_map_;
-    // column_family_name -> keys_found_in_wal map
-    // We store keys that are applicable to the column_family
-    // during recovery (i.e. aren't already flushed to SST file(s))
-    // for verification against the keys we expect.
-    std::map<uint32_t, std::vector<std::string>> cf_wal_keys_;
-  public:
-    virtual void ColumnFamilyLogNumberMap(
-      const std::map<uint32_t, uint64_t>& cf_lognumber_map,
-      const std::map<std::string, uint32_t>& cf_name_id_map) override {
-      cf_log_number_map_ = cf_lognumber_map;
-      cf_name_id_map_ = cf_name_id_map;
-    }
-
-    virtual WalProcessingOption LogRecordFound(unsigned long long log_number,
-      const std::string& log_file_name,
-      const WriteBatch& batch,
-      WriteBatch* new_batch,
-      bool* batch_changed) override {
-      class LogRecordBatchHandler : public WriteBatch::Handler {
-      private:
-        const std::map<uint32_t, uint64_t> & cf_log_number_map_;
-        std::map<uint32_t, std::vector<std::string>> & cf_wal_keys_;
-        unsigned long long log_number_;
-      public:
-        LogRecordBatchHandler(unsigned long long current_log_number,
-          const std::map<uint32_t, uint64_t> & cf_log_number_map,
-          std::map<uint32_t, std::vector<std::string>> & cf_wal_keys) :
-          cf_log_number_map_(cf_log_number_map),
-          cf_wal_keys_(cf_wal_keys),
-          log_number_(current_log_number){}
-
-        virtual Status PutCF(uint32_t column_family_id, const Slice& key,
-          const Slice& /*value*/) override {
-          auto it = cf_log_number_map_.find(column_family_id);
-          assert(it != cf_log_number_map_.end());
-          unsigned long long log_number_for_cf = it->second;
-          // If the current record is applicable for column_family_id
-          // (i.e. isn't flushed to SST file(s) for column_family_id)
-          // add it to the cf_wal_keys_ map for verification.
-          if (log_number_ >= log_number_for_cf) {
-            cf_wal_keys_[column_family_id].push_back(std::string(key.data(),
-              key.size()));
-          }
-          return Status::OK();
-        }
-      } handler(log_number, cf_log_number_map_, cf_wal_keys_);
-
-      batch.Iterate(&handler);
-
-      return WalProcessingOption::kContinueProcessing;
-    }
-
-    virtual const char* Name() const override {
-      return "WalFilterTestWithColumnFamilies";
-    }
-
-    const std::map<uint32_t, std::vector<std::string>> & 
-    GetColumnFamilyKeys() {
-      return cf_wal_keys_;
-    }
-
-    const std::map<std::string, uint32_t> & GetColumnFamilyNameIdMap() {
-      return cf_name_id_map_;
-    }
-  };
-
-  std::vector<std::vector<std::string>> batch_keys_pre_flush(3);
-
-  batch_keys_pre_flush[0].push_back("key1");
-  batch_keys_pre_flush[0].push_back("key2");
-  batch_keys_pre_flush[1].push_back("key3");
-  batch_keys_pre_flush[1].push_back("key4");
-  batch_keys_pre_flush[2].push_back("key5");
-  batch_keys_pre_flush[2].push_back("key6");
-
-  Options options = OptionsForLogIterTest();
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({ "pikachu" }, options);
-
-  // Write given keys in given batches
-  for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
-    WriteBatch batch;
-    for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
-      batch.Put(handles_[0], batch_keys_pre_flush[i][j], DummyString(1024));
-      batch.Put(handles_[1], batch_keys_pre_flush[i][j], DummyString(1024));
-    }
-    dbfull()->Write(WriteOptions(), &batch);
-  }
-
-  //Flush default column-family
-  db_->Flush(FlushOptions(), handles_[0]);
-
-  // Do some more writes
-  std::vector<std::vector<std::string>> batch_keys_post_flush(3);
-
-  batch_keys_post_flush[0].push_back("key7");
-  batch_keys_post_flush[0].push_back("key8");
-  batch_keys_post_flush[1].push_back("key9");
-  batch_keys_post_flush[1].push_back("key10");
-  batch_keys_post_flush[2].push_back("key11");
-  batch_keys_post_flush[2].push_back("key12");
-
-  // Write given keys in given batches
-  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
-    WriteBatch batch;
-    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
-      batch.Put(handles_[0], batch_keys_post_flush[i][j], DummyString(1024));
-      batch.Put(handles_[1], batch_keys_post_flush[i][j], DummyString(1024));
-    }
-    dbfull()->Write(WriteOptions(), &batch);
-  }
-
-  // On Recovery we should only find the second batch applicable to default CF
-  // But both batches applicable to pikachu CF
-
-  // Create a test filter that would add extra keys
-  TestWalFilterWithColumnFamilies test_wal_filter_column_families;
-
-  // Reopen database with option to use WAL filter
-  options = OptionsForLogIterTest();
-  options.wal_filter = &test_wal_filter_column_families;
-  Status status =
-    TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
-  ASSERT_TRUE(status.ok());
-
-  // verify that handles_[0] only has post_flush keys
-  // while handles_[1] has pre and post flush keys
-  auto cf_wal_keys = test_wal_filter_column_families.GetColumnFamilyKeys();
-  auto name_id_map = 
-    test_wal_filter_column_families.GetColumnFamilyNameIdMap();
-  size_t index = 0;
-  auto keys_cf = cf_wal_keys[name_id_map[kDefaultColumnFamilyName]];
-  //default column-family, only post_flush keys are expected
-  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
-    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
-      Slice key_from_the_log(keys_cf[index++]);
-      Slice batch_key(batch_keys_post_flush[i][j]);
-      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
-    }
-  }
-  ASSERT_TRUE(index == keys_cf.size());
-
-  index = 0;
-  keys_cf = cf_wal_keys[name_id_map["pikachu"]];
-  //pikachu column-family, all keys are expected
-  for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
-    for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
-      Slice key_from_the_log(keys_cf[index++]);
-      Slice batch_key(batch_keys_pre_flush[i][j]);
-      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
-    }
-  }
-
-  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
-    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
-      Slice key_from_the_log(keys_cf[index++]);
-      Slice batch_key(batch_keys_post_flush[i][j]);
-      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
-    }
-  }
-  ASSERT_TRUE(index == keys_cf.size());
-}
-#endif  // ROCKSDB_LITE
-
-TEST_F(DBTest2, DISABLED_FirstSnapshotTest) {
-  Options options;
-  options.write_buffer_size = 100000;  // Small write buffer
-  options = CurrentOptions(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  // This snapshot will have sequence number 0.  When compaction encounters
-  // this snapshot, CompactionIterator::findEarliestVisibleSnapshot() will
-  // assert as it expects non-zero snapshots.
-  //
-  // One fix would be to simply remove this assert.  However, a better fix
-  // might
-  // be to always have db sequence numbers start from 1 so that no code is
-  // ever
-  // confused by 0.
-  const Snapshot* s1 = db_->GetSnapshot();
-
-  Put(1, "k1", std::string(100000, 'x'));  // Fill memtable
-  Put(1, "k2", std::string(100000, 'y'));  // Trigger flush
-
-  db_->ReleaseSnapshot(s1);
-}
-
-class PinL0IndexAndFilterBlocksTest : public DBTestBase,
-                                      public testing::WithParamInterface<bool> {
- public:
-  PinL0IndexAndFilterBlocksTest() : DBTestBase("/db_pin_l0_index_bloom_test") {}
-  virtual void SetUp() override { infinite_max_files_ = GetParam(); }
-
-  bool infinite_max_files_;
-};
-
-TEST_P(PinL0IndexAndFilterBlocksTest,
-       IndexAndFilterBlocksOfNewTableAddedToCacheWithPinning) {
-  Options options = CurrentOptions();
-  if (infinite_max_files_) {
-    options.max_open_files = -1;
-  }
-  options.create_if_missing = true;
-  options.statistics = rocksdb::CreateDBStatistics();
-  BlockBasedTableOptions table_options;
-  table_options.cache_index_and_filter_blocks = true;
-  table_options.pin_l0_filter_and_index_blocks_in_cache = true;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  ASSERT_OK(Put(1, "key", "val"));
-  // Create a new table.
-  ASSERT_OK(Flush(1));
-
-  // index/filter blocks added to block cache right after table creation.
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
-
-  // only index/filter were added
-  ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS));
-
-  std::string value;
-  // Miss and hit count should remain the same, they're all pinned.
-  db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value);
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
-
-  // Miss and hit count should remain the same, they're all pinned.
-  value = Get(1, "key");
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
-}
-
-TEST_P(PinL0IndexAndFilterBlocksTest,
-       MultiLevelIndexAndFilterBlocksCachedWithPinning) {
-  Options options = CurrentOptions();
-  if (infinite_max_files_) {
-    options.max_open_files = -1;
-  }
-  options.create_if_missing = true;
-  options.statistics = rocksdb::CreateDBStatistics();
-  BlockBasedTableOptions table_options;
-  table_options.cache_index_and_filter_blocks = true;
-  table_options.pin_l0_filter_and_index_blocks_in_cache = true;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  Put(1, "a", "begin");
-  Put(1, "z", "end");
-  ASSERT_OK(Flush(1));
-  // move this table to L1
-  dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
-
-  // reset block cache
-  table_options.block_cache = NewLRUCache(64 * 1024);
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  TryReopenWithColumnFamilies({"default", "pikachu"}, options);
-  // create new table at L0
-  Put(1, "a2", "begin2");
-  Put(1, "z2", "end2");
-  ASSERT_OK(Flush(1));
-
-  table_options.block_cache->EraseUnRefEntries();
-
-  // get base cache values
-  uint64_t fm = TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS);
-  uint64_t fh = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
-  uint64_t im = TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS);
-  uint64_t ih = TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT);
-
-  std::string value;
-  // this should be read from L0
-  // so cache values don't change
-  value = Get(1, "a2");
-  ASSERT_EQ(fm, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-  ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
-  ASSERT_EQ(im, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-  ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
-
-  // this should be read from L1
-  // the file is opened, prefetching results in a cache filter miss
-  // the block is loaded and added to the cache,
-  // then the get results in a cache hit for L1
-  value = Get(1, "a");
-  ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-  ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-}
-
-INSTANTIATE_TEST_CASE_P(PinL0IndexAndFilterBlocksTest,
-                        PinL0IndexAndFilterBlocksTest, ::testing::Bool());
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  rocksdb::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/db_test_util.cc

@@ -1,4 +1,4 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+// Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
@@ -210,8 +210,6 @@ Options DBTestBase::CurrentOptions(
     const anon::OptionsOverride& options_override) {
   Options options;
   options.write_buffer_size = 4090 * 4096;
-  options.target_file_size_base = 2 * 1024 * 1024;
-  options.max_bytes_for_level_base = 10 * 1024 * 1024;
   return CurrentOptions(options, options_override);
 }
 
@@ -303,7 +301,6 @@ Options DBTestBase::CurrentOptions(
     case kPerfOptions:
       options.soft_rate_limit = 2.0;
       options.delayed_write_rate = 8 * 1024 * 1024;
-      options.report_bg_io_stats = true;
       // TODO(3.13) -- test more options
       break;
     case kDeletesFilterFirst:
@@ -1018,13 +1015,9 @@ void DBTestBase::CopyFile(const std::string& source,
   ASSERT_OK(destfile->Close());
 }
 
-std::unordered_map<std::string, uint64_t> DBTestBase::GetAllSSTFiles(
-    uint64_t* total_size) {
+std::unordered_map<std::string, uint64_t> DBTestBase::GetAllSSTFiles() {
   std::unordered_map<std::string, uint64_t> res;
 
-  if (total_size) {
-    *total_size = 0;
-  }
   std::vector<std::string> files;
   env_->GetChildren(dbname_, &files);
   for (auto& file_name : files) {
@@ -1035,9 +1028,6 @@ std::unordered_map<std::string, uint64_t> DBTestBase::GetAllSSTFiles(
       uint64_t file_size = 0;
       env_->GetFileSize(file_path, &file_size);
       res[file_path] = file_size;
-      if (total_size) {
-        *total_size += file_size;
-      }
     }
   }
   return res;

db/db_test_util.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+// Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
@@ -754,8 +754,7 @@ class DBTestBase : public testing::Test {
   void CopyFile(const std::string& source, const std::string& destination,
                 uint64_t size = 0);
 
-  std::unordered_map<std::string, uint64_t> GetAllSSTFiles(
-      uint64_t* total_size = nullptr);
+  std::unordered_map<std::string, uint64_t> GetAllSSTFiles();
 };
 
 }  // namespace rocksdb

db/db_universal_compaction_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -124,7 +124,8 @@ class DelayFilterFactory : public CompactionFilterFactory {
 // Make sure we don't trigger a problem if the trigger conditon is given
 // to be 0, which is invalid.
 TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
-  Options options = CurrentOptions();
+  Options options;
+  options = CurrentOptions(options);
 
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = num_levels_;
@@ -159,7 +160,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
 }
 
 TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
-  Options options = CurrentOptions();
+  Options options;
+  options = CurrentOptions(options);
   options.compaction_style = kCompactionStyleUniversal;
   options.compaction_options_universal.size_ratio = 5;
   options.num_levels = num_levels_;
@@ -335,12 +337,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
 }
 
 TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = num_levels_;
   options.write_buffer_size = 100 << 10;     // 100KB
   options.target_file_size_base = 32 << 10;  // 32KB
   options.level0_file_num_compaction_trigger = 3;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -385,12 +388,12 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
   ChangeCompactOptions();
   Options options;
   options.create_if_missing = true;
+  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
   options.compaction_style = kCompactionStyleLevel;
   options.num_levels = 1;
   options.target_file_size_base = options.write_buffer_size;
   options.compression = kNoCompression;
   options = CurrentOptions(options);
-  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
   CreateAndReopenWithCF({"pikachu"}, options);
   ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
   Random rnd(301);
@@ -448,11 +451,12 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
 }
 
 TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.write_buffer_size = 100 << 10;     // 100KB
   options.num_levels = 7;
   options.disable_auto_compactions = true;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   // Generate 3 overlapping files
@@ -492,13 +496,14 @@ class DBTestUniversalCompactionMultiLevels
 };
 
 TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = num_levels_;
   options.write_buffer_size = 100 << 10;  // 100KB
   options.level0_file_num_compaction_trigger = 8;
   options.max_background_compactions = 3;
   options.target_file_size_base = 32 * 1024;
+  options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
   // Trigger compaction if size amplification exceeds 110%
@@ -534,7 +539,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
       });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.compaction_options_universal.allow_trivial_move = true;
   options.num_levels = 3;
@@ -542,6 +547,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
   options.level0_file_num_compaction_trigger = 3;
   options.max_background_compactions = 2;
   options.target_file_size_base = 32 * 1024;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -580,7 +586,7 @@ class DBTestUniversalCompactionParallel :
 };
 
 TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = num_levels_;
   options.write_buffer_size = 1 << 10;  // 1KB
@@ -589,6 +595,7 @@ TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
   options.max_background_flushes = 3;
   options.target_file_size_base = 1 * 1024;
   options.compaction_options_universal.max_size_amplification_percent = 110;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -645,7 +652,7 @@ INSTANTIATE_TEST_CASE_P(DBTestUniversalCompactionParallel,
                                            ::testing::Bool()));
 
 TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.write_buffer_size = 105 << 10;    // 105KB
   options.arena_block_size = 4 << 10;       // 4KB
@@ -653,6 +660,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
   options.level0_file_num_compaction_trigger = 4;
   options.num_levels = num_levels_;
   options.compaction_options_universal.compression_size_percent = -1;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -765,13 +773,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
     return;
   }
 
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.write_buffer_size = 100 << 10;     // 100KB
   options.target_file_size_base = 32 << 10;  // 32KB
   options.level0_file_num_compaction_trigger = 2;
   options.num_levels = num_levels_;
   options.compaction_options_universal.compression_size_percent = 70;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   Random rnd(301);
@@ -832,13 +841,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) {
   if (!Snappy_Supported()) {
     return;
   }
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.write_buffer_size = 100 << 10;     // 100KB
   options.target_file_size_base = 32 << 10;  // 32KB
   options.level0_file_num_compaction_trigger = 2;
   options.num_levels = num_levels_;
   options.compaction_options_universal.compression_size_percent = 95;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   Random rnd(301);
@@ -874,7 +884,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
       });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.compaction_options_universal.allow_trivial_move = true;
   options.num_levels = 2;
@@ -882,6 +892,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
   options.level0_file_num_compaction_trigger = 3;
   options.max_background_compactions = 1;
   options.target_file_size_base = 32 * 1024;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -918,7 +929,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
 
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.compaction_options_universal.allow_trivial_move = true;
   options.num_levels = 15;
@@ -926,6 +937,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
   options.level0_file_num_compaction_trigger = 8;
   options.max_background_compactions = 4;
   options.target_file_size_base = 64 * 1024;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -951,7 +963,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
 }
 
 TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
-  Options options = CurrentOptions();
+  Options options;
   options.db_paths.emplace_back(dbname_, 300 * 1024);
   options.db_paths.emplace_back(dbname_ + "_2", 300 * 1024);
   options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
@@ -964,6 +976,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
   options.arena_block_size = 4 << 10;
   options.level0_file_num_compaction_trigger = 2;
   options.num_levels = 1;
+  options = CurrentOptions(options);
 
   std::vector<std::string> filenames;
   env_->GetChildren(options.db_paths[1].path, &filenames);
@@ -1090,11 +1103,16 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
   for (int i = 0; i <= max_key1; i++) {
     // each value is 10K
     ASSERT_OK(Put(1, Key(i), RandomString(&rnd, 10000)));
-    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
   }
   ASSERT_OK(Flush(1));
   dbfull()->TEST_WaitForCompact();
 
+  int non_level0_num_files = 0;
+  for (int i = 1; i < options.num_levels; i++) {
+    non_level0_num_files += NumTableFilesAtLevel(i, 1);
+  }
+  ASSERT_EQ(non_level0_num_files, 0);
+
   // Stage 2: reopen with universal compaction, num_levels=4
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = 4;
@@ -1107,7 +1125,6 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
   for (int i = max_key1 + 1; i <= max_key2; i++) {
     // each value is 10K
     ASSERT_OK(Put(1, Key(i), RandomString(&rnd, 10000)));
-    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
   }
   ASSERT_OK(Flush(1));
   dbfull()->TEST_WaitForCompact();
@@ -1138,7 +1155,6 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
   for (int i = max_key2 + 1; i <= max_key3; i++) {
     // each value is 10K
     ASSERT_OK(Put(1, Key(i), RandomString(&rnd, 10000)));
-    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
   }
   ASSERT_OK(Flush(1));
   dbfull()->TEST_WaitForCompact();
@@ -1150,7 +1166,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
   if (!Snappy_Supported()) {
     return;
   }
-  Options options = CurrentOptions();
+  Options options;
   options.db_paths.emplace_back(dbname_, 500 * 1024);
   options.db_paths.emplace_back(dbname_ + "_2", 1024 * 1024 * 1024);
   options.compaction_style = kCompactionStyleUniversal;
@@ -1162,6 +1178,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
   options.num_levels = 1;
   options.memtable_factory.reset(
       new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+  options = CurrentOptions(options);
 
   std::vector<std::string> filenames;
   env_->GetChildren(options.db_paths[1].path, &filenames);

db/db_wal_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/dbformat.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/dbformat.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -465,12 +465,6 @@ class InternalKeySliceTransform : public SliceTransform {
   const SliceTransform* const transform_;
 };
 
-// Read the key of a record from a write batch.
-// if this record represent the default column family then cf_record
-// must be passed as false, otherwise it must be passed as true.
-extern bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key,
-                                       bool cf_record);
-
 // Read record from a write batch piece from input.
 // tag, column_family, key, value and blob are return values. Callers own the
 // Slice they point to.

db/dbformat_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/deletefile_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -37,7 +37,6 @@ class DeleteFileTest : public testing::Test {
   DeleteFileTest() {
     db_ = nullptr;
     env_ = Env::Default();
-    options_.delete_obsolete_files_period_micros = 0;  // always do full purge
     options_.enable_thread_tracking = true;
     options_.write_buffer_size = 1024*1024*1000;
     options_.target_file_size_base = 1024*1024*1000;

db/event_helpers.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/event_helpers.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/experimental.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/fault_injection_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2015, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/file_indexer.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/file_indexer.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/file_indexer_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/filename.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/filename.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/filename_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/flush_job.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -66,7 +66,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
                    JobContext* job_context, LogBuffer* log_buffer,
                    Directory* db_directory, Directory* output_file_directory,
                    CompressionType output_compression, Statistics* stats,
-                   EventLogger* event_logger, bool measure_io_stats)
+                   EventLogger* event_logger)
     : dbname_(dbname),
       cfd_(cfd),
       db_options_(db_options),
@@ -83,8 +83,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
       output_file_directory_(output_file_directory),
       output_compression_(output_compression),
       stats_(stats),
-      event_logger_(event_logger),
-      measure_io_stats_(measure_io_stats) {
+      event_logger_(event_logger) {
   // Update the thread status to indicate flush.
   ReportStartedFlush();
   TEST_SYNC_POINT("FlushJob::FlushJob()");
@@ -122,21 +121,6 @@ void FlushJob::RecordFlushIOStats() {
 Status FlushJob::Run(FileMetaData* file_meta) {
   AutoThreadOperationStageUpdater stage_run(
       ThreadStatus::STAGE_FLUSH_RUN);
-  // I/O measurement variables
-  PerfLevel prev_perf_level = PerfLevel::kEnableTime;
-  uint64_t prev_write_nanos = 0;
-  uint64_t prev_fsync_nanos = 0;
-  uint64_t prev_range_sync_nanos = 0;
-  uint64_t prev_prepare_write_nanos = 0;
-  if (measure_io_stats_) {
-    prev_perf_level = GetPerfLevel();
-    SetPerfLevel(PerfLevel::kEnableTime);
-    prev_write_nanos = IOSTATS(write_nanos);
-    prev_fsync_nanos = IOSTATS(fsync_nanos);
-    prev_range_sync_nanos = IOSTATS(range_sync_nanos);
-    prev_prepare_write_nanos = IOSTATS(prepare_write_nanos);
-  }
-
   // Save the contents of the earliest memtable as a new Table
   FileMetaData meta;
   autovector<MemTable*> mems;
@@ -196,18 +180,6 @@ Status FlushJob::Run(FileMetaData* file_meta) {
   }
   stream.EndArray();
 
-  if (measure_io_stats_) {
-    if (prev_perf_level != PerfLevel::kEnableTime) {
-      SetPerfLevel(prev_perf_level);
-    }
-    stream << "file_write_nanos" << (IOSTATS(write_nanos) - prev_write_nanos);
-    stream << "file_range_sync_nanos"
-           << (IOSTATS(range_sync_nanos) - prev_range_sync_nanos);
-    stream << "file_fsync_nanos" << (IOSTATS(fsync_nanos) - prev_fsync_nanos);
-    stream << "file_prepare_write_nanos"
-           << (IOSTATS(prepare_write_nanos) - prev_prepare_write_nanos);
-  }
-
   return s;
 }
 
@@ -262,15 +234,14 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
 
       TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:output_compression",
                                &output_compression_);
-      s = BuildTable(
-          dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
-          cfd_->table_cache(), iter.get(), meta, cfd_->internal_comparator(),
-          cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
-          cfd_->GetName(), existing_snapshots_,
-          earliest_write_conflict_snapshot_, output_compression_,
-          cfd_->ioptions()->compression_opts,
-          mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
-          Env::IO_HIGH, &table_properties_, 0 /* level */);
+      s = BuildTable(dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
+                     cfd_->table_cache(), iter.get(), meta,
+                     cfd_->internal_comparator(),
+                     cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
+                     existing_snapshots_, earliest_write_conflict_snapshot_,
+                     output_compression_, cfd_->ioptions()->compression_opts,
+                     mutable_cf_options_.paranoid_file_checks,
+                     cfd_->internal_stats(), Env::IO_HIGH, &table_properties_);
       info.table_properties = table_properties_;
       LogFlush(db_options_.info_log);
     }

db/flush_job.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -62,7 +62,7 @@ class FlushJob {
            JobContext* job_context, LogBuffer* log_buffer,
            Directory* db_directory, Directory* output_file_directory,
            CompressionType output_compression, Statistics* stats,
-           EventLogger* event_logger, bool measure_io_stats);
+           EventLogger* event_logger);
 
   ~FlushJob();
 
@@ -93,7 +93,6 @@ class FlushJob {
   Statistics* stats_;
   EventLogger* event_logger_;
   TableProperties table_properties_;
-  bool measure_io_stats_;
 };
 
 }  // namespace rocksdb

db/flush_job_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -93,7 +93,7 @@ TEST_F(FlushJobTest, Empty) {
                      db_options_, *cfd->GetLatestMutableCFOptions(),
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
                      {}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
-                     nullptr, kNoCompression, nullptr, &event_logger, false);
+                     nullptr, kNoCompression, nullptr, &event_logger);
   ASSERT_OK(flush_job.Run());
   job_context.Clean();
 }
@@ -132,7 +132,7 @@ TEST_F(FlushJobTest, NonEmpty) {
                      db_options_, *cfd->GetLatestMutableCFOptions(),
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
                      {}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
-                     nullptr, kNoCompression, nullptr, &event_logger, true);
+                     nullptr, kNoCompression, nullptr, &event_logger);
   FileMetaData fd;
   mutex_.Lock();
   ASSERT_OK(flush_job.Run(&fd));
@@ -192,11 +192,11 @@ TEST_F(FlushJobTest, Snapshots) {
   }
 
   EventLogger event_logger(db_options_.info_log.get());
-  FlushJob flush_job(
-      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      *cfd->GetLatestMutableCFOptions(), env_options_, versions_.get(), &mutex_,
-      &shutting_down_, snapshots, kMaxSequenceNumber, &job_context, nullptr,
-      nullptr, nullptr, kNoCompression, nullptr, &event_logger, true);
+  FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
+                     db_options_, *cfd->GetLatestMutableCFOptions(),
+                     env_options_, versions_.get(), &mutex_, &shutting_down_,
+                     snapshots, kMaxSequenceNumber, &job_context, nullptr,
+                     nullptr, nullptr, kNoCompression, nullptr, &event_logger);
   mutex_.Lock();
   ASSERT_OK(flush_job.Run());
   mutex_.Unlock();

db/flush_scheduler.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/flush_scheduler.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/forward_iterator.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -10,16 +10,15 @@
 #include <string>
 #include <utility>
 
-#include "db/column_family.h"
+#include "db/job_context.h"
 #include "db/db_impl.h"
 #include "db/db_iter.h"
-#include "db/dbformat.h"
-#include "db/job_context.h"
+#include "db/column_family.h"
 #include "rocksdb/env.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/slice_transform.h"
 #include "table/merger.h"
-#include "util/string_util.h"
+#include "db/dbformat.h"
 #include "util/sync_point.h"
 
 namespace rocksdb {
@@ -472,15 +471,6 @@ Status ForwardIterator::status() const {
   return immutable_status_;
 }
 
-Status ForwardIterator::GetProperty(std::string prop_name, std::string* prop) {
-  assert(prop != nullptr);
-  if (prop_name == "rocksdb.iterator.super-version-number") {
-    *prop = ToString(sv_->version_number);
-    return Status::OK();
-  }
-  return Status::InvalidArgument();
-}
-
 void ForwardIterator::RebuildIterators(bool refresh_sv) {
   // Clean up
   Cleanup(refresh_sv);

db/forward_iterator.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -71,8 +71,6 @@ class ForwardIterator : public InternalIterator {
   virtual Slice key() const override;
   virtual Slice value() const override;
   virtual Status status() const override;
-  virtual Status GetProperty(std::string prop_name, std::string* prop) override;
-
   bool TEST_CheckDeletedIters(int* deleted_iters, int* num_iters);
 
  private:

db/forward_iterator_bench.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/inlineskiplist.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional
 //  grant of patent rights can be found in the PATENTS file in the same

db/inlineskiplist_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/internal_stats.cc

@@ -132,8 +132,6 @@ static const std::string is_file_deletions_enabled =
 static const std::string num_snapshots = "num-snapshots";
 static const std::string oldest_snapshot_time = "oldest-snapshot-time";
 static const std::string num_live_versions = "num-live-versions";
-static const std::string current_version_number =
-    "current-super-version-number";
 static const std::string estimate_live_data_size = "estimate-live-data-size";
 static const std::string base_level = "base-level";
 static const std::string total_sst_files_size = "total-sst-files-size";
@@ -193,8 +191,6 @@ const std::string DB::Properties::kOldestSnapshotTime =
                       rocksdb_prefix + oldest_snapshot_time;
 const std::string DB::Properties::kNumLiveVersions =
                       rocksdb_prefix + num_live_versions;
-const std::string DB::Properties::kCurrentSuperVersionNumber =
-    rocksdb_prefix + current_version_number;
 const std::string DB::Properties::kEstimateLiveDataSize =
                       rocksdb_prefix + estimate_live_data_size;
 const std::string DB::Properties::kTotalSstFilesSize =
@@ -258,8 +254,6 @@ const std::unordered_map<std::string,
      {false, nullptr, &InternalStats::HandleOldestSnapshotTime}},
     {DB::Properties::kNumLiveVersions,
      {false, nullptr, &InternalStats::HandleNumLiveVersions}},
-    {DB::Properties::kCurrentSuperVersionNumber,
-     {false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber}},
     {DB::Properties::kEstimateLiveDataSize,
      {true, nullptr, &InternalStats::HandleEstimateLiveDataSize}},
     {DB::Properties::kBaseLevel,
@@ -525,12 +519,6 @@ bool InternalStats::HandleNumLiveVersions(uint64_t* value, DBImpl* db,
   return true;
 }
 
-bool InternalStats::HandleCurrentSuperVersionNumber(uint64_t* value, DBImpl* db,
-                                                    Version* version) {
-  *value = cfd_->GetSuperVersionNumber();
-  return true;
-}
-
 bool InternalStats::HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db,
                                                  Version* version) {
   *value = db->IsFileDeletionsEnabled();

db/internal_stats.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -328,8 +328,6 @@ class InternalStats {
   bool HandleNumSnapshots(uint64_t* value, DBImpl* db, Version* version);
   bool HandleOldestSnapshotTime(uint64_t* value, DBImpl* db, Version* version);
   bool HandleNumLiveVersions(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleCurrentSuperVersionNumber(uint64_t* value, DBImpl* db,
-                                       Version* version);
   bool HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db,
                                     Version* version);
   bool HandleBaseLevel(uint64_t* value, DBImpl* db, Version* version);

db/job_context.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -22,9 +22,9 @@ class MemTable;
 struct JobContext {
   inline bool HaveSomethingToDelete() const {
     return full_scan_candidate_files.size() || sst_delete_files.size() ||
-           log_delete_files.size() || manifest_delete_files.size() ||
-           new_superversion != nullptr || superversions_to_free.size() > 0 ||
-           memtables_to_free.size() > 0 || logs_to_free.size() > 0;
+           log_delete_files.size() || new_superversion != nullptr ||
+           superversions_to_free.size() > 0 || memtables_to_free.size() > 0 ||
+           logs_to_free.size() > 0;
   }
 
   // Structure to store information for candidate files to delete.
@@ -56,9 +56,6 @@ struct JobContext {
   // a list of log files that we need to delete
   std::vector<uint64_t> log_delete_files;
 
-  // a list of manifest files that we need to delete
-  std::vector<std::string> manifest_delete_files;
-
   // a list of memtables to be free
   autovector<MemTable*> memtables_to_free;
 

db/listener_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.