Disable error as warning

The previuos change that use gcc-5 set USE_SSE to wrong flag by mistake. Fix it.

.gitignore

@@ -42,7 +42,6 @@ unity.a
 tags
 rocksdb_dump
 rocksdb_undump
-db_test2
 
 java/out
 java/target

.travis.yml

@@ -34,7 +34,7 @@ before_script:
 # as EnvPosixTest::AllocateTest expects within the Travis OpenVZ environment.
 script:
   - if [[ "${TRAVIS_OS_NAME}" == 'linux' ]]; then OPT=-DTRAVIS CLANG_FORMAT_DIFF=/tmp/clang-format-diff.py make format || true; fi
-  - OPT=-DTRAVIS V=1 make -j4 check && OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest && make clean && OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib
+  - OPT=-DTRAVIS V=1 make -j4 check && OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest && make clean && OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 check
 
 notifications:
     email:

CMakeLists.txt

@@ -13,8 +13,8 @@
 #        cd build
 # 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
 #    See thirdparty.inc for more information.
-#        sample command: cmake -G "Visual Studio 12 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
-#        OR for VS Studio 15 cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
+#        sample command: cmake -G "Visual Studio 12 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 ..
+#        OR for VS Studio 15 cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 ..
 # 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
 #                                          or simply /m ot use all avail cores)
 #        msbuild rocksdb.sln
@@ -30,9 +30,10 @@ cmake_minimum_required(VERSION 2.6)
 project(rocksdb)
 
 include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
-execute_process(COMMAND powershell -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
-execute_process(COMMAND powershell -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
-string(REGEX REPLACE "(..)_(..)_..(..).*" "\\1/\\2/\\3" DATE ${DATE})
+
+execute_process(COMMAND $ENV{COMSPEC} " /C date /T" OUTPUT_VARIABLE DATE)
+execute_process(COMMAND $ENV{COMSPEC} " /C time /T" OUTPUT_VARIABLE TIME)
+string(REGEX REPLACE "(..)/(..)/..(..).*" "\\1/\\2/\\3" DATE ${DATE})
 string(REGEX REPLACE "(..):(.....).*" " \\1:\\2" TIME ${TIME})
 string(CONCAT GIT_DATE_TIME ${DATE} ${TIME})
 
@@ -98,7 +99,6 @@ add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest)
 
 # Main library source code
 set(SOURCES
-        db/auto_roll_logger.cc
         db/builder.cc
         db/c.cc
         db/column_family.cc
@@ -114,7 +114,6 @@ set(SOURCES
         db/db_impl_debug.cc
         db/db_impl_experimental.cc
         db/db_impl_readonly.cc
-        db/db_info_dumper.cc
         db/db_iter.cc
         db/event_helpers.cc
         db/experimental.cc
@@ -146,12 +145,9 @@ set(SOURCES
         db/write_batch_base.cc
         db/write_controller.cc
         db/write_thread.cc
-        db/xfunc_test_points.cc
         memtable/hash_cuckoo_rep.cc
         memtable/hash_linklist_rep.cc
         memtable/hash_skiplist_rep.cc
-        memtable/skiplistrep.cc
-        memtable/vectorrep.cc
         port/stack_trace.cc
         port/win/env_win.cc
         port/win/port_win.cc
@@ -177,6 +173,7 @@ set(SOURCES
         table/merger.cc
         table/sst_file_writer.cc
         table/meta_blocks.cc
+        table/mock_table.cc
         table/plain_table_builder.cc
         table/plain_table_factory.cc
         table/plain_table_index.cc
@@ -185,9 +182,9 @@ set(SOURCES
         table/table_properties.cc
         table/two_level_iterator.cc
         tools/sst_dump_tool.cc
-        tools/db_bench_tool.cc
         tools/dump/db_dump_tool.cc
         util/arena.cc
+        util/auto_roll_logger.cc
         util/bloom.cc
         util/build_version.cc
         util/cache.cc
@@ -196,18 +193,17 @@ set(SOURCES
         util/comparator.cc
         util/concurrent_arena.cc
         util/crc32c.cc
-        util/delete_scheduler.cc
+        util/db_info_dumper.cc
+        util/delete_scheduler_impl.cc
         util/dynamic_bloom.cc
         util/env.cc
         util/env_hdfs.cc
         util/event_logger.cc
         util/file_util.cc
         util/file_reader_writer.cc
-        util/sst_file_manager_impl.cc
         util/filter_policy.cc
         util/hash.cc
         util/histogram.cc
-        util/histogram_windowing.cc
         util/instrumented_mutex.cc
         util/iostats_context.cc
         tools/ldb_cmd.cc
@@ -215,6 +211,7 @@ set(SOURCES
         util/logging.cc
         util/log_buffer.cc
         util/memenv.cc
+        util/mock_env.cc
         util/murmurhash.cc
         util/mutable_cf_options.cc
         util/options.cc
@@ -226,6 +223,7 @@ set(SOURCES
         util/perf_level.cc
         util/random.cc
         util/rate_limiter.cc
+        util/skiplistrep.cc
         util/slice.cc
         util/statistics.cc
         util/status.cc
@@ -239,12 +237,11 @@ set(SOURCES
         util/thread_status_updater.cc
         util/thread_status_util.cc
         util/thread_status_util_debug.cc
-        util/transaction_test_util.cc
+        util/vectorrep.cc
         util/xfunc.cc
         util/xxhash.cc
         utilities/backupable/backupable_db.cc
         utilities/checkpoint/checkpoint.cc
-        utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
         utilities/document/document_db.cc
         utilities/document/json_document.cc
         utilities/document/json_document_builder.cc
@@ -278,8 +275,6 @@ set(SOURCES
 # and linked to tests. Add test only code that is not #ifdefed for Release here.
 set(TESTUTIL_SOURCE
     db/db_test_util.cc
-    table/mock_table.cc
-    util/mock_env.cc
     util/thread_status_updater_debug.cc
 )
 
@@ -292,19 +287,8 @@ set_target_properties(rocksdb${ARTIFACT_SUFFIX} PROPERTIES COMPILE_FLAGS "-DROCK
 add_dependencies(rocksdb${ARTIFACT_SUFFIX} GenerateBuildVersion)
 target_link_libraries(rocksdb${ARTIFACT_SUFFIX} ${LIBS})
 
-if (DEFINED JNI)
-  if (${JNI} EQUAL 1)
-    message(STATUS "JNI library is enabled")
-    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
-  else()
-    message(STATUS "JNI library is disabled")
-  endif()
-else()
-  message(STATUS "JNI library is disabled")
-endif()
-
 set(APPS
-        tools/db_bench.cc
+        db/db_bench.cc
         db/memtablerep_bench.cc
         table/table_reader_bench.cc
         tools/db_stress.cc
@@ -319,7 +303,6 @@ set(APPS
 set(C_TESTS db/c_test.c)
 
 set(TESTS
-        db/auto_roll_logger_test.cc
         db/column_family_test.cc
         db/compact_files_test.cc
         db/compaction_iterator_test.cc
@@ -329,20 +312,17 @@ set(TESTS
         db/comparator_db_test.cc
         db/corruption_test.cc
         db/cuckoo_table_db_test.cc
+        db/db_iter_test.cc
+        db/db_test.cc
         db/db_compaction_filter_test.cc
         db/db_compaction_test.cc
         db/db_dynamic_level_test.cc
         db/db_inplace_update_test.cc
-        db/db_iter_test.cc
         db/db_log_iter_test.cc
-        db/db_properties_test.cc
-        db/db_table_properties_test.cc
-        db/db_tailing_iter_test.cc
-        db/db_test.cc
-        db/db_test2.cc
-        db/db_block_cache_test.cc
         db/db_universal_compaction_test.cc
         db/db_wal_test.cc
+        db/db_tailing_iter_test.cc
+        db/db_table_properties_test.cc
         db/dbformat_test.cc
         db/deletefile_test.cc
         db/fault_injection_test.cc
@@ -360,7 +340,6 @@ set(TESTS
         db/perf_context_test.cc
         db/plain_table_db_test.cc
         db/prefix_test.cc
-        db/repair_test.cc
         db/skiplist_test.cc
         db/table_properties_collector_test.cc
         db/version_builder_test.cc
@@ -384,6 +363,7 @@ set(TESTS
         tools/sst_dump_test.cc
         util/arena_test.cc
         util/autovector_test.cc
+        util/auto_roll_logger_test.cc
         util/bloom_test.cc
         util/cache_test.cc
         util/coding_test.cc
@@ -396,10 +376,8 @@ set(TESTS
         util/file_reader_writer_test.cc
         util/heap_test.cc
         util/histogram_test.cc
-        util/iostats_context_test.cc
         util/memenv_test.cc
         util/mock_env_test.cc
-        util/options_settable_test.cc
         util/options_test.cc
         util/rate_limiter_test.cc
         util/slice_transform_test.cc

DEFAULT_OPTIONS_HISTORY.md

@@ -1,9 +0,0 @@
-# RocksDB default options change log
-## 4.7.0 (4/8/2016)
-* options.write_buffer_size changes from 4MB to 64MB
-* options.target_file_size_base changes from 2MB to 64MB
-* options.max_bytes_for_level_base changes from 10MB to 256MB
-* options.soft_pending_compaction_bytes_limit changes from 0 (disabled) to 64GB
-* options.hard_pending_compaction_bytes_limit changes from 0 (disabled) to 256GB
-* table_cache_numshardbits changes from 4 to 6
-* max_file_opening_threads changes from 1 to 16 

HISTORY.md

@@ -1,29 +1,6 @@
 # Rocksdb Change Log
-## 4.7.0 (4/8/2016)
-### Public API Change
-* rename options compaction_measure_io_stats to report_bg_io_stats and include flush too.
-* Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.
 
-## 4.6.0 (3/10/2016)
-### Public API Changes
-* Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier.
-* Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signiture of Cache::Insert() is updated accordingly.
-* Tickers [NUMBER_DB_NEXT, NUMBER_DB_PREV, NUMBER_DB_NEXT_FOUND, NUMBER_DB_PREV_FOUND, ITER_BYTES_READ] are not updated immediately. The are updated when the Iterator is deleted.  
-* Add monotonically increasing counter (DB property "rocksdb.current-super-version-number") that increments upon any change to the LSM tree.
-### New Features
-* Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification.
-* Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned"
-
-## 4.5.0 (2/5/2016)
-### Public API Changes
-* Add a new perf context level between kEnableCount and kEnableTime. Level 2 now does not include timers for mutexes.
-* Statistics of mutex operation durations will not be measured by default. If you want to have them enabled, you need to set Statistics::stats_level_ to kAll.
-* DBOptions::delete_scheduler and NewDeleteScheduler() are removed, please use DBOptions::sst_file_manager and NewSstFileManager() instead
-
-### New Features
-* ldb tool now supports operations to non-default column families.
-* Add kPersistedTier to ReadTier.  This option allows Get and MultiGet to read only the persited data and skip mem-tables if writes were done with disableWAL = true.
-* Add DBOptions::sst_file_manager. Use NewSstFileManager() in include/rocksdb/sst_file_manager.h to create a SstFileManager that can be used to track the total size of SST files and control the SST files deletion rate.  
+## 4.4.1 (2/18/2016)
 
 ## 4.4.0 (1/14/2016)
 ### Public API Changes

INSTALL.md

@@ -21,7 +21,7 @@ depend on gflags. You will need to have gflags installed to run `make all`. This
 use binaries compiled by `make all` in production.
 
 * By default the binary we produce is optimized for the platform you're compiling on
-(-march=native or the equivalent). If you want to build a portable binary, add 'PORTABLE=1' before
+(-march=native). If you want to build a portable binary, add 'PORTABLE=1' before
 your make commands, like this: `PORTABLE=1 make static_lib`
 
 ## Dependencies

LANGUAGE-BINDINGS.md

@@ -1,11 +0,0 @@
-This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it.
-
-* Java - https://github.com/facebook/rocksdb/tree/master/java
-* Python - http://pyrocksdb.readthedocs.org/en/latest/
-* Perl - https://metacpan.org/pod/RocksDB
-* Node.js - https://npmjs.org/package/rocksdb
-* Go - https://github.com/tecbot/gorocksdb
-* Ruby - http://rubygems.org/gems/rocksdb-ruby
-* Haskell - https://hackage.haskell.org/package/rocksdb-haskell
-* PHP - https://github.com/Photonios/rocksdb-php
-* C# - https://github.com/warrenfalk/rocksdb-sharp

LICENSE

@@ -2,7 +2,7 @@ BSD License
 
 For rocksdb software
 
-Copyright (c) 2011-present, Facebook, Inc.
+Copyright (c) 2014, Facebook, Inc.
 All rights reserved.
 ---------------------------------------------------------------------
 

Makefile

@@ -84,8 +84,7 @@ endif
 # compile with -O2 if debug level is not 2
 ifneq ($(DEBUG_LEVEL), 2)
 OPT += -O2 -fno-omit-frame-pointer
-# Skip for archs that don't support -momit-leaf-frame-pointer
-ifeq (,$(shell $(CXX) -fsyntax-only -momit-leaf-frame-pointer -xc /dev/null 2>&1))
+ifneq ($(MACHINE),ppc64) # ppc64 doesn't support -momit-leaf-frame-pointer
 OPT += -momit-leaf-frame-pointer
 endif
 endif
@@ -144,9 +143,6 @@ else
 OPT += -DNDEBUG
 endif
 
-ifeq ($(PLATFORM), OS_SOLARIS)
-	PLATFORM_CXXFLAGS += -D _GLIBCXX_USE_C99
-endif
 ifneq ($(filter -DROCKSDB_LITE,$(OPT)),)
 	# found
 	CFLAGS += -fno-exceptions
@@ -174,15 +170,6 @@ else
 	pg = -pg
 endif
 
-# USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
-ifdef COMPILE_WITH_UBSAN
-	DISABLE_JEMALLOC=1
-	EXEC_LDFLAGS += -fsanitize=undefined
-	PLATFORM_CCFLAGS += -fsanitize=undefined
-	PLATFORM_CXXFLAGS += -fsanitize=undefined
-endif
-
-
 ifndef DISABLE_JEMALLOC
 	EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
@@ -246,12 +233,8 @@ VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)
 
 VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full
 
-BENCHTOOLOBJECTS = $(BENCH_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
-
 TESTS = \
 	db_test \
-	db_test2 \
-	db_block_cache_test \
 	db_iter_test \
 	db_log_iter_test \
 	db_compaction_filter_test \
@@ -261,7 +244,6 @@ TESTS = \
 	db_tailing_iter_test \
 	db_universal_compaction_test \
 	db_wal_test \
-	db_properties_test \
 	db_table_properties_test \
 	block_hash_index_test \
 	autovector_test \
@@ -326,7 +308,6 @@ TESTS = \
 	rate_limiter_test \
 	delete_scheduler_test \
 	options_test \
-	options_settable_test \
 	options_util_test \
 	event_logger_test \
 	cuckoo_table_builder_test \
@@ -347,8 +328,7 @@ TESTS = \
 	compact_on_deletion_collector_test \
 	compaction_job_stats_test \
 	transaction_test \
-	ldb_cmd_test \
-	iostats_context_test
+	ldb_cmd_test
 
 SUBSET :=  $(shell echo $(TESTS) |sed s/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/)
 
@@ -476,7 +456,7 @@ test_names = \
       -e '/^(\s*)(\S+)/; !$$1 and do {$$p=$$2; break};'			\
       -e 'print qq! $$p$$2!'
 
-ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
+ifeq ($(MAKECMDGOALS),check)
 # Use /dev/shm if it has the sticky bit set (otherwise, /tmp),
 # and create a randomly-named rocksdb.XXXX directory therein.
 # We'll use that directory in the "make check" rules.
@@ -606,7 +586,7 @@ ldb_tests: ldb
 crash_test: whitebox_crash_test blackbox_crash_test
 
 blackbox_crash_test: db_stress
-	python -u tools/db_crashtest.py --simple blackbox
+	python -u tools/db_crashtest.py --simple blackbox 
 	python -u tools/db_crashtest.py blackbox
 
 whitebox_crash_test: db_stress
@@ -623,18 +603,8 @@ asan_crash_test:
 	COMPILE_WITH_ASAN=1 $(MAKE) crash_test
 	$(MAKE) clean
 
-ubsan_check:
-	$(MAKE) clean
-	COMPILE_WITH_UBSAN=1 $(MAKE) check -j32
-	$(MAKE) clean
-
-ubsan_crash_test:
-	$(MAKE) clean
-	COMPILE_WITH_UBSAN=1 $(MAKE) crash_test
-	$(MAKE) clean
-
 valgrind_check: $(TESTS)
-	for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \
+	for t in $(filter-out %skiplist_test,$(TESTS)); do \
 		$(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \
 		ret_code=$$?; \
 		if [ $$ret_code -ne 0 ]; then \
@@ -642,46 +612,6 @@ valgrind_check: $(TESTS)
 		fi; \
 	done
 
-
-ifneq ($(PAR_TEST),)
-parloop:
-	ret_bad=0;							\
-	for t in $(PAR_TEST); do		\
-		echo "===== Running $$t in parallel $(NUM_PAR)";\
-		if [ $(db_test) -eq 1 ]; then \
-			seq $(J) | v="$$t" parallel --gnu 's=$(TMPD)/rdb-{};  export TEST_TMPDIR=$$s;' \
-				'timeout 2m ./db_test --gtest_filter=$$v >> $$s/log-{} 2>1'; \
-		else\
-			seq $(J) | v="./$$t" parallel --gnu 's=$(TMPD)/rdb-{};' \
-			     'export TEST_TMPDIR=$$s; timeout 10m $$v >> $$s/log-{} 2>1'; \
-		fi; \
-		ret_code=$$?; \
-		if [ $$ret_code -ne 0 ]; then \
-			ret_bad=$$ret_code; \
-			echo $$t exited with $$ret_code; \
-		fi; \
-	done; \
-	exit $$ret_bad;
-endif
-
-parallel_check: $(TESTS)
-	$(AM_V_GEN)if test "$(J)" > 1                                  \
-	    && (parallel --gnu --help 2>/dev/null) |                    \
-	        grep -q 'GNU Parallel';                                 \
-	then                                                            \
-	    echo Running in parallel $(J);			\
-	else                                                            \
-	    echo "Need to have GNU Parallel and J > 1"; exit 1;		\
-	fi;								\
-	ret_bad=0;							\
-	echo $(J);\
-	echo Test Dir: $(TMPD); \
-        seq $(J) | parallel --gnu 's=$(TMPD)/rdb-{}; rm -rf $$s; mkdir $$s'; \
-	$(MAKE)  PAR_TEST="$(shell $(test_names))" TMPD=$(TMPD) \
-		J=$(J) db_test=1 parloop; \
-	$(MAKE) PAR_TEST="$(filter-out db_test, $(TESTS))" \
-		TMPD=$(TMPD) J=$(J) db_test=0 parloop;
-
 analyze: clean
 	$(CLANG_SCAN_BUILD) --use-analyzer=$(CLANG_ANALYZER) \
 		--use-c++=$(CXX) --use-cc=$(CC) --status-bugs \
@@ -718,7 +648,7 @@ clean:
 
 tags:
 	ctags * -R
-	cscope -b `find . -name '*.cc'` `find . -name '*.h'` `find . -name '*.c'`
+	cscope -b `find . -name '*.cc'` `find . -name '*.h'`
 
 format:
 	build_tools/format-diff.sh
@@ -733,7 +663,7 @@ $(LIBRARY): $(LIBOBJECTS)
 	$(AM_V_AR)rm -f $@
 	$(AM_V_at)$(AR) $(ARFLAGS) $@ $(LIBOBJECTS)
 
-db_bench: tools/db_bench.o $(BENCHTOOLOBJECTS)
+db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL)
 	$(AM_LINK)
 
 cache_bench: util/cache_bench.o $(LIBOBJECTS) $(TESTUTIL)
@@ -808,12 +738,6 @@ slice_transform_test: util/slice_transform_test.o $(LIBOBJECTS) $(TESTHARNESS)
 db_test: db/db_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-db_test2: db/db_test2.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-db_block_cache_test: db/db_block_cache_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_log_iter_test: db/db_log_iter_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -841,9 +765,6 @@ db_universal_compaction_test: db/db_universal_compaction_test.o db/db_test_util.
 db_wal_test: db/db_wal_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-db_properties_test: db/db_properties_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_table_properties_test: db/db_table_properties_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1021,9 +942,6 @@ compact_files_test: db/compact_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
 options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-options_settable_test: util/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1048,7 +966,7 @@ manual_compaction_test: db/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
 filelock_test: util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-auto_roll_logger_test: db/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
+auto_roll_logger_test: util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 memtable_list_test: db/memtable_list_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1066,18 +984,12 @@ transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TEST
 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 	$(AM_LINK)
 
-repair_test: db/repair_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 ldb_cmd_test: tools/ldb_cmd_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 ldb: tools/ldb.o $(LIBOBJECTS)
 	$(AM_LINK)
 
-iostats_context_test: util/iostats_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
-
 #-------------------------------------------------
 # make install related stuff
 INSTALL_PATH ?= /usr/local
@@ -1120,11 +1032,7 @@ install: install-static
 # ---------------------------------------------------------------------------
 
 JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux
-ifeq ($(PLATFORM), OS_SOLARIS)
-	ARCH := $(shell isainfo -b)
-else
-	ARCH := $(shell getconf LONG_BIT)
-endif
+ARCH := $(shell getconf LONG_BIT)
 ROCKSDBJNILIB = librocksdbjni-linux$(ARCH).so
 ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linux$(ARCH).jar
 ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar
@@ -1132,19 +1040,14 @@ ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PA
 ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar
 
 ifeq ($(PLATFORM), OS_MACOSX)
-	ROCKSDBJNILIB = librocksdbjni-osx.jnilib
-	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar
+ROCKSDBJNILIB = librocksdbjni-osx.jnilib
+ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar
 ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","")
 	JAVA_INCLUDE = -I$(JAVA_HOME)/include -I $(JAVA_HOME)/include/darwin
 else
 	JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/
 endif
 endif
-ifeq ($(PLATFORM), OS_SOLARIS)
-	ROCKSDBJNILIB = librocksdbjni-solaris$(ARCH).so
-	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-solaris$(ARCH).jar
-	JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris
-endif
 
 libz.a:
 	-rm -rf zlib-1.2.8
@@ -1238,10 +1141,11 @@ jtest: rocksdbjava
 jdb_bench:
 	cd java;$(MAKE) db_bench;
 
-commit_prereq: build_tools/rocksdb-lego-determinator \
-               build_tools/precommit_checker.py
-	J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit tsan asan ubsan lite
+commit-prereq:
+	$(MAKE) clean && $(MAKE) all check;
 	$(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;
+	$(MAKE) clean && USE_CLANG=1 $(MAKE) all;
+	$(MAKE) clean && OPT=-DROCKSDB_LITE $(MAKE) static_lib;
 
 xfunc:
 	for xftest in $(XFUNC_TESTS); do \

USERS.md

@@ -45,14 +45,4 @@ Airbnb is using RocksDB as a storage engine for their personalized search servic
 Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtube.com/watch?v=MtFEVEs_2Vo
 
 ## Smyte
-[Smyte](https://www.smyte.com/) uses RocksDB as the storage layer for their core key-value storage, high-performance counters and time-windowed HyperLogLog services.
-
-## Rakuten Marketing
-[Rakuten Marketing](https://marketing.rakuten.com/) uses RocksDB as the disk cache layer for the real-time bidding service in their Performance DSP.
-
-## VWO, Wingify
-[VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed.
-
-## quasardb
-[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark. 
-quasardb uses a heavily tuned RocksDB as its persistence layer.
+[Smyte](https://www.smyte.com/) uses RocksDB as the storage layer for their core key-value storage, high-performance counters and time-windowed HyperLogLog services.

arcanist_util/config/FacebookArcanistConfiguration.php

@@ -10,184 +10,13 @@ class FacebookArcanistConfiguration extends ArcanistConfiguration {
                                  ArcanistBaseWorkflow $workflow,
                                  $error_code) {
     if ($command == 'diff' && !$workflow->isRawDiffSource()) {
-      $this->startTestsInJenkins($workflow);
-      $this->startTestsInSandcastle($workflow);
+      $this->maybePushToJenkins($workflow);
     }
   }
 
-  //////////////////////////////////////////////////////////////////////
-  /*  Run tests in sandcastle */
-  function postURL($diffID, $url) {
-    $cmd = 'echo \'{"diff_id": "' . $diffID . '", '
-           . '"name":"click here for sandcastle tests for D' . $diffID . '", '
-           . '"link":"' . $url . '"}\' | '
-           . 'http_proxy=fwdproxy.any.facebook.com:8080 '
-           . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit '
-           . 'differential.updateunitresults';
-    shell_exec($cmd);
-  }
-
-  function updateTestCommand($diffID, $test, $status) {
-    $cmd = 'echo \'{"diff_id": "' . $diffID . '", '
-           . '"name":"' . $test . '", '
-           . '"result":"' . $status . '"}\' | '
-           . 'http_proxy=fwdproxy.any.facebook.com:8080 '
-           . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit '
-           . 'differential.updateunitresults';
-    return $cmd;
-  }
-
-  function updateTest($diffID, $test) {
-    shell_exec($this->updateTestCommand($diffID, $test, "waiting"));
-  }
-
-  function getSteps($diffID, $username, $test) {
-    $arcrc_content = exec("cat ~/.arcrc | gzip -f | base64 -w0");
-
-    // Sandcastle machines don't have arc setup. We copy the user certificate
-    // and authenticate using that in sandcastle
-    $setup = array(
-      "name" => "Setup arcrc",
-      "shell" => "echo " . $arcrc_content . " | base64 --decode"
-                 . " | gzip -d > ~/.arcrc",
-      "user" => "root"
-    );
-
-    // arc demands certain permission on its config
-    $fix_permission = array(
-      "name" => "Fix environment",
-      "shell" => "chmod 600 ~/.arcrc",
-      "user" => "root"
-    );
-
-    // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
-    // git thinks it is uncommited change
-    $fix_git_ignore = array(
-      "name" => "Fix git ignore",
-      "shell" => "echo fbcode >> .git/info/exclude",
-      "user" => "root"
-    );
-
-    // Patch the code (keep your fingures crossed)
-    $patch = array(
-      "name" => "Patch " . $diffID,
-      "shell" => "HTTPS_PROXY=fwdproxy:8080 arc --arcrc-file ~/.arcrc "
-                  . "patch --diff " . $diffID,
-      "user" => "root"
-    );
-
-    // Clean up the user arc config we are using
-    $cleanup = array(
-      "name" => "Arc cleanup",
-      "shell" => "rm -f ~/.arcrc",
-      "user" => "root"
-    );
-
-    // Construct the steps in the order of execution
-    $steps[] = $setup;
-    $steps[] = $fix_permission;
-    $steps[] = $fix_git_ignore;
-    $steps[] = $patch;
-
-    // Run the actual command
-    $this->updateTest($diffID, $test);
-    $cmd = $this->updateTestCommand($diffID, $test, "running") . ";"
-           . "(./build_tools/precommit_checker.py " . $test
-           . "&& "
-           . $this->updateTestCommand($diffID, $test, "pass") . ")"
-           . "|| " . $this->updateTestCommand($diffID, $test, "fail")
-           . "; cat /tmp/precommit-check.log"
-           . "; for f in `ls t/log-*`; do echo \$f; cat \$f; done";
-
-    $run_test = array(
-      "name" => "Run " . $test,
-      "shell" => $cmd,
-      "user" => "root",
-    );
-
-    $steps[] = $run_test;
-    $steps[] = $cleanup;
-
-    return $steps;
-  }
-
-  function startTestsInSandcastle($workflow) {
-    // extract information we need from workflow or CLI
-    $diffID = $workflow->getDiffId();
-    $username = exec("whoami");
-
-    if ($diffID == null || $username == null) {
-      // there is no diff and we can't extract username
-      // we cannot schedule sandcasstle job
-      return;
-    }
-
-    if (strcmp(getenv("ROCKSDB_CHECK_ALL"), 1) == 0) {
-      // extract all tests from the CI definition
-      $output = file_get_contents("build_tools/rocksdb-lego-determinator");
-      preg_match_all('/[ ]{2}([a-zA-Z0-9_]+)[\)]{1}/', $output, $matches);
-      $tests = $matches[1];
-    } else {
-      // manually list of tests we want to run in sandcastle
-      $tests = array(
-        "unit", "unit_481", "clang_unit", "tsan", "asan", "lite", "valgrind"
-      );
-    }
-
-    // construct a job definition for each test and add it to the master plan
-    foreach ($tests as $test) {
-      $arg[] = array(
-        "name" => "RocksDB diff " . $diffID . " test " . $test,
-        "steps" => $this->getSteps($diffID, $username, $test)
-      );
-    }
-
-    // we cannot submit the parallel execution master plan to sandcastle
-    // we need supply the job plan as a determinator
-    // so we construct a small job that will spit out the master job plan
-    // which sandcastle will parse and execute
-    // Why compress ? Otherwise we run over the max string size.
-    $cmd = "echo " . base64_encode(json_encode($arg))
-           . " | gzip -f | base64 -w0";
-    $arg_encoded = shell_exec($cmd);
-
-    $command = array(
-      "name" => "Run diff " . $diffID . "for user " . $username,
-      "steps" => array()
-    );
-
-    $command["steps"][] = array(
-      "name" => "Generate determinator",
-      "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d"
-                 . " | base64 --decode",
-      "determinator" => true,
-      "user" => "root"
-    );
-
-    // submit to sandcastle
-    $url = 'https://interngraph.intern.facebook.com/sandcastle/generate?'
-            .'command=SandcastleUniversalCommand'
-            .'&vcs=rocksdb-git&revision=origin%2Fmaster&type=lego'
-            .'&user=krad&alias=rocksdb-precommit'
-            .'&command-args=' . urlencode(json_encode($command));
-
-    $cmd = 'https_proxy= HTTPS_PROXY= curl -s -k -F app=659387027470559 '
-            . '-F token=AeO_3f2Ya3TujjnxGD4 "' . $url . '"';
-
-    $output = shell_exec($cmd);
-
-    // extract sandcastle URL from the response
-    preg_match('/url": "(.+)"/', $output, $sandcastle_url);
-
-    echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n";
-
-    // Ask phabricator to display it on the diff UI
-    $this->postURL($diffID, $sandcastle_url[1]);
-  }
-
   //////////////////////////////////////////////////////////////////////
   /* Send off builds to jenkins */
-  function startTestsInJenkins($workflow) {
+  function maybePushToJenkins($workflow) {
     $diffID = $workflow->getDiffID();
     if ($diffID === null) {
       return;
@@ -202,4 +31,5 @@ class FacebookArcanistConfiguration extends ArcanistConfiguration {
                ."buildWithParameters?token=AUTH&DIFF_ID=$diffID";
     system("curl --noproxy '*' \"$url\" > /dev/null 2>&1");
   }
+
 }

arcanist_util/cpp_linter/cpplint.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+# Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.

build_tools/build_detect_platform

@@ -184,19 +184,17 @@ if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
     # Also don't need any compilation tests if compiling on fbcode
     true
 else
-    if ! test $ROCKSDB_DISABLE_FALLOCATE; then
-        # Test whether fallocate is available
-        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <fcntl.h>
-          #include <linux/falloc.h>
-          int main() {
-      int fd = open("/dev/null", 0);
-      fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
-          }
+    # Test whether fallocate is available
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <fcntl.h>
+      #include <linux/falloc.h>
+      int main() {
+	int fd = open("/dev/null", 0);
+  fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
     fi
 
     # Test whether Snappy library is installed
@@ -356,15 +354,7 @@ if test "$USE_SSE"; then
   # if Intel SSE instruction set is supported, set USE_SSE=1
   COMMON_FLAGS="$COMMON_FLAGS -msse -msse4.2 "
 elif test -z "$PORTABLE"; then
-  if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
-    # Tune for this POWER processor, treating '+' models as base models
-    POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
-    COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
-    COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
-  else
-    COMMON_FLAGS="$COMMON_FLAGS -march=native "
-  fi
+  COMMON_FLAGS="$COMMON_FLAGS -march=native "
 fi
 
 PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"

build_tools/dependencies_4.8.1.sh

@@ -1,16 +1,16 @@
 GCC_BASE=/mnt/gvfs/third-party2/gcc/ebc96bc2fb751b5a0300b8d91a95bdf24ac1d88b/4.8.1/centos6-native/cc6c9dc
-CLANG_BASE=/mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/9d9ecb9/
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/f97108c12512b3b0789ac4515d836bdb1eae1142/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
+CLANG_BASE=/mnt/vol/engshare/fbcode/third-party2/clang/3.7.1/centos6-native/9d9ecb9/
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ea2fd1278810d3af2ea52218d2767e09d786dbd0/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
 GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f5484f168c0e4d19823d41df052c5870c6e575a4/2.17/gcc-4.8.1-glibc-2.17/99df8fc
 SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
 ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
 BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
 LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.8.1-glibc-2.17/c3f970a
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/cb6c4880fcb4fee471574ba6af63a3882155a16a/0.5.1/gcc-4.8.1-glibc-2.17/c3f970a
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9c833012dea9ae6de1df2cbdcb7aebc57ba2cb39/0.5.0/gcc-4.8.1-glibc-2.17/c3f970a
 GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/d9acac8a9a2c9378eb696e22ffa8bd0e97d9206b/master/gcc-4.8.1-glibc-2.17/8d31e51
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/378c8f95cf28c833e121ca73432101e451c3e62a/4.0.3/gcc-4.8.1-glibc-2.17/8d31e51
 NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/41bfa3759ce52c071f5fd547ec9ecd2522929f0a/trunk/gcc-4.8.1-glibc-2.17/675d945
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/303048f72efc92ae079e62dfc84823401aecfd94/trunk/gcc-4.8.1-glibc-2.17/675d945
 KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.8.1-glibc-2.17/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/71454c53fffcb716a0beb9a90047aff7fb5c984a/2.26/centos6-native/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/a5b8152b2a15ce8a98808cf954fbccec825a97bc/2.25/centos6-native/da39a3e
 VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/af85c56f424cd5edfc2c97588299b44ecdec96bb/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a

build_tools/make_new_version.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-#  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+#  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 #  This source code is licensed under the BSD-style license found in the
 #  LICENSE file in the root directory of this source tree. An additional grant
 #  of patent rights can be found in the PATENTS file in the same directory.
@@ -10,7 +10,7 @@ then
   GIT="git"
 fi
 
-# Print out the colored progress info so that it can be brainlessly
+# Print out the colored progress info so that it can be brainlessly 
 # distinguished by users.
 function title() {
   echo -e "\033[1;32m$*\033[0m"

build_tools/precommit_checker.py

@@ -1,198 +0,0 @@
-#!/usr/local/fbcode/gcc-4.8.1-glibc-2.17-fb/bin/python2.7
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-import argparse
-import commands
-import subprocess
-import sys
-import re
-import os
-import time
-
-#
-# Simple logger
-#
-
-
-class Log:
-
-    LOG_FILE = "/tmp/precommit-check.log"
-
-    def __init__(self):
-        self.filename = Log.LOG_FILE
-        self.f = open(self.filename, 'w+', 0)
-
-    def caption(self, str):
-        line = "\n##### %s #####\n" % str
-        if self.f:
-            self.f.write("%s \n" % line)
-        else:
-            print(line)
-
-    def error(self, str):
-        data = "\n\n##### ERROR ##### %s" % str
-        if self.f:
-            self.f.write("%s \n" % data)
-        else:
-            print(data)
-
-    def log(self, str):
-        if self.f:
-            self.f.write("%s \n" % str)
-        else:
-            print(str)
-
-#
-# Shell Environment
-#
-
-
-class Env(object):
-
-    def __init__(self, tests):
-        self.tests = tests
-        self.log = Log()
-
-    def shell(self, cmd, path=os.getcwd()):
-        if path:
-            os.chdir(path)
-
-        self.log.log("==== shell session ===========================")
-        self.log.log("%s> %s" % (path, cmd))
-        status = subprocess.call("cd %s; %s" % (path, cmd), shell=True,
-                                 stdout=self.log.f, stderr=self.log.f)
-        self.log.log("status = %s" % status)
-        self.log.log("============================================== \n\n")
-        return status
-
-    def GetOutput(self, cmd, path=os.getcwd()):
-        if path:
-            os.chdir(path)
-
-        self.log.log("==== shell session ===========================")
-        self.log.log("%s> %s" % (path, cmd))
-        status, out = commands.getstatusoutput(cmd)
-        self.log.log("status = %s" % status)
-        self.log.log("out = %s" % out)
-        self.log.log("============================================== \n\n")
-        return status, out
-
-#
-# Pre-commit checker
-#
-
-
-class PreCommitChecker(Env):
-
-    def __init__(self, tests):
-        Env.__init__(self, tests)
-
-    #
-    #   Get commands for a given job from the determinator file
-    #
-    def get_commands(self, test):
-        status, out = self.GetOutput(
-            "build_tools/rocksdb-lego-determinator %s" % test, ".")
-        return status, out
-
-    #
-    # Run a specific CI job
-    #
-    def run_test(self, test):
-        self.log.caption("Running test %s locally" % test)
-
-        # get commands for the CI job determinator
-        status, cmds = self.get_commands(test)
-        if status != 0:
-            self.log.error("Error getting commands for test %s" % test)
-            return False
-
-        # Parse the JSON to extract the commands to run
-        cmds = re.findall("'shell':'([^\']*)'", cmds)
-
-        if len(cmds) == 0:
-            self.log.log("No commands found")
-            return False
-
-        # Run commands
-        for cmd in cmds:
-            # Replace J=<..> with the local environment variable
-            if "J" in os.environ:
-                cmd = cmd.replace("J=1", "J=%s" % os.environ["J"])
-                cmd = cmd.replace("make ", "make -j%s " % os.environ["J"])
-            # Run the command
-            status = self.shell(cmd, ".")
-            if status != 0:
-                self.log.error("Error running command %s for test %s"
-                               % (cmd, test))
-                return False
-
-        return True
-
-    #
-    # Run specified CI jobs
-    #
-    def run_tests(self):
-        if not self.tests:
-            self.log.error("Invalid args. Please provide tests")
-            return False
-
-        self.print_separator()
-        self.print_row("TEST", "RESULT")
-        self.print_separator()
-
-        for test in self.tests:
-            start_time = time.time()
-            self.print_test(test)
-            result = self.run_test(test)
-            elapsed_min = (time.time() - start_time) / 60
-            if not result:
-                self.log.error("Error running test %s" % test)
-                self.print_result("FAIL (%dm)" % elapsed_min)
-                return False
-            self.print_result("PASS (%dm)" % elapsed_min)
-
-        self.print_separator()
-        return True
-
-    #
-    # Print a line
-    #
-    def print_separator(self):
-        print("".ljust(60, "-"))
-
-    #
-    # Print two colums
-    #
-    def print_row(self, c0, c1):
-        print("%s%s" % (c0.ljust(40), c1.ljust(20)))
-
-    def print_test(self, test):
-        print(test.ljust(40), end="")
-        sys.stdout.flush()
-
-    def print_result(self, result):
-        print(result.ljust(20))
-
-#
-# Main
-#
-parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.')
-
-# <test ....>
-parser.add_argument('test', nargs='+',
-                    help='CI test(s) to run. e.g: unit punit asan tsan ubsan')
-
-print("Please follow log %s" % Log.LOG_FILE)
-
-args = parser.parse_args()
-checker = PreCommitChecker(args.test)
-
-if not checker.run_tests():
-    print("Error running tests. Please check log file %s" % Log.LOG_FILE)
-    sys.exit(1)
-
-sys.exit(0)

build_tools/rocksdb-lego-determinator

@@ -67,7 +67,6 @@ ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
 LITE="OPT=-DROCKSDB_LITE"
 TSAN="COMPILE_WITH_TSAN=1"
-UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 PARSER="'parser':'egrep \'Failure|^#|Abort|Expected|Actual|GoogleTestFailure|^==\''"
 
@@ -377,48 +376,6 @@ ASAN_CRASH_TEST_COMMANDS="[
     }
 ]"
 
-#
-# RocksDB test under undefined behavior sanitizer
-#
-UBSAN_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb Unit Test under UBSAN',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Test RocksDB debug under UBSAN',
-                'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make J=1 ubsan_check',
-                'user':'root',
-                $PARSER
-            }
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB crash testing under udnefined behavior sanitizer
-#
-UBSAN_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb crash test under UBSAN',
-        'oncall':'$ONCALL',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug ubsan_crash_test',
-                'timeout': 86400,
-                'shell':'$SHM $DEBUG make J=1 ubsan_crash_test',
-                'user':'root',
-                $PARSER
-            },
-        ],
-        $REPORT
-    }
-]"
-
 #
 # RocksDB unit test under valgrind
 #
@@ -505,9 +462,9 @@ run_format_compatible()
   if [ -e "build_detect_platform" ]
   then
     sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
-    rm -rf build_detect_platform
+    rm -rf build_detect_platform 
     cp $TEST_TMPDIR/temp_build_file build_detect_platform
-    chmod +x build_detect_platform
+    chmod +x build_detect_platform 
   fi
 
   make ldb -j32
@@ -675,12 +632,6 @@ case $1 in
   asan_crash)
     echo $ASAN_CRASH_TEST_COMMANDS
     ;;
-  ubsan)
-    echo $UBSAN_TEST_COMMANDS
-    ;;
-  ubsan_crash)
-    echo $UBSAN_CRASH_TEST_COMMANDS
-    ;;
   valgrind)
     echo $VALGRIND_TEST_COMMANDS
     ;;

build_tools/run_ci_db_test.ps1

@@ -9,7 +9,7 @@ Param(
   [string]$WorkFolder = "",  # Direct tests to use that folder
   [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
   [string]$Exclude = "", # Expect a comma separated list, no spaces
-  [string]$Run = "db_test",  # Run db_test|db_test2|tests|testname1,testname2...
+  [string]$Run = "db_test",  # Run db_test|tests|testname1,testname2...
    # Number of async tasks that would run concurrently. Recommend a number below 64.
    # However, CPU utlization really depends on the storage media. Recommend ram based disk.
    # a value of 1 will run everything serially
@@ -46,25 +46,15 @@ if($WorkFolder -eq "") {
   $Env:TEST_TMPDIR = $WorkFolder
 }
 
-Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
-
 # Use JEMALLOC executables
-if($Run -ceq "db_test" -or
-   $Run -ceq "db_test2" ) {
-
-   $file_name = $Run
-
-   if($EnableJE) {
-     $file_name += "_je"
-   }
-
-   $file_name += ".exe"
-
-   $db_test = -Join ($BinariesFolder, $file_name)
-
-   Write-Output "Binaries: $BinariesFolder db_test: $db_test"
+if($EnableJE) {
+    $db_test = -Join ($BinariesFolder, "db_test_je.exe")
+} else {
+    $db_test = -Join ($BinariesFolder, "db_test.exe")
 }
 
+Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
+Write-Output "Binaries: $BinariesFolder db_test: $db_test"
 
 #Exclusions that we do not want to run
 $ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
@@ -163,8 +153,7 @@ function Discover-TestBinaries([string]$Pattern, $HashTable) {
 
 $TestsToRun = [ordered]@{}
 
-if($Run -ceq "db_test" -or
-   $Run -ceq "db_test2") {
+if($Run -ceq "db_test") {
     Normalize-DbTests -HashTable $TestsToRun
 } elseif($Run -ceq "tests") {
     if($EnableJE) {
@@ -236,8 +225,7 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
             Write-Host "Starting $k"
             $log_path = ($TestToLog.$k)
 
-            if($Run -ceq "db_test" -or
-               $Run -ceq "db_test2") {
+            if($Run -ceq "db_test") {
               $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
             } else {
               [string]$Exe =  -Join ($BinariesFolder, $k)

build_tools/update_dependencies.sh

@@ -65,7 +65,7 @@ echo "Writing dependencies to $OUTPUT"
 
 # Compilers locations
 GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.9.x/centos6-native/*/`
-CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
 
 log_variable GCC_BASE
 log_variable CLANG_BASE
@@ -102,7 +102,7 @@ echo "Writing 4.8.1 dependencies to $OUTPUT"
 
 # Compilers locations
 GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/`
-CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
 
 log_variable GCC_BASE
 log_variable CLANG_BASE

db/builder.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -41,17 +41,13 @@ TableBuilder* NewTableBuilder(
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
-    WritableFileWriter* file, const CompressionType compression_type,
+    uint32_t column_family_id, WritableFileWriter* file,
+    const CompressionType compression_type,
     const CompressionOptions& compression_opts, const bool skip_filters) {
-  assert((column_family_id ==
-          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
-         column_family_name.empty());
   return ioptions.table_factory->NewTableBuilder(
       TableBuilderOptions(ioptions, internal_comparator,
                           int_tbl_prop_collector_factories, compression_type,
-                          compression_opts, skip_filters,
-                          column_family_name),
+                          compression_opts, skip_filters),
       column_family_id, file);
 }
 
@@ -62,16 +58,12 @@ Status BuildTable(
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
-    std::vector<SequenceNumber> snapshots,
+    uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
     const CompressionType compression,
     const CompressionOptions& compression_opts, bool paranoid_file_checks,
     InternalStats* internal_stats, const Env::IOPriority io_priority,
-    TableProperties* table_properties, int level) {
-  assert((column_family_id ==
-          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
-         column_family_name.empty());
+    TableProperties* table_properties) {
   // Reports the IOStats for flush for every following bytes.
   const size_t kReportFlushIOStatsEvery = 1048576;
   Status s;
@@ -95,8 +87,7 @@ Status BuildTable(
 
       builder = NewTableBuilder(
           ioptions, internal_comparator, int_tbl_prop_collector_factories,
-          column_family_id, column_family_name, file_writer.get(), compression,
-          compression_opts);
+          column_family_id, file_writer.get(), compression, compression_opts);
     }
 
     MergeHelper merge(env, internal_comparator.user_comparator(),
@@ -158,8 +149,7 @@ Status BuildTable(
           ReadOptions(), env_options, internal_comparator, meta->fd, nullptr,
           (internal_stats == nullptr) ? nullptr
                                       : internal_stats->GetFileReadHist(0),
-          false /* for_compaction */, nullptr /* arena */,
-          false /* skip_filter */, level));
+          false));
       s = it->status();
       if (s.ok() && paranoid_file_checks) {
         for (it->SeekToFirst(); it->Valid(); it->Next()) {

db/builder.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -33,16 +33,13 @@ class WritableFileWriter;
 class InternalStats;
 class InternalIterator;
 
-// @param column_family_name Name of the column family that is also identified
-//    by column_family_id, or empty string if unknown. It must outlive the
-//    TableBuilder returned by this function.
 TableBuilder* NewTableBuilder(
     const ImmutableCFOptions& options,
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
-    WritableFileWriter* file, const CompressionType compression_type,
+    uint32_t column_family_id, WritableFileWriter* file,
+    const CompressionType compression_type,
     const CompressionOptions& compression_opts,
     const bool skip_filters = false);
 
@@ -51,9 +48,6 @@ TableBuilder* NewTableBuilder(
 // *meta will be filled with metadata about the generated table.
 // If no data is present in *iter, meta->file_size will be set to
 // zero, and no Table file will be produced.
-//
-// @param column_family_name Name of the column family that is also identified
-//    by column_family_id, or empty string if unknown.
 extern Status BuildTable(
     const std::string& dbname, Env* env, const ImmutableCFOptions& options,
     const EnvOptions& env_options, TableCache* table_cache,
@@ -61,13 +55,12 @@ extern Status BuildTable(
     const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    uint32_t column_family_id, const std::string& column_family_name,
-    std::vector<SequenceNumber> snapshots,
+    uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
     const CompressionType compression,
     const CompressionOptions& compression_opts, bool paranoid_file_checks,
     InternalStats* internal_stats,
     const Env::IOPriority io_priority = Env::IO_HIGH,
-    TableProperties* table_properties = nullptr, int level = -1);
+    TableProperties* table_properties = nullptr);
 
 }  // namespace rocksdb

db/c.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1288,11 +1288,6 @@ void rocksdb_block_based_options_set_cache_index_and_filter_blocks(
   options->rep.cache_index_and_filter_blocks = v;
 }
 
-void rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(
-    rocksdb_block_based_table_options_t* options, unsigned char v) {
-  options->rep.pin_l0_filter_and_index_blocks_in_cache = v;
-}
-
 void rocksdb_block_based_options_set_skip_table_builder_flush(
     rocksdb_block_based_table_options_t* options, unsigned char v) {
   options->rep.skip_table_builder_flush = v;

db/column_family.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -147,10 +147,13 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
                                     const ColumnFamilyOptions& src) {
   ColumnFamilyOptions result = src;
   result.comparator = icmp;
-  size_t clamp_max = std::conditional<
-      sizeof(size_t) == 4, std::integral_constant<size_t, 0xffffffff>,
-      std::integral_constant<size_t, 64ull << 30>>::type::value;
-  ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, clamp_max);
+#ifdef OS_MACOSX
+  // TODO(icanadi) make write_buffer_size uint64_t instead of size_t
+  ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, ((size_t)1) << 30);
+#else
+  ClipToRange(&result.write_buffer_size,
+              ((size_t)64) << 10, ((size_t)64) << 30);
+#endif
   // if user sets arena_block_size, we trust user to use this value. Otherwise,
   // calculate a proper value from writer_buffer_size;
   if (result.arena_block_size <= 0) {
@@ -240,17 +243,6 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
          result.level0_slowdown_writes_trigger,
          result.level0_file_num_compaction_trigger);
   }
-
-  if (result.soft_pending_compaction_bytes_limit == 0) {
-    result.soft_pending_compaction_bytes_limit =
-        result.hard_pending_compaction_bytes_limit;
-  } else if (result.hard_pending_compaction_bytes_limit > 0 &&
-             result.soft_pending_compaction_bytes_limit >
-                 result.hard_pending_compaction_bytes_limit) {
-    result.soft_pending_compaction_bytes_limit =
-        result.hard_pending_compaction_bytes_limit;
-  }
-
   if (result.level_compaction_dynamic_level_bytes) {
     if (result.compaction_style != kCompactionStyleLevel ||
         db_options.db_paths.size() > 1U) {
@@ -525,21 +517,6 @@ std::unique_ptr<WriteControllerToken> SetupDelay(
   }
   return write_controller->GetDelayToken(write_rate);
 }
-
-int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
-                                    int level0_slowdown_writes_trigger) {
-  // SanitizeOptions() ensures it.
-  assert(level0_file_num_compaction_trigger <= level0_slowdown_writes_trigger);
-
-  // 1/4 of the way between L0 compaction trigger threshold and slowdown
-  // condition.
-  // Or twice as compaction trigger, if it is smaller.
-  return std::min(level0_file_num_compaction_trigger * 2,
-                  level0_file_num_compaction_trigger +
-                      (level0_slowdown_writes_trigger -
-                       level0_file_num_compaction_trigger) /
-                          4);
-}
 }  // namespace
 
 void ColumnFamilyData::RecalculateWriteStallConditions(
@@ -558,6 +535,21 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
           "(waiting for flush), max_write_buffer_number is set to %d",
           name_.c_str(), imm()->NumNotFlushed(),
           mutable_cf_options.max_write_buffer_number);
+    } else if (mutable_cf_options.max_write_buffer_number > 3 &&
+               imm()->NumNotFlushed() >=
+                   mutable_cf_options.max_write_buffer_number - 1) {
+      write_controller_token_ =
+          SetupDelay(ioptions_.delayed_write_rate, write_controller,
+                     compaction_needed_bytes, prev_compaction_needed_bytes_,
+                     mutable_cf_options.disable_auto_compactions);
+      internal_stats_->AddCFStats(InternalStats::MEMTABLE_SLOWDOWN, 1);
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+          "[%s] Stalling writes because we have %d immutable memtables "
+          "(waiting for flush), max_write_buffer_number is set to %d "
+          "rate %" PRIu64,
+          name_.c_str(), imm()->NumNotFlushed(),
+          mutable_cf_options.max_write_buffer_number,
+          write_controller->delayed_write_rate());
     } else if (vstorage->l0_delay_trigger_count() >=
                mutable_cf_options.level0_stop_writes_trigger) {
       write_controller_token_ = write_controller->GetStopToken();
@@ -579,21 +571,6 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
           "[%s] Stopping writes because of estimated pending compaction "
           "bytes %" PRIu64,
           name_.c_str(), compaction_needed_bytes);
-    } else if (mutable_cf_options.max_write_buffer_number > 3 &&
-               imm()->NumNotFlushed() >=
-                   mutable_cf_options.max_write_buffer_number - 1) {
-      write_controller_token_ =
-          SetupDelay(ioptions_.delayed_write_rate, write_controller,
-                     compaction_needed_bytes, prev_compaction_needed_bytes_,
-                     mutable_cf_options.disable_auto_compactions);
-      internal_stats_->AddCFStats(InternalStats::MEMTABLE_SLOWDOWN, 1);
-      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
-          "[%s] Stalling writes because we have %d immutable memtables "
-          "(waiting for flush), max_write_buffer_number is set to %d "
-          "rate %" PRIu64,
-          name_.c_str(), imm()->NumNotFlushed(),
-          mutable_cf_options.max_write_buffer_number,
-          write_controller->delayed_write_rate());
     } else if (mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
                vstorage->l0_delay_trigger_count() >=
                    mutable_cf_options.level0_slowdown_writes_trigger) {
@@ -625,29 +602,6 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
           "bytes %" PRIu64 " rate %" PRIu64,
           name_.c_str(), vstorage->estimated_compaction_needed_bytes(),
           write_controller->delayed_write_rate());
-    } else if (vstorage->l0_delay_trigger_count() >=
-               GetL0ThresholdSpeedupCompaction(
-                   mutable_cf_options.level0_file_num_compaction_trigger,
-                   mutable_cf_options.level0_slowdown_writes_trigger)) {
-      write_controller_token_ = write_controller->GetCompactionPressureToken();
-      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
-          "[%s] Increasing compaction threads because we have %d level-0 "
-          "files ",
-          name_.c_str(), vstorage->l0_delay_trigger_count());
-    } else if (vstorage->estimated_compaction_needed_bytes() >=
-               mutable_cf_options.soft_pending_compaction_bytes_limit / 4) {
-      // Increase compaction threads if bytes needed for compaction exceeds
-      // 1/4 of threshold for slowing down.
-      // If soft pending compaction byte limit is not set, always speed up
-      // compaction.
-      write_controller_token_ = write_controller->GetCompactionPressureToken();
-      if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) {
-        Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
-            "[%s] Increasing compaction threads because of estimated pending "
-            "compaction "
-            "bytes %" PRIu64,
-            name_.c_str(), vstorage->estimated_compaction_needed_bytes());
-      }
     } else {
       write_controller_token_.reset();
     }

db/column_family.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -465,8 +465,6 @@ class ColumnFamilySet {
   // Don't call while iterating over ColumnFamilySet
   void FreeDeadColumnFamilies();
 
-  Cache* get_table_cache() { return table_cache_; }
-
  private:
   friend class ColumnFamilyData;
   // helper function that gets called from cfd destructor

db/column_family_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -65,7 +65,6 @@ class ColumnFamilyTest : public testing::Test {
 
   ~ColumnFamilyTest() {
     Close();
-    rocksdb::SyncPoint::GetInstance()->DisableProcessing();
     Destroy();
     delete env_;
   }
@@ -1952,27 +1951,13 @@ TEST_F(ColumnFamilyTest, ReadDroppedColumnFamily) {
     PutRandomData(1, kKeysNum, 100);
     PutRandomData(2, kKeysNum, 100);
 
-    {
-      std::unique_ptr<Iterator> iterator(
-          db_->NewIterator(ReadOptions(), handles_[2]));
-      iterator->SeekToFirst();
-
-      if (iter == 0) {
-        // Drop CF two
-        ASSERT_OK(db_->DropColumnFamily(handles_[2]));
-      } else {
-        // delete CF two
-        delete handles_[2];
-        handles_[2] = nullptr;
-      }
-      // Make sure iterator created can still be used.
-      int count = 0;
-      for (; iterator->Valid(); iterator->Next()) {
-        ASSERT_OK(iterator->status());
-        ++count;
-      }
-      ASSERT_OK(iterator->status());
-      ASSERT_EQ(count, kKeysNum);
+    if (iter == 0) {
+      // Drop CF two
+      ASSERT_OK(db_->DropColumnFamily(handles_[2]));
+    } else {
+      // delete CF two
+      delete handles_[2];
+      handles_[2] = nullptr;
     }
 
     // Add bunch more data to other CFs
@@ -2014,9 +1999,7 @@ TEST_F(ColumnFamilyTest, FlushAndDropRaceCondition) {
   Reopen({options, options});
 
   rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"VersionSet::LogAndApply::ColumnFamilyDrop:0",
-        "FlushJob::WriteLevel0Table"},
-       {"VersionSet::LogAndApply::ColumnFamilyDrop:1",
+      {{"VersionSet::LogAndApply::ColumnFamilyDrop:1",
         "FlushJob::InstallResults"},
        {"FlushJob::InstallResults",
         "VersionSet::LogAndApply::ColumnFamilyDrop:2"}});
@@ -2062,6 +2045,7 @@ TEST_F(ColumnFamilyTest, FlushAndDropRaceCondition) {
 
   Close();
   Destroy();
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
 
 #ifndef ROCKSDB_LITE
@@ -2139,6 +2123,7 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
   drop_cf_thread.join();
   Close();
   Destroy();
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
   for (auto* comparator : comparators) {
     if (comparator) {
       delete comparator;
@@ -2150,9 +2135,6 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
 TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
   const uint64_t kBaseRate = 810000u;
   db_options_.delayed_write_rate = kBaseRate;
-  db_options_.base_background_compactions = 2;
-  db_options_.max_background_compactions = 6;
-
   Open({"default"});
   ColumnFamilyData* cfd =
       static_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())->cfd();
@@ -2178,7 +2160,6 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
   ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
   ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(400);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2186,7 +2167,6 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
   ASSERT_EQ(kBaseRate / 1.2,
             dbfull()->TEST_write_controler().delayed_write_rate());
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(500);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2242,7 +2222,6 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
   ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(3001);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2267,7 +2246,6 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
   ASSERT_EQ(kBaseRate / 1.2,
             dbfull()->TEST_write_controler().delayed_write_rate());
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
 
   vstorage->set_l0_delay_trigger_count(101);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2340,73 +2318,6 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
             dbfull()->TEST_write_controler().delayed_write_rate());
 }
 
-TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
-  db_options_.base_background_compactions = 2;
-  db_options_.max_background_compactions = 6;
-  Open({"default"});
-  ColumnFamilyData* cfd =
-      static_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())->cfd();
-
-  VersionStorageInfo* vstorage = cfd->current()->storage_info();
-
-  MutableCFOptions mutable_cf_options(
-      Options(db_options_, column_family_options_),
-      ImmutableCFOptions(Options(db_options_, column_family_options_)));
-
-  // Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8
-  mutable_cf_options.level0_file_num_compaction_trigger = 4;
-  mutable_cf_options.level0_slowdown_writes_trigger = 36;
-  mutable_cf_options.level0_stop_writes_trigger = 50;
-  // Speedup threshold = 200 / 4 = 50
-  mutable_cf_options.soft_pending_compaction_bytes_limit = 200;
-  mutable_cf_options.hard_pending_compaction_bytes_limit = 2000;
-
-  vstorage->TEST_set_estimated_compaction_needed_bytes(40);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
-
-  vstorage->TEST_set_estimated_compaction_needed_bytes(50);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
-
-  vstorage->TEST_set_estimated_compaction_needed_bytes(300);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
-
-  vstorage->TEST_set_estimated_compaction_needed_bytes(45);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
-
-  vstorage->set_l0_delay_trigger_count(7);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
-
-  vstorage->set_l0_delay_trigger_count(9);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
-
-  vstorage->set_l0_delay_trigger_count(6);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
-
-  // Speed up threshold = min(4 * 2, 4 + (12 - 4)/4) = 6
-  mutable_cf_options.level0_file_num_compaction_trigger = 4;
-  mutable_cf_options.level0_slowdown_writes_trigger = 16;
-  mutable_cf_options.level0_stop_writes_trigger = 30;
-
-  vstorage->set_l0_delay_trigger_count(5);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
-
-  vstorage->set_l0_delay_trigger_count(7);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
-
-  vstorage->set_l0_delay_trigger_count(3);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
-}
-
 TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
   const uint64_t kBaseRate = 810000u;
   db_options_.delayed_write_rate = kBaseRate;
@@ -2488,104 +2399,6 @@ TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
   ASSERT_EQ(kBaseRate / 1.2,
             dbfull()->TEST_write_controler().delayed_write_rate());
 }
-
-TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
-  db_options_.base_background_compactions = 2;
-  db_options_.max_background_compactions = 6;
-  column_family_options_.soft_pending_compaction_bytes_limit = 200;
-  column_family_options_.hard_pending_compaction_bytes_limit = 2000;
-  Open();
-  CreateColumnFamilies({"one"});
-  ColumnFamilyData* cfd =
-      static_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())->cfd();
-  VersionStorageInfo* vstorage = cfd->current()->storage_info();
-
-  ColumnFamilyData* cfd1 =
-      static_cast<ColumnFamilyHandleImpl*>(handles_[1])->cfd();
-  VersionStorageInfo* vstorage1 = cfd1->current()->storage_info();
-
-  MutableCFOptions mutable_cf_options(
-      Options(db_options_, column_family_options_),
-      ImmutableCFOptions(Options(db_options_, column_family_options_)));
-  // Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8
-  mutable_cf_options.level0_file_num_compaction_trigger = 4;
-  mutable_cf_options.level0_slowdown_writes_trigger = 36;
-  mutable_cf_options.level0_stop_writes_trigger = 30;
-  // Speedup threshold = 200 / 4 = 50
-  mutable_cf_options.soft_pending_compaction_bytes_limit = 200;
-  mutable_cf_options.hard_pending_compaction_bytes_limit = 2000;
-
-  MutableCFOptions mutable_cf_options1 = mutable_cf_options;
-  mutable_cf_options1.level0_slowdown_writes_trigger = 16;
-
-  vstorage->TEST_set_estimated_compaction_needed_bytes(40);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
-
-  vstorage->TEST_set_estimated_compaction_needed_bytes(60);
-  cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
-
-  vstorage1->TEST_set_estimated_compaction_needed_bytes(30);
-  cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
-
-  vstorage1->TEST_set_estimated_compaction_needed_bytes(70);
-  cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
-
-  vstorage->TEST_set_estimated_compaction_needed_bytes(20);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
-
-  vstorage1->TEST_set_estimated_compaction_needed_bytes(3);
-  cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
-
-  vstorage->set_l0_delay_trigger_count(9);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
-
-  vstorage1->set_l0_delay_trigger_count(2);
-  cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
-
-  vstorage->set_l0_delay_trigger_count(0);
-  cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
-}
-
-TEST_F(ColumnFamilyTest, LogSyncConflictFlush) {
-  Open();
-  CreateColumnFamiliesAndReopen({"one", "two"});
-
-  Put(0, "", "");
-  Put(1, "foo", "bar");
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"DBImpl::SyncWAL:BeforeMarkLogsSynced:1",
-        "ColumnFamilyTest::LogSyncConflictFlush:1"},
-       {"ColumnFamilyTest::LogSyncConflictFlush:2",
-        "DBImpl::SyncWAL:BeforeMarkLogsSynced:2"}});
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  std::thread thread([&] { db_->SyncWAL(); });
-
-  TEST_SYNC_POINT("ColumnFamilyTest::LogSyncConflictFlush:1");
-  Flush(1);
-  Put(1, "foo", "bar");
-  Flush(1);
-
-  TEST_SYNC_POINT("ColumnFamilyTest::LogSyncConflictFlush:2");
-
-  thread.join();
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  Close();
-}
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/compact_files_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -7,7 +7,6 @@
 
 #include <mutex>
 #include <string>
-#include <thread>
 #include <vector>
 
 #include "rocksdb/db.h"
@@ -143,6 +142,9 @@ TEST_F(CompactFilesTest, ObsoleteFiles) {
   }
 
   auto l0_files = collector->GetFlushedFiles();
+  CompactionOptions compact_opt;
+  compact_opt.compression = kNoCompression;
+  compact_opt.output_file_size_limit = kWriteBufferSize * 5;
   ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1));
 
   // verify all compaction input files are deleted
@@ -152,62 +154,6 @@ TEST_F(CompactFilesTest, ObsoleteFiles) {
   delete db;
 }
 
-TEST_F(CompactFilesTest, CapturingPendingFiles) {
-  Options options;
-  options.create_if_missing = true;
-  // Disable RocksDB background compaction.
-  options.compaction_style = kCompactionStyleNone;
-  // Always do full scans for obsolete files (needed to reproduce the issue).
-  options.delete_obsolete_files_period_micros = 0;
-
-  // Add listener.
-  FlushedFileCollector* collector = new FlushedFileCollector();
-  options.listeners.emplace_back(collector);
-
-  DB* db = nullptr;
-  DestroyDB(db_name_, options);
-  Status s = DB::Open(options, db_name_, &db);
-  assert(s.ok());
-  assert(db);
-
-  // Create 5 files.
-  for (int i = 0; i < 5; ++i) {
-    db->Put(WriteOptions(), "key" + ToString(i), "value");
-    db->Flush(FlushOptions());
-  }
-
-  auto l0_files = collector->GetFlushedFiles();
-  EXPECT_EQ(5, l0_files.size());
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency({
-      {"CompactFilesImpl:2", "CompactFilesTest.CapturingPendingFiles:0"},
-      {"CompactFilesTest.CapturingPendingFiles:1", "CompactFilesImpl:3"},
-  });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  // Start compacting files.
-  std::thread compaction_thread(
-      [&] { EXPECT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1)); });
-
-  // In the meantime flush another file.
-  TEST_SYNC_POINT("CompactFilesTest.CapturingPendingFiles:0");
-  db->Put(WriteOptions(), "key5", "value");
-  db->Flush(FlushOptions());
-  TEST_SYNC_POINT("CompactFilesTest.CapturingPendingFiles:1");
-
-  compaction_thread.join();
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  delete db;
-
-  // Make sure we can reopen the DB.
-  s = DB::Open(options, db_name_, &db);
-  ASSERT_TRUE(s.ok());
-  assert(db);
-  delete db;
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/compacted_db_impl.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/compacted_db_impl.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/compaction.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -162,6 +162,9 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
       deletion_compaction_(_deletion_compaction),
       inputs_(std::move(_inputs)),
       grandparents_(std::move(_grandparents)),
+      grandparent_index_(0),
+      seen_key_(false),
+      overlapped_bytes_(0),
       score_(_score),
       bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
       is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
@@ -287,6 +290,32 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
   return true;
 }
 
+bool Compaction::ShouldStopBefore(const Slice& internal_key) {
+  // Scan to find earliest grandparent file that contains key.
+  const InternalKeyComparator* icmp = &cfd_->internal_comparator();
+  while (grandparent_index_ < grandparents_.size() &&
+      icmp->Compare(internal_key,
+                    grandparents_[grandparent_index_]->largest.Encode()) > 0) {
+    if (seen_key_) {
+      overlapped_bytes_ += grandparents_[grandparent_index_]->fd.GetFileSize();
+    }
+    assert(grandparent_index_ + 1 >= grandparents_.size() ||
+           icmp->Compare(grandparents_[grandparent_index_]->largest.Encode(),
+                         grandparents_[grandparent_index_+1]->smallest.Encode())
+                         < 0);
+    grandparent_index_++;
+  }
+  seen_key_ = true;
+
+  if (overlapped_bytes_ > max_grandparent_overlap_bytes_) {
+    // Too much overlap for current output; start new output
+    overlapped_bytes_ = 0;
+    return true;
+  } else {
+    return false;
+  }
+}
+
 // Mark (or clear) each file that is being compacted
 void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
   for (size_t i = 0; i < num_input_levels(); i++) {
@@ -392,7 +421,7 @@ void Compaction::Summary(char* output, int len) {
   snprintf(output + write, len - write, "]");
 }
 
-uint64_t Compaction::OutputFilePreallocationSize() const {
+uint64_t Compaction::OutputFilePreallocationSize() {
   uint64_t preallocation_size = 0;
 
   if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||

db/compaction.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -104,7 +104,7 @@ class Compaction {
   }
 
   // Returns the LevelFilesBrief of the specified compaction input level.
-  const LevelFilesBrief* input_levels(size_t compaction_input_level) const {
+  LevelFilesBrief* input_levels(size_t compaction_input_level) {
     return &input_levels_[compaction_input_level];
   }
 
@@ -132,10 +132,12 @@ class Compaction {
   bool KeyNotExistsBeyondOutputLevel(const Slice& user_key,
                                      std::vector<size_t>* level_ptrs) const;
 
+  // Returns true iff we should stop building the current output
+  // before processing "internal_key".
+  bool ShouldStopBefore(const Slice& internal_key);
+
   // Clear all files to indicate that they are not being compacted
   // Delete this compaction from the list of running compactions.
-  //
-  // Requirement: DB mutex held
   void ReleaseCompactionFiles(Status status);
 
   // Returns the summary of the compaction in "output" with maximum "len"
@@ -147,13 +149,13 @@ class Compaction {
   double score() const { return score_; }
 
   // Is this compaction creating a file in the bottom most level?
-  bool bottommost_level() const { return bottommost_level_; }
+  bool bottommost_level() { return bottommost_level_; }
 
   // Does this compaction include all sst files?
-  bool is_full_compaction() const { return is_full_compaction_; }
+  bool is_full_compaction() { return is_full_compaction_; }
 
   // Was this compaction triggered manually by the client?
-  bool is_manual_compaction() const { return is_manual_compaction_; }
+  bool is_manual_compaction() { return is_manual_compaction_; }
 
   // Used when allow_trivial_move option is set in
   // Universal compaction. If all the input files are
@@ -166,21 +168,19 @@ class Compaction {
   // Used when allow_trivial_move option is set in
   // Universal compaction. Returns true, if the input files
   // are non-overlapping and can be trivially moved.
-  bool is_trivial_move() const { return is_trivial_move_; }
+  bool is_trivial_move() { return is_trivial_move_; }
 
   // How many total levels are there?
   int number_levels() const { return number_levels_; }
 
   // Return the MutableCFOptions that should be used throughout the compaction
   // procedure
-  const MutableCFOptions* mutable_cf_options() const {
-    return &mutable_cf_options_;
-  }
+  const MutableCFOptions* mutable_cf_options() { return &mutable_cf_options_; }
 
   // Returns the size in bytes that the output file should be preallocated to.
   // In level compaction, that is max_file_size_. In universal compaction, that
   // is the sum of all input file sizes.
-  uint64_t OutputFilePreallocationSize() const;
+  uint64_t OutputFilePreallocationSize();
 
   void SetInputVersion(Version* input_version);
 
@@ -223,14 +223,6 @@ class Compaction {
 
   CompactionReason compaction_reason() { return compaction_reason_; }
 
-  const std::vector<FileMetaData*>& grandparents() const {
-    return grandparents_;
-  }
-
-  uint64_t max_grandparent_overlap_bytes() const {
-    return max_grandparent_overlap_bytes_;
-  }
-
  private:
   // mark (or clear) all files that are being compacted
   void MarkFilesBeingCompacted(bool mark_as_compacted);
@@ -274,6 +266,10 @@ class Compaction {
   // State used to check for number of of overlapping grandparent files
   // (grandparent == "output_level_ + 1")
   std::vector<FileMetaData*> grandparents_;
+  size_t grandparent_index_;   // Index in grandparent_starts_
+  bool seen_key_;              // Some output key has been seen
+  uint64_t overlapped_bytes_;  // Bytes of overlap between current output
+                               // and grandparent files
   const double score_;         // score that was used to pick this compaction.
 
   // Is this compaction creating a file in the bottom most level?

db/compaction_iterator.cc

@@ -1,6 +1,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -14,7 +14,7 @@ CompactionIterator::CompactionIterator(
     InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
     SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
     SequenceNumber earliest_write_conflict_snapshot, Env* env,
-    bool expect_valid_internal_key, const Compaction* compaction,
+    bool expect_valid_internal_key, Compaction* compaction,
     const CompactionFilter* compaction_filter, LogBuffer* log_buffer)
     : input_(input),
       cmp_(cmp),

db/compaction_iterator.h

@@ -1,6 +1,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -42,7 +42,7 @@ class CompactionIterator {
                      std::vector<SequenceNumber>* snapshots,
                      SequenceNumber earliest_write_conflict_snapshot, Env* env,
                      bool expect_valid_internal_key,
-                     const Compaction* compaction = nullptr,
+                     Compaction* compaction = nullptr,
                      const CompactionFilter* compaction_filter = nullptr,
                      LogBuffer* log_buffer = nullptr);
 
@@ -92,7 +92,7 @@ class CompactionIterator {
   const SequenceNumber earliest_write_conflict_snapshot_;
   Env* env_;
   bool expect_valid_internal_key_;
-  const Compaction* compaction_;
+  Compaction* compaction_;
   const CompactionFilter* compaction_filter_;
   LogBuffer* log_buffer_;
   bool bottommost_level_;

db/compaction_iterator_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/compaction_job.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -51,7 +51,6 @@
 #include "util/iostats_context_imp.h"
 #include "util/log_buffer.h"
 #include "util/logging.h"
-#include "util/sst_file_manager_impl.h"
 #include "util/mutexlock.h"
 #include "util/perf_context_imp.h"
 #include "util/stop_watch.h"
@@ -63,7 +62,7 @@ namespace rocksdb {
 
 // Maintains state for each sub-compaction
 struct CompactionJob::SubcompactionState {
-  const Compaction* compaction;
+  Compaction* compaction;
   std::unique_ptr<CompactionIterator> c_iter;
 
   // The boundaries of the key-range this compaction is interested in. No two
@@ -104,14 +103,6 @@ struct CompactionJob::SubcompactionState {
   uint64_t num_output_records;
   CompactionJobStats compaction_job_stats;
   uint64_t approx_size;
-  // An index that used to speed up Compaction::ShouldStopBefore().
-  size_t grandparent_index = 0;
-  // The number of bytes overlapping between the current output and
-  // grandparent files used in Compaction::ShouldStopBefore().
-  uint64_t overlapped_bytes = 0;
-  // A flag determine whether the key has been seen in
-  // Compaction::ShouldStopBefore()
-  bool seen_key = false;
 
   SubcompactionState(Compaction* c, Slice* _start, Slice* _end,
                      uint64_t size = 0)
@@ -123,10 +114,7 @@ struct CompactionJob::SubcompactionState {
         total_bytes(0),
         num_input_records(0),
         num_output_records(0),
-        approx_size(size),
-        grandparent_index(0),
-        overlapped_bytes(0),
-        seen_key(false) {
+        approx_size(size) {
     assert(compaction != nullptr);
   }
 
@@ -145,9 +133,6 @@ struct CompactionJob::SubcompactionState {
     num_output_records = std::move(o.num_output_records);
     compaction_job_stats = std::move(o.compaction_job_stats);
     approx_size = std::move(o.approx_size);
-    grandparent_index = std::move(o.grandparent_index);
-    overlapped_bytes = std::move(o.overlapped_bytes);
-    seen_key = std::move(o.seen_key);
     return *this;
   }
 
@@ -155,38 +140,6 @@ struct CompactionJob::SubcompactionState {
   SubcompactionState(const SubcompactionState&) = delete;
 
   SubcompactionState& operator=(const SubcompactionState&) = delete;
-
-  // Returns true iff we should stop building the current output
-  // before processing "internal_key".
-  bool ShouldStopBefore(const Slice& internal_key) {
-    const InternalKeyComparator* icmp =
-        &compaction->column_family_data()->internal_comparator();
-    const std::vector<FileMetaData*>& grandparents = compaction->grandparents();
-
-    // Scan to find earliest grandparent file that contains key.
-    while (grandparent_index < grandparents.size() &&
-           icmp->Compare(internal_key,
-                         grandparents[grandparent_index]->largest.Encode()) >
-               0) {
-      if (seen_key) {
-        overlapped_bytes += grandparents[grandparent_index]->fd.GetFileSize();
-      }
-      assert(grandparent_index + 1 >= grandparents.size() ||
-             icmp->Compare(
-                 grandparents[grandparent_index]->largest.Encode(),
-                 grandparents[grandparent_index + 1]->smallest.Encode()) < 0);
-      grandparent_index++;
-    }
-    seen_key = true;
-
-    if (overlapped_bytes > compaction->max_grandparent_overlap_bytes()) {
-      // Too much overlap for current output; start new output
-      overlapped_bytes = 0;
-      return true;
-    }
-
-    return false;
-  }
 };
 
 // Maintains state for the entire compaction
@@ -258,7 +211,6 @@ CompactionJob::CompactionJob(
     const EnvOptions& env_options, VersionSet* versions,
     std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
     Directory* db_directory, Directory* output_directory, Statistics* stats,
-    InstrumentedMutex* db_mutex, Status* db_bg_error,
     std::vector<SequenceNumber> existing_snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
     std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
@@ -278,8 +230,6 @@ CompactionJob::CompactionJob(
       db_directory_(db_directory),
       output_directory_(output_directory),
       stats_(stats),
-      db_mutex_(db_mutex),
-      db_bg_error_(db_bg_error),
       existing_snapshots_(std::move(existing_snapshots)),
       earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
       table_cache_(std::move(table_cache)),
@@ -287,9 +237,7 @@ CompactionJob::CompactionJob(
       paranoid_file_checks_(paranoid_file_checks),
       measure_io_stats_(measure_io_stats) {
   assert(log_buffer_ != nullptr);
-  const auto* cfd = compact_->compaction->column_family_data();
-  ThreadStatusUtil::SetColumnFamily(cfd, cfd->ioptions()->env,
-                                    cfd->options()->enable_thread_tracking);
+  ThreadStatusUtil::SetColumnFamily(compact_->compaction->column_family_data());
   ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION);
   ReportStartedCompaction(compaction);
 }
@@ -301,9 +249,8 @@ CompactionJob::~CompactionJob() {
 
 void CompactionJob::ReportStartedCompaction(
     Compaction* compaction) {
-  const auto* cfd = compact_->compaction->column_family_data();
-  ThreadStatusUtil::SetColumnFamily(cfd, cfd->ioptions()->env,
-                                    cfd->options()->enable_thread_tracking);
+  ThreadStatusUtil::SetColumnFamily(
+      compact_->compaction->column_family_data());
 
   ThreadStatusUtil::SetThreadOperationProperty(
       ThreadStatus::COMPACTION_JOB_ID,
@@ -409,7 +356,7 @@ void CompactionJob::GenSubcompactionBoundaries() {
       size_t num_files = flevel->num_files;
 
       if (num_files == 0) {
-        continue;
+        break;
       }
 
       if (lvl == 0) {
@@ -468,9 +415,12 @@ void CompactionJob::GenSubcompactionBoundaries() {
 
   // Group the ranges into subcompactions
   const double min_file_fill_percent = 4.0 / 5;
-  uint64_t max_output_files = static_cast<uint64_t>(std::ceil(
-      sum / min_file_fill_percent /
-      cfd->GetCurrentMutableCFOptions()->MaxFileSizeForLevel(out_lvl)));
+  uint64_t max_output_files = 
+    static_cast<uint64_t>(
+        std::ceil(
+          sum / min_file_fill_percent /
+          cfd->GetCurrentMutableCFOptions()->MaxFileSizeForLevel(out_lvl))
+          );
   uint64_t subcompactions =
       std::min({static_cast<uint64_t>(ranges.size()),
                 static_cast<uint64_t>(db_options_.max_subcompactions),
@@ -568,17 +518,18 @@ Status CompactionJob::Run() {
   return status;
 }
 
-Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
+Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
+                              InstrumentedMutex* db_mutex) {
   AutoThreadOperationStageUpdater stage_updater(
       ThreadStatus::STAGE_COMPACTION_INSTALL);
-  db_mutex_->AssertHeld();
+  db_mutex->AssertHeld();
   Status status = compact_->status;
   ColumnFamilyData* cfd = compact_->compaction->column_family_data();
   cfd->internal_stats()->AddCompactionStats(
       compact_->compaction->output_level(), compaction_stats_);
 
   if (status.ok()) {
-    status = InstallCompactionResults(mutable_cf_options);
+    status = InstallCompactionResults(mutable_cf_options, db_mutex);
   }
   VersionStorageInfo::LevelSummaryStorage tmp;
   auto vstorage = cfd->current()->storage_info();
@@ -715,7 +666,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
     if (end != nullptr &&
         cfd->user_comparator()->Compare(c_iter->user_key(), *end) >= 0) {
       break;
-    } else if (sub_compact->ShouldStopBefore(key) &&
+    } else if (sub_compact->compaction->ShouldStopBefore(key) &&
                sub_compact->builder != nullptr) {
       status = FinishCompactionOutputFile(input->status(), sub_compact);
       if (!status.ok()) {
@@ -904,33 +855,13 @@ Status CompactionJob::FinishCompactionOutputFile(
           event_logger_, cfd->ioptions()->listeners, meta->fd, info);
     }
   }
-
-  // Report new file to SstFileManagerImpl
-  auto sfm =
-      static_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
-  if (sfm && meta->fd.GetPathId() == 0) {
-    ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
-    auto fn = TableFileName(cfd->ioptions()->db_paths, meta->fd.GetNumber(),
-                            meta->fd.GetPathId());
-    sfm->OnAddFile(fn);
-    if (sfm->IsMaxAllowedSpaceReached()) {
-      InstrumentedMutexLock l(db_mutex_);
-      if (db_bg_error_->ok()) {
-        s = Status::IOError("Max allowed space was reached");
-        *db_bg_error_ = s;
-        TEST_SYNC_POINT(
-            "CompactionJob::FinishCompactionOutputFile:MaxAllowedSpaceReached");
-      }
-    }
-  }
-
   sub_compact->builder.reset();
   return s;
 }
 
 Status CompactionJob::InstallCompactionResults(
-    const MutableCFOptions& mutable_cf_options) {
-  db_mutex_->AssertHeld();
+    const MutableCFOptions& mutable_cf_options, InstrumentedMutex* db_mutex) {
+  db_mutex->AssertHeld();
 
   auto* compaction = compact_->compaction;
   // paranoia: verify that the files that we started with
@@ -965,7 +896,7 @@ Status CompactionJob::InstallCompactionResults(
   }
   return versions_->LogAndApply(compaction->column_family_data(),
                                 mutable_cf_options, compaction->edit(),
-                                db_mutex_, db_directory_);
+                                db_mutex, db_directory_);
 }
 
 void CompactionJob::RecordCompactionIOStats() {
@@ -1019,7 +950,7 @@ Status CompactionJob::OpenCompactionOutputFile(
       cfd->ioptions()->optimize_filters_for_hits && bottommost_level_;
   sub_compact->builder.reset(NewTableBuilder(
       *cfd->ioptions(), cfd->internal_comparator(),
-      cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
+      cfd->int_tbl_prop_collector_factories(), cfd->GetID(),
       sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
       cfd->ioptions()->compression_opts, skip_filters));
   LogFlush(db_options_.info_log);

db/compaction_job.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -56,8 +56,7 @@ class CompactionJob {
                 const EnvOptions& env_options, VersionSet* versions,
                 std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
                 Directory* db_directory, Directory* output_directory,
-                Statistics* stats, InstrumentedMutex* db_mutex,
-                Status* db_bg_error,
+                Statistics* stats,
                 std::vector<SequenceNumber> existing_snapshots,
                 SequenceNumber earliest_write_conflict_snapshot,
                 std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
@@ -78,7 +77,8 @@ class CompactionJob {
   Status Run();
 
   // REQUIRED: mutex held
-  Status Install(const MutableCFOptions& mutable_cf_options);
+  Status Install(const MutableCFOptions& mutable_cf_options,
+                 InstrumentedMutex* db_mutex);
 
  private:
   struct SubcompactionState;
@@ -95,7 +95,8 @@ class CompactionJob {
 
   Status FinishCompactionOutputFile(const Status& input_status,
                                     SubcompactionState* sub_compact);
-  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options);
+  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options,
+                                  InstrumentedMutex* db_mutex);
   void RecordCompactionIOStats();
   Status OpenCompactionOutputFile(SubcompactionState* sub_compact);
   void CleanupCompaction();
@@ -129,8 +130,6 @@ class CompactionJob {
   Directory* db_directory_;
   Directory* output_directory_;
   Statistics* stats_;
-  InstrumentedMutex* db_mutex_;
-  Status* db_bg_error_;
   // If there were two snapshots with seq numbers s1 and
   // s2 and s1 < s2, and if we find two instances of a key k1 then lies
   // entirely within s1 and s2, then the earlier version of k1 can be safely

db/compaction_job_stats_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -660,7 +660,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
   options.max_subcompactions = max_subcompactions_;
   options.bytes_per_sync = 512 * 1024;
 
-  options.report_bg_io_stats = true;
+  options.compaction_measure_io_stats = true;
   for (int test = 0; test < 2; ++test) {
     DestroyAndReopen(options);
     CreateAndReopenWithCF({"pikachu"}, options);

db/compaction_job_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -250,9 +250,9 @@ class CompactionJobTest : public testing::Test {
     EventLogger event_logger(db_options_.info_log.get());
     CompactionJob compaction_job(
         0, &compaction, db_options_, env_options_, versions_.get(),
-        &shutting_down_, &log_buffer, nullptr, nullptr, nullptr, &mutex_,
-        &bg_error_, snapshots, earliest_write_conflict_snapshot, table_cache_,
-        &event_logger, false, false, dbname_, &compaction_job_stats_);
+        &shutting_down_, &log_buffer, nullptr, nullptr, nullptr, snapshots,
+        earliest_write_conflict_snapshot, table_cache_, &event_logger, false,
+        false, dbname_, &compaction_job_stats_);
 
     VerifyInitializationOfCompactionJobStats(compaction_job_stats_);
 
@@ -262,7 +262,8 @@ class CompactionJobTest : public testing::Test {
     s = compaction_job.Run();
     ASSERT_OK(s);
     mutex_.Lock();
-    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions()));
+    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions(),
+                                     &mutex_));
     mutex_.Unlock();
 
     if (expected_results.size() == 0) {
@@ -294,7 +295,6 @@ class CompactionJobTest : public testing::Test {
   ColumnFamilyData* cfd_;
   std::unique_ptr<CompactionFilter> compaction_filter_;
   std::shared_ptr<MergeOperator> merge_op_;
-  Status bg_error_;
 };
 
 TEST_F(CompactionJobTest, Simple) {

db/compaction_picker.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -141,8 +141,7 @@ CompactionPicker::~CompactionPicker() {}
 
 // Delete this compaction from the list of running compactions.
 void CompactionPicker::ReleaseCompactionFiles(Compaction* c, Status status) {
-  if (c->start_level() == 0 ||
-      ioptions_.compaction_style == kCompactionStyleUniversal) {
+  if (c->start_level() == 0) {
     level0_compactions_in_progress_.erase(c);
   }
   if (!status.ok()) {
@@ -613,17 +612,6 @@ Compaction* CompactionPicker::CompactRange(
   if (input_level == 0) {
     level0_compactions_in_progress_.insert(compaction);
   }
-
-  // Creating a compaction influences the compaction score because the score
-  // takes running compactions into account (by skipping files that are already
-  // being compacted). Since we just changed compaction score, we recalculate it
-  // here
-  {  // this piece of code recomputes compaction score
-    CompactionOptionsFIFO dummy_compaction_options_fifo;
-    vstorage->ComputeCompactionScore(mutable_cf_options,
-                                     dummy_compaction_options_fifo);
-  }
-
   return compaction;
 }
 

db/compaction_picker.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -83,8 +83,6 @@ class CompactionPicker {
 #endif  // ROCKSDB_LITE
 
   // Free up the files that participated in a compaction
-  //
-  // Requirement: DB mutex held
   void ReleaseCompactionFiles(Compaction* c, Status status);
 
   // Returns true if any one of the specified files are being compacted

db/compaction_picker_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -195,7 +195,6 @@ TEST_F(CompactionPickerTest, LevelMaxScore) {
   NewVersionStorage(6, kCompactionStyleLevel);
   mutable_cf_options_.target_file_size_base = 10000000;
   mutable_cf_options_.target_file_size_multiplier = 10;
-  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
   Add(0, 1U, "150", "200", 1000000000U);
   // Level 1 score 1.2
   Add(1, 66U, "150", "200", 6000000U);
@@ -488,90 +487,6 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
 }
 #endif  // ROCKSDB_LITE
 
-TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.target_file_size_base = 10000000;
-  mutable_cf_options_.target_file_size_multiplier = 10;
-  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
-  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
-
-  Add(2, 6U, "150", "179", 50000000U);
-  Add(2, 7U, "180", "220", 50000000U);
-  Add(2, 8U, "321", "400", 50000000U);  // File not overlapping
-  Add(2, 9U, "721", "800", 50000000U);
-
-  Add(3, 26U, "150", "170", 260000000U);
-  Add(3, 27U, "171", "179", 260000000U);
-  Add(3, 28U, "191", "220", 260000000U);
-  Add(3, 29U, "221", "300", 260000000U);
-  Add(3, 30U, "750", "900", 260000000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  // Pick file 8 because it overlaps with 0 files on level 3.
-  ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.target_file_size_base = 10000000;
-  mutable_cf_options_.target_file_size_multiplier = 10;
-  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
-  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
-
-  Add(2, 6U, "150", "175",
-      60000000U);  // Overlaps with file 26, 27, total size 521M
-  Add(2, 7U, "176", "200", 60000000U);  // Overlaps with file 27, 28, total size
-                                        // 520M, the smalelst overlapping
-  Add(2, 8U, "201", "300",
-      60000000U);  // Overlaps with file 28, 29, total size 521M
-
-  Add(3, 26U, "100", "110", 261000000U);
-  Add(3, 26U, "150", "170", 261000000U);
-  Add(3, 27U, "171", "179", 260000000U);
-  Add(3, 28U, "191", "220", 260000000U);
-  Add(3, 29U, "221", "300", 261000000U);
-  Add(3, 30U, "321", "400", 261000000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  // Picking file 7 because overlapping ratio is the biggest.
-  ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_bytes_for_level_base = 10000000;
-  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
-  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
-
-  // file 7 and 8 over lap with the same file, but file 8 is smaller so
-  // it will be picked.
-  Add(2, 6U, "150", "167", 60000000U);  // Overlaps with file 26, 27
-  Add(2, 7U, "168", "169", 60000000U);  // Overlaps with file 27
-  Add(2, 8U, "201", "300", 61000000U);  // Overlaps with file 28, but the file
-                                        // itself is larger. Should be picked.
-
-  Add(3, 26U, "160", "165", 260000000U);
-  Add(3, 27U, "166", "170", 260000000U);
-  Add(3, 28U, "180", "400", 260000000U);
-  Add(3, 29U, "401", "500", 260000000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  // Picking file 8 because overlapping ratio is the biggest.
-  ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
-}
-
 // This test exhibits the bug where we don't properly reset parent_index in
 // PickCompaction()
 TEST_F(CompactionPickerTest, ParentIndexResetBug) {

db/comparator_db_test.cc

@@ -1,6 +1,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/convenience.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/corruption_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/cuckoo_table_db_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_bench.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -11,7 +11,14 @@
 #define __STDC_FORMAT_MACROS
 #endif
 
-#ifdef GFLAGS
+#ifndef GFLAGS
+#include <cstdio>
+int main() {
+  fprintf(stderr, "Please install gflags to run rocksdb tools\n");
+  return 1;
+}
+#else
+
 #ifdef NUMA
 #include <numa.h>
 #include <numaif.h>
@@ -36,41 +43,39 @@
 
 #include "db/db_impl.h"
 #include "db/version_set.h"
-#include "hdfs/env_hdfs.h"
-#include "port/port.h"
-#include "port/stack_trace.h"
+#include "rocksdb/options.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
-#include "rocksdb/filter_policy.h"
 #include "rocksdb/memtablerep.h"
-#include "rocksdb/options.h"
-#include "rocksdb/perf_context.h"
-#include "rocksdb/rate_limiter.h"
+#include "rocksdb/write_batch.h"
 #include "rocksdb/slice.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/rate_limiter.h"
 #include "rocksdb/slice_transform.h"
+#include "rocksdb/perf_context.h"
 #include "rocksdb/utilities/flashcache.h"
-#include "rocksdb/utilities/optimistic_transaction_db.h"
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
-#include "rocksdb/write_batch.h"
-#include "util/compression.h"
+#include "rocksdb/utilities/optimistic_transaction_db.h"
+#include "port/port.h"
+#include "port/stack_trace.h"
 #include "util/crc32c.h"
+#include "util/compression.h"
 #include "util/histogram.h"
 #include "util/mutexlock.h"
 #include "util/random.h"
-#include "util/statistics.h"
 #include "util/string_util.h"
+#include "util/statistics.h"
 #include "util/testutil.h"
-#include "util/transaction_test_util.h"
 #include "util/xxhash.h"
+#include "hdfs/env_hdfs.h"
 #include "utilities/merge_operators.h"
 
 #ifdef OS_WIN
 #include <io.h>  // open/close
 #endif
 
-namespace {
 using GFLAGS::ParseCommandLineFlags;
 using GFLAGS::RegisterFlagValidator;
 using GFLAGS::SetUsageMessage;
@@ -340,9 +345,6 @@ DEFINE_int64(cache_size, -1, "Number of bytes to use as a cache of uncompressed"
 DEFINE_bool(cache_index_and_filter_blocks, false,
             "Cache index/filter blocks in block cache.");
 
-DEFINE_bool(pin_l0_filter_and_index_blocks_in_cache, false,
-            "Pin index/filter blocks of L0 files in block cache.");
-
 DEFINE_int32(block_size,
              static_cast<int32_t>(rocksdb::BlockBasedTableOptions().block_size),
              "Number of bytes in a block.");
@@ -516,9 +518,10 @@ DEFINE_uint64(transaction_lock_timeout, 100,
               " milliseconds before failing a transaction waiting on a lock");
 #endif  // ROCKSDB_LITE
 
-DEFINE_bool(report_bg_io_stats, false,
+DEFINE_bool(compaction_measure_io_stats, false,
             "Measure times spents on I/Os while in compactions. ");
 
+namespace {
 enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
   assert(ctype);
 
@@ -538,7 +541,7 @@ enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
     return rocksdb::kZSTDNotFinalCompression;
 
   fprintf(stdout, "Cannot parse compression type '%s'\n", ctype);
-  return rocksdb::kSnappyCompression;  // default value
+  return rocksdb::kSnappyCompression; //default value
 }
 
 std::string ColumnFamilyName(size_t i) {
@@ -550,6 +553,7 @@ std::string ColumnFamilyName(size_t i) {
     return std::string(name);
   }
 }
+}  // namespace
 
 DEFINE_string(compression_type, "snappy",
               "Algorithm to use to compress the database");
@@ -760,6 +764,7 @@ enum RepFactory {
   kCuckoo
 };
 
+namespace {
 enum RepFactory StringToRepFactory(const char* ctype) {
   assert(ctype);
 
@@ -777,6 +782,7 @@ enum RepFactory StringToRepFactory(const char* ctype) {
   fprintf(stdout, "Cannot parse memreptable %s\n", ctype);
   return kSkipList;
 }
+}  // namespace
 
 static enum RepFactory FLAGS_rep_factory;
 DEFINE_string(memtablerep, "skip_list", "");
@@ -828,7 +834,6 @@ static const bool FLAGS_deletepercent_dummy __attribute__((unused)) =
 static const bool FLAGS_table_cache_numshardbits_dummy __attribute__((unused)) =
     RegisterFlagValidator(&FLAGS_table_cache_numshardbits,
                           &ValidateTableCacheNumshardbits);
-}  // namespace
 
 namespace rocksdb {
 
@@ -1209,7 +1214,7 @@ class Stats {
   uint64_t bytes_;
   uint64_t last_op_finish_;
   uint64_t last_report_finish_;
-  std::unordered_map<OperationType, std::shared_ptr<HistogramImpl>,
+  std::unordered_map<OperationType, HistogramImpl,
                      std::hash<unsigned char>> hist_;
   std::string message_;
   bool exclude_from_merge_;
@@ -1246,7 +1251,7 @@ class Stats {
     for (auto it = other.hist_.begin(); it != other.hist_.end(); ++it) {
       auto this_it = hist_.find(it->first);
       if (this_it != hist_.end()) {
-        this_it->second->Merge(*(other.hist_.at(it->first)));
+        this_it->second.Merge(other.hist_.at(it->first));
       } else {
         hist_.insert({ it->first, it->second });
       }
@@ -1320,10 +1325,10 @@ class Stats {
 
       if (hist_.find(op_type) == hist_.end())
       {
-        auto hist_temp = std::make_shared<HistogramImpl>();
-        hist_.insert({op_type, std::move(hist_temp)});
+        HistogramImpl hist_temp;
+        hist_.insert({op_type, hist_temp});
       }
-      hist_[op_type]->Add(micros);
+      hist_[op_type].Add(micros);
 
       if (micros > 20000 && !FLAGS_stats_interval) {
         fprintf(stderr, "long op: %" PRIu64 " micros%30s\r", micros, "");
@@ -1456,7 +1461,7 @@ class Stats {
       for (auto it = hist_.begin(); it != hist_.end(); ++it) {
         fprintf(stdout, "Microseconds per %s:\n%s\n",
                 OperationTypeString[it->first].c_str(),
-                it->second->ToString().c_str());
+                it->second.ToString().c_str());
       }
     }
     if (FLAGS_report_file_operations) {
@@ -2244,7 +2249,7 @@ class Benchmark {
       count++;
       thread->stats.FinishedOps(nullptr, nullptr, 1, kOthers);
     }
-    if (ptr == nullptr) exit(1);  // Disable unused variable warning.
+    if (ptr == nullptr) exit(1); // Disable unused variable warning.
   }
 
   void Compress(ThreadState *thread) {
@@ -2257,7 +2262,6 @@ class Benchmark {
 
     // Compress 1G
     while (ok && bytes < int64_t(1) << 30) {
-      compressed.clear();
       ok = CompressSlice(input, &compressed);
       produced += compressed.size();
       bytes += input.size();
@@ -2514,8 +2518,6 @@ class Benchmark {
       }
       block_based_options.cache_index_and_filter_blocks =
           FLAGS_cache_index_and_filter_blocks;
-      block_based_options.pin_l0_filter_and_index_blocks_in_cache =
-          FLAGS_pin_l0_filter_and_index_blocks_in_cache;
       block_based_options.block_cache = cache_;
       block_based_options.block_cache_compressed = compressed_cache_;
       block_based_options.block_size = FLAGS_block_size;
@@ -2599,7 +2601,7 @@ class Benchmark {
       exit(1);
     }
     options.max_successive_merges = FLAGS_max_successive_merges;
-    options.report_bg_io_stats = FLAGS_report_bg_io_stats;
+    options.compaction_measure_io_stats = FLAGS_compaction_measure_io_stats;
 
     // set universal style compaction configurations, if applicable
     if (FLAGS_universal_size_ratio != 0) {
@@ -3769,22 +3771,18 @@ class Benchmark {
     ReadOptions options(FLAGS_verify_checksum, true);
     Duration duration(FLAGS_duration, readwrites_);
     ReadOptions read_options(FLAGS_verify_checksum, true);
-    uint16_t num_prefix_ranges = static_cast<uint16_t>(FLAGS_transaction_sets);
+    std::string value;
+    DB* db = db_.db;
     uint64_t transactions_done = 0;
+    uint64_t transactions_aborted = 0;
+    Status s;
+    uint64_t num_prefix_ranges = FLAGS_transaction_sets;
 
     if (num_prefix_ranges == 0 || num_prefix_ranges > 9999) {
       fprintf(stderr, "invalid value for transaction_sets\n");
       abort();
     }
 
-    TransactionOptions txn_options;
-    txn_options.lock_timeout = FLAGS_transaction_lock_timeout;
-    txn_options.set_snapshot = FLAGS_transaction_set_snapshot;
-
-    RandomTransactionInserter inserter(&thread->rand, write_options_,
-                                       read_options, FLAGS_num,
-                                       num_prefix_ranges);
-
     if (FLAGS_num_multi_db > 1) {
       fprintf(stderr,
               "Cannot run RandomTransaction benchmark with "
@@ -3793,26 +3791,126 @@ class Benchmark {
     }
 
     while (!duration.Done(1)) {
-      bool success;
+      Transaction* txn = nullptr;
+      WriteBatch* batch = nullptr;
 
-      // RandomTransactionInserter will attempt to insert a key for each
-      // # of FLAGS_transaction_sets
       if (FLAGS_optimistic_transaction_db) {
-        success = inserter.OptimisticTransactionDBInsert(db_.opt_txn_db);
+        txn = db_.opt_txn_db->BeginTransaction(write_options_);
+        assert(txn);
       } else if (FLAGS_transaction_db) {
         TransactionDB* txn_db = reinterpret_cast<TransactionDB*>(db_.db);
-        success = inserter.TransactionDBInsert(txn_db, txn_options);
+
+        TransactionOptions txn_options;
+        txn_options.lock_timeout = FLAGS_transaction_lock_timeout;
+
+        txn = txn_db->BeginTransaction(write_options_, txn_options);
+        assert(txn);
       } else {
-        success = inserter.DBInsert(db_.db);
+        batch = new WriteBatch();
       }
 
-      if (!success) {
-        fprintf(stderr, "Unexpected error: %s\n",
-                inserter.GetLastStatus().ToString().c_str());
-        abort();
+      if (txn && FLAGS_transaction_set_snapshot) {
+        txn->SetSnapshot();
+      }
+
+      // pick a random number to use to increment a key in each set
+      uint64_t incr = (thread->rand.Next() % 100) + 1;
+
+      bool failed = false;
+      // For each set, pick a key at random and increment it
+      for (uint8_t i = 0; i < num_prefix_ranges; i++) {
+        uint64_t int_value;
+        char prefix_buf[5];
+
+        // key format:  [SET#][random#]
+        std::string rand_key = ToString(thread->rand.Next() % FLAGS_num);
+        Slice base_key(rand_key);
+
+        // Pad prefix appropriately so we can iterate over each set
+        snprintf(prefix_buf, sizeof(prefix_buf), "%04d", i + 1);
+        std::string full_key = std::string(prefix_buf) + base_key.ToString();
+        Slice key(full_key);
+
+        if (txn) {
+          s = txn->GetForUpdate(read_options, key, &value);
+        } else {
+          s = db->Get(read_options, key, &value);
+        }
+
+        if (s.ok()) {
+          int_value = std::stoull(value);
+
+          if (int_value == 0 || int_value == ULONG_MAX) {
+            fprintf(stderr, "Get returned unexpected value: %s\n",
+                    value.c_str());
+            abort();
+          }
+        } else if (s.IsNotFound()) {
+          int_value = 0;
+        } else if (!(s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) {
+          fprintf(stderr, "Get returned an unexpected error: %s\n",
+                  s.ToString().c_str());
+          abort();
+        } else {
+          failed = true;
+          break;
+        }
+
+        if (FLAGS_transaction_sleep > 0) {
+          FLAGS_env->SleepForMicroseconds(thread->rand.Next() %
+                                          FLAGS_transaction_sleep);
+        }
+
+        std::string sum = ToString(int_value + incr);
+        if (txn) {
+          s = txn->Put(key, sum);
+          if (!s.ok()) {
+            // Since we did a GetForUpdate, Put should not fail.
+            fprintf(stderr, "Put returned an unexpected error: %s\n",
+                    s.ToString().c_str());
+            abort();
+          }
+        } else {
+          batch->Put(key, sum);
+        }
+      }
+
+      if (txn) {
+        if (failed) {
+          transactions_aborted++;
+          txn->Rollback();
+          s = Status::OK();
+        } else {
+          s = txn->Commit();
+        }
+      } else {
+        s = db->Write(write_options_, batch);
+      }
+
+      if (!s.ok()) {
+        failed = true;
+
+        // Ideally, we'd want to run this stress test with enough concurrency
+        // on a small enough set of keys that we get some failed transactions
+        // due to conflicts.
+        if (FLAGS_optimistic_transaction_db &&
+            (s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) {
+          transactions_aborted++;
+        } else if (FLAGS_transaction_db && s.IsExpired()) {
+          transactions_aborted++;
+        } else {
+          fprintf(stderr, "Unexpected write error: %s\n", s.ToString().c_str());
+          abort();
+        }
+      }
+
+      delete txn;
+      delete batch;
+
+      if (!failed) {
+        thread->stats.FinishedOps(nullptr, db, 1, kOthers);
       }
 
-      thread->stats.FinishedOps(nullptr, db_.db, 1, kOthers);
       transactions_done++;
     }
 
@@ -3820,7 +3918,7 @@ class Benchmark {
     if (FLAGS_optimistic_transaction_db || FLAGS_transaction_db) {
       snprintf(msg, sizeof(msg),
                "( transactions:%" PRIu64 " aborts:%" PRIu64 ")",
-               transactions_done, inserter.GetFailureCount());
+               transactions_done, transactions_aborted);
     } else {
       snprintf(msg, sizeof(msg), "( batches:%" PRIu64 " )", transactions_done);
     }
@@ -3840,15 +3938,50 @@ class Benchmark {
       return;
     }
 
-    Status s =
-        RandomTransactionInserter::Verify(db_.db,
-                            static_cast<uint16_t>(FLAGS_transaction_sets));
+    uint64_t prev_total = 0;
 
-    if (s.ok()) {
-      fprintf(stdout, "RandomTransactionVerify Success.\n");
-    } else {
-      fprintf(stdout, "RandomTransactionVerify FAILED!!\n");
+    // For each set of keys with the same prefix, sum all the values
+    for (uint32_t i = 0; i < FLAGS_transaction_sets; i++) {
+      char prefix_buf[5];
+      snprintf(prefix_buf, sizeof(prefix_buf), "%04u", i + 1);
+      uint64_t total = 0;
+
+      Iterator* iter = db_.db->NewIterator(ReadOptions());
+
+      for (iter->Seek(Slice(prefix_buf, 4)); iter->Valid(); iter->Next()) {
+        Slice key = iter->key();
+
+        // stop when we reach a different prefix
+        if (key.ToString().compare(0, 4, prefix_buf) != 0) {
+          break;
+        }
+
+        Slice value = iter->value();
+        uint64_t int_value = std::stoull(value.ToString());
+        if (int_value == 0 || int_value == ULONG_MAX) {
+          fprintf(stderr, "Iter returned unexpected value: %s\n",
+                  value.ToString().c_str());
+          abort();
+        }
+
+        total += int_value;
+      }
+      delete iter;
+
+      if (i > 0) {
+        if (total != prev_total) {
+          fprintf(stderr,
+                  "RandomTransactionVerify found inconsistent totals. "
+                  "Set[%" PRIu32 "]: %" PRIu64 ", Set[%" PRIu32 "]: %" PRIu64
+                  " \n",
+                  i - 1, prev_total, i, total);
+          abort();
+        }
+      }
+      prev_total = total;
     }
+
+    fprintf(stdout, "RandomTransactionVerify Success!\n");
   }
 #endif  // ROCKSDB_LITE
 
@@ -3938,7 +4071,9 @@ class Benchmark {
   }
 };
 
-int db_bench_tool(int argc, char** argv) {
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
   rocksdb::port::InstallStackTraceHandler();
   SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
                   " [OPTIONS]...");
@@ -4007,5 +4142,5 @@ int db_bench_tool(int argc, char** argv) {
   benchmark.Run();
   return 0;
 }
-}  // namespace rocksdb
-#endif
+
+#endif  // GFLAGS

db/db_block_cache_test.cc

@@ -1,240 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include <cstdlib>
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-
-namespace rocksdb {
-
-static uint64_t TestGetTickerCount(const Options& options,
-                                   Tickers ticker_type) {
-  return options.statistics->getTickerCount(ticker_type);
-}
-
-class DBBlockCacheTest : public DBTestBase {
- private:
-  size_t miss_count_ = 0;
-  size_t hit_count_ = 0;
-  size_t insert_count_ = 0;
-  size_t failure_count_ = 0;
-  size_t compressed_miss_count_ = 0;
-  size_t compressed_hit_count_ = 0;
-  size_t compressed_insert_count_ = 0;
-  size_t compressed_failure_count_ = 0;
-
- public:
-  const size_t kNumBlocks = 10;
-  const size_t kValueSize = 100;
-
-  DBBlockCacheTest() : DBTestBase("/db_block_cache_test") {}
-
-  BlockBasedTableOptions GetTableOptions() {
-    BlockBasedTableOptions table_options;
-    // Set a small enough block size so that each key-value get its own block.
-    table_options.block_size = 1;
-    return table_options;
-  }
-
-  Options GetOptions(const BlockBasedTableOptions& table_options) {
-    Options options = CurrentOptions();
-    options.create_if_missing = true;
-    // options.compression = kNoCompression;
-    options.statistics = rocksdb::CreateDBStatistics();
-    options.table_factory.reset(new BlockBasedTableFactory(table_options));
-    return options;
-  }
-
-  void InitTable(const Options& options) {
-    std::string value(kValueSize, 'a');
-    for (size_t i = 0; i < kNumBlocks; i++) {
-      ASSERT_OK(Put(ToString(i), value.c_str()));
-    }
-  }
-
-  void RecordCacheCounters(const Options& options) {
-    miss_count_ = TestGetTickerCount(options, BLOCK_CACHE_MISS);
-    hit_count_ = TestGetTickerCount(options, BLOCK_CACHE_HIT);
-    insert_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD);
-    failure_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES);
-    compressed_miss_count_ =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS);
-    compressed_hit_count_ =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT);
-    compressed_insert_count_ =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD);
-    compressed_failure_count_ =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD_FAILURES);
-  }
-
-  void CheckCacheCounters(const Options& options, size_t expected_misses,
-                          size_t expected_hits, size_t expected_inserts,
-                          size_t expected_failures) {
-    size_t new_miss_count = TestGetTickerCount(options, BLOCK_CACHE_MISS);
-    size_t new_hit_count = TestGetTickerCount(options, BLOCK_CACHE_HIT);
-    size_t new_insert_count = TestGetTickerCount(options, BLOCK_CACHE_ADD);
-    size_t new_failure_count =
-        TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES);
-    ASSERT_EQ(miss_count_ + expected_misses, new_miss_count);
-    ASSERT_EQ(hit_count_ + expected_hits, new_hit_count);
-    ASSERT_EQ(insert_count_ + expected_inserts, new_insert_count);
-    ASSERT_EQ(failure_count_ + expected_failures, new_failure_count);
-    miss_count_ = new_miss_count;
-    hit_count_ = new_hit_count;
-    insert_count_ = new_insert_count;
-    failure_count_ = new_failure_count;
-  }
-
-  void CheckCompressedCacheCounters(const Options& options,
-                                    size_t expected_misses,
-                                    size_t expected_hits,
-                                    size_t expected_inserts,
-                                    size_t expected_failures) {
-    size_t new_miss_count =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS);
-    size_t new_hit_count =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT);
-    size_t new_insert_count =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD);
-    size_t new_failure_count =
-        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD_FAILURES);
-    ASSERT_EQ(compressed_miss_count_ + expected_misses, new_miss_count);
-    ASSERT_EQ(compressed_hit_count_ + expected_hits, new_hit_count);
-    ASSERT_EQ(compressed_insert_count_ + expected_inserts, new_insert_count);
-    ASSERT_EQ(compressed_failure_count_ + expected_failures, new_failure_count);
-    compressed_miss_count_ = new_miss_count;
-    compressed_hit_count_ = new_hit_count;
-    compressed_insert_count_ = new_insert_count;
-    compressed_failure_count_ = new_failure_count;
-  }
-};
-
-TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
-  ReadOptions read_options;
-  auto table_options = GetTableOptions();
-  auto options = GetOptions(table_options);
-  InitTable(options);
-
-  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
-  table_options.block_cache = cache;
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  Reopen(options);
-  RecordCacheCounters(options);
-
-  std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1);
-  Iterator* iter = nullptr;
-
-  // Load blocks into cache.
-  for (size_t i = 0; i < kNumBlocks - 1; i++) {
-    iter = db_->NewIterator(read_options);
-    iter->Seek(ToString(i));
-    ASSERT_OK(iter->status());
-    CheckCacheCounters(options, 1, 0, 1, 0);
-    iterators[i].reset(iter);
-  }
-  size_t usage = cache->GetUsage();
-  ASSERT_LT(0, usage);
-  cache->SetCapacity(usage);
-  ASSERT_EQ(usage, cache->GetPinnedUsage());
-
-  // Test with strict capacity limit.
-  cache->SetStrictCapacityLimit(true);
-  iter = db_->NewIterator(read_options);
-  iter->Seek(ToString(kNumBlocks - 1));
-  ASSERT_TRUE(iter->status().IsIncomplete());
-  CheckCacheCounters(options, 1, 0, 0, 1);
-  delete iter;
-  iter = nullptr;
-
-  // Release interators and access cache again.
-  for (size_t i = 0; i < kNumBlocks - 1; i++) {
-    iterators[i].reset();
-    CheckCacheCounters(options, 0, 0, 0, 0);
-  }
-  ASSERT_EQ(0, cache->GetPinnedUsage());
-  for (size_t i = 0; i < kNumBlocks - 1; i++) {
-    iter = db_->NewIterator(read_options);
-    iter->Seek(ToString(i));
-    ASSERT_OK(iter->status());
-    CheckCacheCounters(options, 0, 1, 0, 0);
-    iterators[i].reset(iter);
-  }
-}
-
-#ifdef SNAPPY
-TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
-  ReadOptions read_options;
-  auto table_options = GetTableOptions();
-  auto options = GetOptions(table_options);
-  options.compression = CompressionType::kSnappyCompression;
-  InitTable(options);
-
-  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
-  std::shared_ptr<Cache> compressed_cache = NewLRUCache(0, 0, false);
-  table_options.block_cache = cache;
-  table_options.block_cache_compressed = compressed_cache;
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  Reopen(options);
-  RecordCacheCounters(options);
-
-  std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1);
-  Iterator* iter = nullptr;
-
-  // Load blocks into cache.
-  for (size_t i = 0; i < kNumBlocks - 1; i++) {
-    iter = db_->NewIterator(read_options);
-    iter->Seek(ToString(i));
-    ASSERT_OK(iter->status());
-    CheckCacheCounters(options, 1, 0, 1, 0);
-    CheckCompressedCacheCounters(options, 1, 0, 1, 0);
-    iterators[i].reset(iter);
-  }
-  size_t usage = cache->GetUsage();
-  ASSERT_LT(0, usage);
-  ASSERT_EQ(usage, cache->GetPinnedUsage());
-  size_t compressed_usage = compressed_cache->GetUsage();
-  ASSERT_LT(0, compressed_usage);
-  // Compressed block cache cannot be pinned.
-  ASSERT_EQ(0, compressed_cache->GetPinnedUsage());
-
-  // Set strict capacity limit flag. Now block will only load into compressed
-  // block cache.
-  cache->SetCapacity(usage);
-  cache->SetStrictCapacityLimit(true);
-  ASSERT_EQ(usage, cache->GetPinnedUsage());
-  // compressed_cache->SetCapacity(compressed_usage);
-  compressed_cache->SetCapacity(0);
-  // compressed_cache->SetStrictCapacityLimit(true);
-  iter = db_->NewIterator(read_options);
-  iter->Seek(ToString(kNumBlocks - 1));
-  ASSERT_TRUE(iter->status().IsIncomplete());
-  CheckCacheCounters(options, 1, 0, 0, 1);
-  CheckCompressedCacheCounters(options, 1, 0, 1, 0);
-  delete iter;
-  iter = nullptr;
-
-  // Clear strict capacity limit flag. This time we shall hit compressed block
-  // cache.
-  cache->SetStrictCapacityLimit(false);
-  iter = db_->NewIterator(read_options);
-  iter->Seek(ToString(kNumBlocks - 1));
-  ASSERT_OK(iter->status());
-  CheckCacheCounters(options, 1, 0, 1, 0);
-  CheckCompressedCacheCounters(options, 0, 1, 0, 0);
-  delete iter;
-  iter = nullptr;
-}
-#endif
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  rocksdb::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/db_compaction_filter_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -368,10 +368,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
 // entries are deleted. The compaction should create bunch of 'DeleteFile'
 // entries in VersionEdit, but none of the 'AddFile's.
 TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
   options.disable_auto_compactions = true;
   options.create_if_missing = true;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   // put some data
@@ -399,10 +400,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
 
 TEST_F(DBTestCompactionFilter, CompactionFilterWithValueChange) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.num_levels = 3;
     options.compaction_filter_factory =
       std::make_shared<ChangeFilterFactory>();
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Write 100K+1 keys, these are written to a few files
@@ -465,7 +467,8 @@ TEST_F(DBTestCompactionFilter, CompactionFilterWithMergeOperator) {
   PutFixed64(&three, 3);
   PutFixed64(&four, 4);
 
-  Options options = CurrentOptions();
+  Options options;
+  options = CurrentOptions(options);
   options.create_if_missing = true;
   options.merge_operator = MergeOperators::CreateUInt64AddOperator();
   options.num_levels = 3;
@@ -618,10 +621,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextCfId) {
 // Compaction filters should only be applied to records that are newer than the
 // latest snapshot. This test inserts records and applies a delete filter.
 TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
   options.disable_auto_compactions = true;
   options.create_if_missing = true;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   // Put some data.
@@ -655,10 +659,11 @@ TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
 // records newer than the snapshot will also be processed
 TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
   std::string five = ToString(5);
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_filter_factory = std::make_shared<DeleteISFilterFactory>();
   options.disable_auto_compactions = true;
   options.create_if_missing = true;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   // Put some data.

db/db_compaction_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -288,7 +288,8 @@ TEST_F(DBCompactionTest, SkipStatsUpdateTest) {
 }
 
 TEST_F(DBCompactionTest, TestTableReaderForCompaction) {
-  Options options = CurrentOptions();
+  Options options;
+  options = CurrentOptions(options);
   options.env = env_;
   options.new_table_reader_for_compaction_inputs = true;
   options.max_open_files = 100;
@@ -492,13 +493,14 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
 TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
   const int kNumKeysPerFile = 100;
 
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 110 << 10;  // 110KB
   options.arena_block_size = 4 << 10;
   options.num_levels = 3;
   options.level0_file_num_compaction_trigger = 3;
   options.max_subcompactions = max_subcompactions_;
   options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+  options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
   Random rnd(301);
@@ -531,107 +533,11 @@ TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
   ASSERT_EQ(NumTableFilesAtLevel(1, 1), 1);
 }
 
-TEST_F(DBCompactionTest, BGCompactionsAllowed) {
-  // Create several column families. Make compaction triggers in all of them
-  // and see number of compactions scheduled to be less than allowed.
-  const int kNumKeysPerFile = 100;
-
-  Options options = CurrentOptions();
-  options.write_buffer_size = 110 << 10;  // 110KB
-  options.arena_block_size = 4 << 10;
-  options.num_levels = 3;
-  // Should speed up compaction when there are 4 files.
-  options.level0_file_num_compaction_trigger = 2;
-  options.level0_slowdown_writes_trigger = 20;
-  options.soft_pending_compaction_bytes_limit = 1 << 30;  // Infinitely large
-  options.base_background_compactions = 1;
-  options.max_background_compactions = 3;
-  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
-
-  // Block all threads in thread pool.
-  const size_t kTotalTasks = 4;
-  env_->SetBackgroundThreads(4, Env::LOW);
-  test::SleepingBackgroundTask sleeping_tasks[kTotalTasks];
-  for (size_t i = 0; i < kTotalTasks; i++) {
-    env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
-                   &sleeping_tasks[i], Env::Priority::LOW);
-    sleeping_tasks[i].WaitUntilSleeping();
-  }
-
-  CreateAndReopenWithCF({"one", "two", "three"}, options);
-
-  Random rnd(301);
-  for (int cf = 0; cf < 4; cf++) {
-    for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
-      for (int i = 0; i < kNumKeysPerFile; i++) {
-        ASSERT_OK(Put(cf, Key(i), ""));
-      }
-      // put extra key to trigger flush
-      ASSERT_OK(Put(cf, "", ""));
-      dbfull()->TEST_WaitForFlushMemTable(handles_[cf]);
-      ASSERT_EQ(NumTableFilesAtLevel(0, cf), num + 1);
-    }
-  }
-
-  // Now all column families qualify compaction but only one should be
-  // scheduled, because no column family hits speed up condition.
-  ASSERT_EQ(1, env_->GetThreadPoolQueueLen(Env::Priority::LOW));
-
-  // Create two more files for one column family, which triggers speed up
-  // condition, three compactions will be scheduled.
-  for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
-    for (int i = 0; i < kNumKeysPerFile; i++) {
-      ASSERT_OK(Put(2, Key(i), ""));
-    }
-    // put extra key to trigger flush
-    ASSERT_OK(Put(2, "", ""));
-    dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
-    ASSERT_EQ(options.level0_file_num_compaction_trigger + num + 1,
-              NumTableFilesAtLevel(0, 2));
-  }
-  ASSERT_EQ(3, env_->GetThreadPoolQueueLen(Env::Priority::LOW));
-
-  // Unblock all threads to unblock all compactions.
-  for (size_t i = 0; i < kTotalTasks; i++) {
-    sleeping_tasks[i].WakeUp();
-    sleeping_tasks[i].WaitUntilDone();
-  }
-  dbfull()->TEST_WaitForCompact();
-
-  // Verify number of compactions allowed will come back to 1.
-
-  for (size_t i = 0; i < kTotalTasks; i++) {
-    sleeping_tasks[i].Reset();
-    env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
-                   &sleeping_tasks[i], Env::Priority::LOW);
-    sleeping_tasks[i].WaitUntilSleeping();
-  }
-  for (int cf = 0; cf < 4; cf++) {
-    for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
-      for (int i = 0; i < kNumKeysPerFile; i++) {
-        ASSERT_OK(Put(cf, Key(i), ""));
-      }
-      // put extra key to trigger flush
-      ASSERT_OK(Put(cf, "", ""));
-      dbfull()->TEST_WaitForFlushMemTable(handles_[cf]);
-      ASSERT_EQ(NumTableFilesAtLevel(0, cf), num + 1);
-    }
-  }
-
-  // Now all column families qualify compaction but only one should be
-  // scheduled, because no column family hits speed up condition.
-  ASSERT_EQ(1, env_->GetThreadPoolQueueLen(Env::Priority::LOW));
-
-  for (size_t i = 0; i < kTotalTasks; i++) {
-    sleeping_tasks[i].WakeUp();
-    sleeping_tasks[i].WaitUntilDone();
-  }
-}
-
 TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 100000000;        // Large write buffer
   options.max_subcompactions = max_subcompactions_;
+  options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
   Random rnd(301);
@@ -658,8 +564,9 @@ TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
 
 TEST_F(DBCompactionTest, MinorCompactionsHappen) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.write_buffer_size = 10000;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     const int N = 500;
@@ -684,7 +591,7 @@ TEST_F(DBCompactionTest, MinorCompactionsHappen) {
 }
 
 TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleLevel;
   options.level0_file_num_compaction_trigger = 3;
 
@@ -698,6 +605,7 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
   const size_t key_len =
     static_cast<size_t>(compact_opt.output_file_size_limit) / 5;
 
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   std::vector<const Snapshot*> snaps;
@@ -741,8 +649,10 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
 // if the database is shutdown during the memtable compaction.
 TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.env = env_;
+    options.write_buffer_size = 1000000;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Trigger a long memtable compaction and reopen the database during it
@@ -766,9 +676,10 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveOneFile) {
       [&](void* arg) { trivial_move++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 100000000;
   options.max_subcompactions = max_subcompactions_;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   int32_t num_keys = 80;
@@ -1360,9 +1271,10 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveToLastLevelWithFiles) {
       [&](void* arg) { non_trivial_move++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 100000000;
   options.max_subcompactions = max_subcompactions_;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   int32_t value_size = 10 * 1024;  // 10 KB
@@ -1651,7 +1563,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
   int max_key_universal_insert = 600;
 
   // Stage 1: generate a db with level compaction
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 110 << 10;  // 110KB
   options.arena_block_size = 4 << 10;
   options.num_levels = 4;
@@ -1661,6 +1573,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
   options.target_file_size_base = 200 << 10;  // 200KB
   options.target_file_size_multiplier = 1;
   options.max_subcompactions = max_subcompactions_;
+  options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
   for (int i = 0; i <= max_key_level_insert; i++) {
@@ -1985,7 +1898,7 @@ TEST_P(DBCompactionTestWithParam, DISABLED_CompactFilesOnLevelCompaction) {
     std::set<std::string> overlapping_file_names;
     std::vector<std::string> compaction_input_file_names;
     for (int f = 0; f < file_picked; ++f) {
-      int level = 0;
+      int level;
       auto file_meta = PickFileRandomly(cf_meta, &rnd, &level);
       compaction_input_file_names.push_back(file_meta->name);
       GetOverlappingFileNumbersForLevelCompaction(
@@ -2285,25 +2198,6 @@ TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) {
   Destroy(options);
 }
 
-TEST_F(DBCompactionTest, SanitizeCompactionOptionsTest) {
-  Options options = CurrentOptions();
-  options.max_background_compactions = 5;
-  options.soft_pending_compaction_bytes_limit = 0;
-  options.hard_pending_compaction_bytes_limit = 100;
-  options.create_if_missing = true;
-  DestroyAndReopen(options);
-  ASSERT_EQ(5, db_->GetOptions().base_background_compactions);
-  ASSERT_EQ(100, db_->GetOptions().soft_pending_compaction_bytes_limit);
-
-  options.base_background_compactions = 4;
-  options.max_background_compactions = 3;
-  options.soft_pending_compaction_bytes_limit = 200;
-  options.hard_pending_compaction_bytes_limit = 150;
-  DestroyAndReopen(options);
-  ASSERT_EQ(3, db_->GetOptions().base_background_compactions);
-  ASSERT_EQ(150, db_->GetOptions().soft_pending_compaction_bytes_limit);
-}
-
 // This tests for a bug that could cause two level0 compactions running
 // concurrently
 // TODO(aekmekji): Make sure that the reason this fails when run with
@@ -2375,9 +2269,10 @@ TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
       [&](void* arg) { non_trivial_move++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 100000000;
   options.max_subcompactions = max_subcompactions_;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   int32_t value_size = 10 * 1024;  // 10 KB
@@ -2467,14 +2362,14 @@ class CompactionPriTest : public DBTestBase,
 };
 
 TEST_P(CompactionPriTest, Test) {
-  Options options = CurrentOptions();
+  Options options;
   options.write_buffer_size = 16 * 1024;
   options.compaction_pri = static_cast<CompactionPri>(compaction_pri_);
   options.hard_pending_compaction_bytes_limit = 256 * 1024;
   options.max_bytes_for_level_base = 64 * 1024;
   options.max_bytes_for_level_multiplier = 4;
   options.compression = kNoCompression;
-
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   Random rnd(301);
@@ -2495,12 +2390,8 @@ TEST_P(CompactionPriTest, Test) {
   }
 }
 
-INSTANTIATE_TEST_CASE_P(
-    CompactionPriTest, CompactionPriTest,
-    ::testing::Values(CompactionPri::kByCompensatedSize,
-                      CompactionPri::kOldestLargestSeqFirst,
-                      CompactionPri::kOldestSmallestSeqFirst,
-                      CompactionPri::kMinOverlappingRatio));
+INSTANTIATE_TEST_CASE_P(CompactionPriTest, CompactionPriTest,
+                        ::testing::Values(0, 1, 2));
 
 #endif // !defined(ROCKSDB_LITE)
 }  // namespace rocksdb

db/db_dynamic_level_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_filesnapshot.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_impl.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -347,10 +347,6 @@ class DBImpl : public DB {
 
 #endif  // NDEBUG
 
-  // Return maximum background compaction alowed to be scheduled based on
-  // compaction status.
-  int BGCompactionsAllowed() const;
-
   // Returns the list of live files in 'live' and the list
   // of all files in the filesystem in 'candidate_files'.
   // If force == false and the last call was less than
@@ -494,11 +490,10 @@ class DBImpl : public DB {
 
   // Background process needs to call
   //     auto x = CaptureCurrentFileNumberInPendingOutputs()
-  //     auto file_num = versions_->NewFileNumber();
   //     <do something>
   //     ReleaseFileNumberFromPendingOutputs(x)
-  // This will protect any file with number `file_num` or greater from being
-  // deleted while <do something> is running.
+  // This will protect any temporary files created while <do something> is
+  // executing from being deleted.
   // -----------
   // This function will capture current file number and append it to
   // pending_outputs_. This will prevent any background process to delete any
@@ -523,7 +518,7 @@ class DBImpl : public DB {
                          SequenceNumber* max_sequence, bool read_only);
 
   // The following two methods are used to flush a memtable to
-  // storage. The first one is used at database RecoveryTime (when the
+  // storage. The first one is used atdatabase RecoveryTime (when the
   // database is opened) and is heavyweight because it holds the mutex
   // for the entire period. The second method WriteLevel0Table supports
   // concurrent flush memtables to storage.
@@ -823,10 +818,7 @@ class DBImpl : public DB {
   // they're unique
   std::atomic<int> next_job_id_;
 
-  // A flag indicating whether the current rocksdb database has any
-  // data that is not yet persisted into either WAL or SST file.
-  // Used when disableWAL is true.
-  bool has_unpersisted_data_;
+  bool flush_on_destroy_; // Used when disableWAL is true.
 
   static const int KEEP_LOG_FILE_NUM = 1000;
   // MSVC version 1800 still does not have constexpr for ::max()
@@ -844,12 +836,9 @@ class DBImpl : public DB {
   // Unified interface for logging events
   EventLogger event_logger_;
 
-  // A value of > 0 temporarily disables scheduling of background work
+  // A value of >0 temporarily disables scheduling of background work
   int bg_work_paused_;
 
-  // A value of > 0 temporarily disables scheduling of background compaction
-  int bg_compaction_paused_;
-
   // Guard against multiple concurrent refitting
   bool refitting_level_;
 
@@ -900,8 +889,9 @@ class DBImpl : public DB {
                  bool* value_found = nullptr);
 
   bool GetIntPropertyInternal(ColumnFamilyData* cfd,
-                              const DBPropertyInfo& property_info,
-                              bool is_locked, uint64_t* value);
+                              DBPropertyType property_type,
+                              bool need_out_of_mutex, bool is_locked,
+                              uint64_t* value);
 
   bool HasPendingManualCompaction();
   bool HasExclusiveManualCompaction();

db/db_impl_debug.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_impl_experimental.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_impl_readonly.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -54,11 +54,10 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
   auto db_iter = NewArenaWrappedDbIterator(
       env_, *cfd->ioptions(), cfd->user_comparator(),
       (read_options.snapshot != nullptr
-           ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
-                 ->number_
+           ? reinterpret_cast<const SnapshotImpl*>(
+                read_options.snapshot)->number_
            : latest_snapshot),
-      super_version->mutable_cf_options.max_sequential_skip_in_iterations,
-      super_version->version_number);
+      super_version->mutable_cf_options.max_sequential_skip_in_iterations);
   auto internal_iter = NewInternalIterator(
       read_options, cfd, super_version, db_iter->GetArena());
   db_iter->SetIterUnderDBIter(internal_iter);
@@ -82,11 +81,10 @@ Status DBImplReadOnly::NewIterators(
     auto* db_iter = NewArenaWrappedDbIterator(
         env_, *cfd->ioptions(), cfd->user_comparator(),
         (read_options.snapshot != nullptr
-             ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
-                   ->number_
-             : latest_snapshot),
-        sv->mutable_cf_options.max_sequential_skip_in_iterations,
-        sv->version_number);
+            ? reinterpret_cast<const SnapshotImpl*>(
+                  read_options.snapshot)->number_
+            : latest_snapshot),
+        sv->mutable_cf_options.max_sequential_skip_in_iterations);
     auto* internal_iter = NewInternalIterator(
         read_options, cfd, sv, db_iter->GetArena());
     db_iter->SetIterUnderDBIter(internal_iter);

db/db_impl_readonly.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_inplace_update_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -18,11 +18,12 @@ class DBTestInPlaceUpdate : public DBTestBase {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
     options.env = env_;
     options.write_buffer_size = 100000;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Update key with values of smaller size
@@ -40,11 +41,12 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
     options.env = env_;
     options.write_buffer_size = 100000;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Update key with values of larger size
@@ -62,7 +64,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
 
@@ -70,6 +72,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
     options.write_buffer_size = 100000;
     options.inplace_callback =
       rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerSize;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Update key with values of smaller size
@@ -89,7 +92,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
 
@@ -97,6 +100,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
     options.write_buffer_size = 100000;
     options.inplace_callback =
       rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerVarintSize;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Update key with values of smaller varint size
@@ -116,7 +120,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
 
@@ -124,6 +128,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
     options.write_buffer_size = 100000;
     options.inplace_callback =
       rocksdb::DBTestInPlaceUpdate::updateInPlaceLargerSize;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Update key with values of larger size
@@ -141,7 +146,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
 
 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
   do {
-    Options options = CurrentOptions();
+    Options options;
     options.create_if_missing = true;
     options.inplace_update_support = true;
 
@@ -149,6 +154,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
     options.write_buffer_size = 100000;
     options.inplace_callback =
       rocksdb::DBTestInPlaceUpdate::updateInPlaceNoAction;
+    options = CurrentOptions(options);
     CreateAndReopenWithCF({"pikachu"}, options);
 
     // Callback function requests no actions from db

db/db_iter.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -13,20 +13,18 @@
 #include <string>
 #include <limits>
 
-#include "db/dbformat.h"
 #include "db/filename.h"
-#include "db/merge_context.h"
+#include "db/dbformat.h"
 #include "port/port.h"
 #include "rocksdb/env.h"
+#include "rocksdb/options.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/merge_operator.h"
-#include "rocksdb/options.h"
 #include "table/internal_iterator.h"
 #include "util/arena.h"
 #include "util/logging.h"
 #include "util/mutexlock.h"
 #include "util/perf_context_imp.h"
-#include "util/string_util.h"
 
 namespace rocksdb {
 
@@ -61,47 +59,9 @@ class DBIter: public Iterator {
     kReverse
   };
 
-  // LocalStatistics contain Statistics counters that will be aggregated per
-  // each iterator instance and then will be sent to the global statistics when
-  // the iterator is destroyed.
-  //
-  // The purpose of this approach is to avoid perf regression happening
-  // when multiple threads bump the atomic counters from a DBIter::Next().
-  struct LocalStatistics {
-    explicit LocalStatistics() { ResetCounters(); }
-
-    void ResetCounters() {
-      next_count_ = 0;
-      next_found_count_ = 0;
-      prev_count_ = 0;
-      prev_found_count_ = 0;
-      bytes_read_ = 0;
-    }
-
-    void BumpGlobalStatistics(Statistics* global_statistics) {
-      RecordTick(global_statistics, NUMBER_DB_NEXT, next_count_);
-      RecordTick(global_statistics, NUMBER_DB_NEXT_FOUND, next_found_count_);
-      RecordTick(global_statistics, NUMBER_DB_PREV, prev_count_);
-      RecordTick(global_statistics, NUMBER_DB_PREV_FOUND, prev_found_count_);
-      RecordTick(global_statistics, ITER_BYTES_READ, bytes_read_);
-      ResetCounters();
-    }
-
-    // Map to Tickers::NUMBER_DB_NEXT
-    uint64_t next_count_;
-    // Map to Tickers::NUMBER_DB_NEXT_FOUND
-    uint64_t next_found_count_;
-    // Map to Tickers::NUMBER_DB_PREV
-    uint64_t prev_count_;
-    // Map to Tickers::NUMBER_DB_PREV_FOUND
-    uint64_t prev_found_count_;
-    // Map to Tickers::ITER_BYTES_READ
-    uint64_t bytes_read_;
-  };
-
   DBIter(Env* env, const ImmutableCFOptions& ioptions, const Comparator* cmp,
          InternalIterator* iter, SequenceNumber s, bool arena_mode,
-         uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+         uint64_t max_sequential_skip_in_iterations,
          const Slice* iterate_upper_bound = nullptr,
          bool prefix_same_as_start = false)
       : arena_mode_(arena_mode),
@@ -115,7 +75,6 @@ class DBIter: public Iterator {
         valid_(false),
         current_entry_is_merged_(false),
         statistics_(ioptions.statistics),
-        version_number_(version_number),
         iterate_upper_bound_(iterate_upper_bound),
         prefix_same_as_start_(prefix_same_as_start),
         iter_pinned_(false) {
@@ -125,7 +84,6 @@ class DBIter: public Iterator {
   }
   virtual ~DBIter() {
     RecordTick(statistics_, NO_ITERATORS, -1);
-    local_stats_.BumpGlobalStatistics(statistics_);
     if (!arena_mode_) {
       delete iter_;
     } else {
@@ -178,27 +136,9 @@ class DBIter: public Iterator {
     }
     return s;
   }
-
-  virtual Status GetProperty(std::string prop_name,
-                             std::string* prop) override {
-    if (prop == nullptr) {
-      return Status::InvalidArgument("prop is nullptr");
-    }
-    if (prop_name == "rocksdb.iterator.super-version-number") {
-      // First try to pass the value returned from inner iterator.
-      if (!iter_->GetProperty(prop_name, prop).ok()) {
-        *prop = ToString(version_number_);
-      }
-      return Status::OK();
-    } else if (prop_name == "rocksdb.iterator.is-key-pinned") {
-      if (valid_) {
-        *prop = (iter_pinned_ && saved_key_.IsKeyPinned()) ? "1" : "0";
-      } else {
-        *prop = "Iterator is not valid.";
-      }
-      return Status::OK();
-    }
-    return Status::InvalidArgument("Undentified property.");
+  virtual bool IsKeyPinned() const override {
+    assert(valid_);
+    return iter_pinned_ && saved_key_.IsKeyPinned();
   }
 
   virtual void Next() override;
@@ -246,14 +186,12 @@ class DBIter: public Iterator {
   bool current_entry_is_merged_;
   Statistics* statistics_;
   uint64_t max_skip_;
-  uint64_t version_number_;
   const Slice* iterate_upper_bound_;
   IterKey prefix_start_;
   bool prefix_same_as_start_;
   bool iter_pinned_;
   // List of operands for merge operator.
-  MergeContext merge_context_;
-  LocalStatistics local_stats_;
+  std::deque<std::string> merge_operands_;
 
   // No copying allowed
   DBIter(const DBIter&);
@@ -291,9 +229,6 @@ void DBIter::Next() {
     PERF_COUNTER_ADD(internal_key_skipped_count, 1);
   }
 
-  if (statistics_ != nullptr) {
-    local_stats_.next_count_++;
-  }
   // Now we point to the next internal position, for both of merge and
   // not merge cases.
   if (!iter_->Valid()) {
@@ -301,15 +236,18 @@ void DBIter::Next() {
     return;
   }
   FindNextUserEntry(true /* skipping the current user key */);
+  if (statistics_ != nullptr) {
+    RecordTick(statistics_, NUMBER_DB_NEXT);
+    if (valid_) {
+      RecordTick(statistics_, NUMBER_DB_NEXT_FOUND);
+      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
+    }
+  }
   if (valid_ && prefix_extractor_ && prefix_same_as_start_ &&
       prefix_extractor_->Transform(saved_key_.GetKey())
               .compare(prefix_start_.GetKey()) != 0) {
     valid_ = false;
   }
-  if (statistics_ != nullptr && valid_) {
-    local_stats_.next_found_count_++;
-    local_stats_.bytes_read_ += (key().size() + value().size());
-  }
 }
 
 // PRE: saved_key_ has the current user key if skipping
@@ -337,7 +275,7 @@ void DBIter::FindNextUserEntryInternal(bool skipping) {
 
     if (ParseKey(&ikey)) {
       if (iterate_upper_bound_ != nullptr &&
-          user_comparator_->Compare(ikey.user_key, *iterate_upper_bound_) >= 0) {
+          ikey.user_key.compare(*iterate_upper_bound_) >= 0) {
         break;
       }
 
@@ -412,9 +350,9 @@ void DBIter::MergeValuesNewToOld() {
     return;
   }
 
-  merge_context_.Clear();
   // Start the merge process by pushing the first operand
-  merge_context_.PushOperand(iter_->value());
+  std::deque<std::string> operands;
+  operands.push_front(iter_->value().ToString());
 
   ParsedInternalKey ikey;
   for (iter_->Next(); iter_->Valid(); iter_->Next()) {
@@ -439,8 +377,7 @@ void DBIter::MergeValuesNewToOld() {
       {
         StopWatchNano timer(env_, statistics_ != nullptr);
         PERF_TIMER_GUARD(merge_operator_time_nanos);
-        user_merge_operator_->FullMerge(ikey.user_key, &val,
-                                        merge_context_.GetOperands(),
+        user_merge_operator_->FullMerge(ikey.user_key, &val, operands,
                                         &saved_value_, logger_);
         RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                    timer.ElapsedNanos());
@@ -452,7 +389,7 @@ void DBIter::MergeValuesNewToOld() {
       // hit a merge, add the value as an operand and run associative merge.
       // when complete, add result to operands and continue.
       const Slice& val = iter_->value();
-      merge_context_.PushOperand(val);
+      operands.push_front(val.ToString());
     } else {
       assert(false);
     }
@@ -465,9 +402,8 @@ void DBIter::MergeValuesNewToOld() {
     // a deletion marker.
     // feed null as the existing value to the merge operator, such that
     // client can differentiate this scenario and do things accordingly.
-    user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
-                                    merge_context_.GetOperands(), &saved_value_,
-                                    logger_);
+    user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
+                                    &saved_value_, logger_);
     RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
   }
 }
@@ -479,10 +415,10 @@ void DBIter::Prev() {
   }
   PrevInternal();
   if (statistics_ != nullptr) {
-    local_stats_.prev_count_++;
+    RecordTick(statistics_, NUMBER_DB_PREV);
     if (valid_) {
-      local_stats_.prev_found_count_++;
-      local_stats_.bytes_read_ += (key().size() + value().size());
+      RecordTick(statistics_, NUMBER_DB_PREV_FOUND);
+      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
     }
   }
   if (valid_ && prefix_extractor_ && prefix_same_as_start_ &&
@@ -559,7 +495,7 @@ void DBIter::PrevInternal() {
 // saved_value_
 bool DBIter::FindValueForCurrentKey() {
   assert(iter_->Valid());
-  merge_context_.Clear();
+  merge_operands_.clear();
   // last entry before merge (could be kTypeDeletion, kTypeSingleDeletion or
   // kTypeValue)
   ValueType last_not_merge_type = kTypeDeletion;
@@ -579,19 +515,19 @@ bool DBIter::FindValueForCurrentKey() {
     last_key_entry_type = ikey.type;
     switch (last_key_entry_type) {
       case kTypeValue:
-        merge_context_.Clear();
+        merge_operands_.clear();
         saved_value_ = iter_->value().ToString();
         last_not_merge_type = kTypeValue;
         break;
       case kTypeDeletion:
       case kTypeSingleDeletion:
-        merge_context_.Clear();
+        merge_operands_.clear();
         last_not_merge_type = last_key_entry_type;
         PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
         break;
       case kTypeMerge:
         assert(user_merge_operator_ != nullptr);
-        merge_context_.PushOperandBack(iter_->value());
+        merge_operands_.push_back(iter_->value().ToString());
         break;
       default:
         assert(false);
@@ -614,8 +550,8 @@ bool DBIter::FindValueForCurrentKey() {
         StopWatchNano timer(env_, statistics_ != nullptr);
         PERF_TIMER_GUARD(merge_operator_time_nanos);
         user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
-                                        merge_context_.GetOperands(),
-                                        &saved_value_, logger_);
+                                        merge_operands_, &saved_value_,
+                                        logger_);
         RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                    timer.ElapsedNanos());
       } else {
@@ -626,8 +562,8 @@ bool DBIter::FindValueForCurrentKey() {
           StopWatchNano timer(env_, statistics_ != nullptr);
           PERF_TIMER_GUARD(merge_operator_time_nanos);
           user_merge_operator_->FullMerge(saved_key_.GetKey(), &temp_slice,
-                                          merge_context_.GetOperands(),
-                                          &saved_value_, logger_);
+                                          merge_operands_, &saved_value_,
+                                          logger_);
           RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                      timer.ElapsedNanos());
         }
@@ -670,11 +606,11 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
 
   // kTypeMerge. We need to collect all kTypeMerge values and save them
   // in operands
-  merge_context_.Clear();
+  std::deque<std::string> operands;
   while (iter_->Valid() &&
          user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) &&
          ikey.type == kTypeMerge) {
-    merge_context_.PushOperand(iter_->value());
+    operands.push_front(iter_->value().ToString());
     iter_->Next();
     FindParseableKey(&ikey, kForward);
   }
@@ -685,8 +621,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
     {
       StopWatchNano timer(env_, statistics_ != nullptr);
       PERF_TIMER_GUARD(merge_operator_time_nanos);
-      user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
-                                      merge_context_.GetOperands(),
+      user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
                                       &saved_value_, logger_);
       RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
     }
@@ -704,9 +639,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   {
     StopWatchNano timer(env_, statistics_ != nullptr);
     PERF_TIMER_GUARD(merge_operator_time_nanos);
-    user_merge_operator_->FullMerge(saved_key_.GetKey(), &val,
-                                    merge_context_.GetOperands(), &saved_value_,
-                                    logger_);
+    user_merge_operator_->FullMerge(saved_key_.GetKey(), &val, operands,
+                                    &saved_value_, logger_);
     RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
   }
   valid_ = true;
@@ -884,13 +818,12 @@ Iterator* NewDBIterator(Env* env, const ImmutableCFOptions& ioptions,
                         InternalIterator* internal_iter,
                         const SequenceNumber& sequence,
                         uint64_t max_sequential_skip_in_iterations,
-                        uint64_t version_number,
                         const Slice* iterate_upper_bound,
                         bool prefix_same_as_start, bool pin_data) {
   DBIter* db_iter =
       new DBIter(env, ioptions, user_key_comparator, internal_iter, sequence,
-                 false, max_sequential_skip_in_iterations, version_number,
-                 iterate_upper_bound, prefix_same_as_start);
+                 false, max_sequential_skip_in_iterations, iterate_upper_bound,
+                 prefix_same_as_start);
   if (pin_data) {
     db_iter->PinData();
   }
@@ -917,13 +850,12 @@ inline Slice ArenaWrappedDBIter::key() const { return db_iter_->key(); }
 inline Slice ArenaWrappedDBIter::value() const { return db_iter_->value(); }
 inline Status ArenaWrappedDBIter::status() const { return db_iter_->status(); }
 inline Status ArenaWrappedDBIter::PinData() { return db_iter_->PinData(); }
-inline Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
-                                              std::string* prop) {
-  return db_iter_->GetProperty(prop_name, prop);
-}
 inline Status ArenaWrappedDBIter::ReleasePinnedData() {
   return db_iter_->ReleasePinnedData();
 }
+inline bool ArenaWrappedDBIter::IsKeyPinned() const {
+  return db_iter_->IsKeyPinned();
+}
 void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1,
                                          void* arg2) {
   db_iter_->RegisterCleanup(function, arg1, arg2);
@@ -932,7 +864,7 @@ void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1,
 ArenaWrappedDBIter* NewArenaWrappedDbIterator(
     Env* env, const ImmutableCFOptions& ioptions,
     const Comparator* user_key_comparator, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+    uint64_t max_sequential_skip_in_iterations,
     const Slice* iterate_upper_bound, bool prefix_same_as_start,
     bool pin_data) {
   ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
@@ -940,7 +872,7 @@ ArenaWrappedDBIter* NewArenaWrappedDbIterator(
   auto mem = arena->AllocateAligned(sizeof(DBIter));
   DBIter* db_iter =
       new (mem) DBIter(env, ioptions, user_key_comparator, nullptr, sequence,
-                       true, max_sequential_skip_in_iterations, version_number,
+                       true, max_sequential_skip_in_iterations,
                        iterate_upper_bound, prefix_same_as_start);
 
   iter->SetDBIter(db_iter);

db/db_iter.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -9,7 +9,6 @@
 
 #pragma once
 #include <stdint.h>
-#include <string>
 #include "rocksdb/db.h"
 #include "rocksdb/iterator.h"
 #include "db/dbformat.h"
@@ -25,12 +24,14 @@ class InternalIterator;
 // Return a new iterator that converts internal keys (yielded by
 // "*internal_iter") that were live at the specified "sequence" number
 // into appropriate user keys.
-extern Iterator* NewDBIterator(
-    Env* env, const ImmutableCFOptions& options,
-    const Comparator* user_key_comparator, InternalIterator* internal_iter,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
-    uint64_t version_number, const Slice* iterate_upper_bound = nullptr,
-    bool prefix_same_as_start = false, bool pin_data = false);
+extern Iterator* NewDBIterator(Env* env, const ImmutableCFOptions& options,
+                               const Comparator* user_key_comparator,
+                               InternalIterator* internal_iter,
+                               const SequenceNumber& sequence,
+                               uint64_t max_sequential_skip_in_iterations,
+                               const Slice* iterate_upper_bound = nullptr,
+                               bool prefix_same_as_start = false,
+                               bool pin_data = false);
 
 // A wrapper iterator which wraps DB Iterator and the arena, with which the DB
 // iterator is supposed be allocated. This class is used as an entry point of
@@ -65,7 +66,7 @@ class ArenaWrappedDBIter : public Iterator {
   void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2);
   virtual Status PinData();
   virtual Status ReleasePinnedData();
-  virtual Status GetProperty(std::string prop_name, std::string* prop) override;
+  virtual bool IsKeyPinned() const override;
 
  private:
   DBIter* db_iter_;
@@ -76,7 +77,7 @@ class ArenaWrappedDBIter : public Iterator {
 extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
     Env* env, const ImmutableCFOptions& options,
     const Comparator* user_key_comparator, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+    uint64_t max_sequential_skip_in_iterations,
     const Slice* iterate_upper_bound = nullptr,
     bool prefix_same_as_start = false, bool pin_data = false);
 

db/db_iter_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -181,9 +181,10 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     internal_iter->AddPut("b", "val_b");
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 10,
+                      options.max_sequential_skip_in_iterations));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -214,7 +215,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0));
+        10, options.max_sequential_skip_in_iterations));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -240,8 +241,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -273,8 +273,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -309,8 +308,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
@@ -339,8 +337,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        7, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        7, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     SetPerfLevel(kEnableCount);
     ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -377,8 +374,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        4, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -403,8 +399,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
@@ -426,8 +421,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -462,8 +456,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        7, options.max_sequential_skip_in_iterations, 0,
-        ro.iterate_upper_bound));
+        7, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
 
     SetPerfLevel(kEnableCount);
     ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -489,9 +482,10 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     internal_iter->AddPut("b", "val_b");
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 10,
+                      options.max_sequential_skip_in_iterations));
 
     db_iter->SeekToFirst();
     ASSERT_TRUE(db_iter->Valid());
@@ -530,9 +524,10 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     internal_iter->AddPut("b", "val_b");
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        2, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 2,
+                      options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -560,9 +555,10 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     internal_iter->AddPut("c", "val_c");
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 10,
+                      options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -587,9 +583,10 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {
     TestIterator* internal_iter = new TestIterator(BytewiseComparator());
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 0,
+                      options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
   }
@@ -598,9 +595,10 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {
     TestIterator* internal_iter = new TestIterator(BytewiseComparator());
     internal_iter->Finish();
 
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 0,
+                      options.max_sequential_skip_in_iterations));
     db_iter->SeekToFirst();
     ASSERT_TRUE(!db_iter->Valid());
   }
@@ -619,9 +617,10 @@ TEST_F(DBIteratorTest, DBIteratorUseSkipCountSkips) {
   }
   internal_iter->Finish();
 
-  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 2,
-      options.max_sequential_skip_in_iterations, 0));
+  std::unique_ptr<Iterator> db_iter(
+      NewDBIterator(env_, ImmutableCFOptions(options),
+                    BytewiseComparator(), internal_iter, 2,
+                    options.max_sequential_skip_in_iterations));
   db_iter->SeekToLast();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -660,8 +659,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
 
       options.statistics = rocksdb::CreateDBStatistics();
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
+          env_, ImmutableCFOptions(options),
+          BytewiseComparator(), internal_iter, i + 2,
+          options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -695,8 +695,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->Finish();
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
+          env_, ImmutableCFOptions(options),
+          BytewiseComparator(), internal_iter, i + 2,
+          options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -723,8 +724,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->Finish();
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, 202, options.max_sequential_skip_in_iterations, 0));
+          env_, ImmutableCFOptions(options),
+          BytewiseComparator(), internal_iter, 202,
+          options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -754,9 +756,10 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       }
       internal_iter->AddPut("c", "200");
       internal_iter->Finish();
-      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, i, options.max_sequential_skip_in_iterations, 0));
+      std::unique_ptr<Iterator> db_iter(
+          NewDBIterator(env_, ImmutableCFOptions(options),
+                        BytewiseComparator(), internal_iter, i,
+                        options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(!db_iter->Valid());
 
@@ -770,9 +773,10 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
     }
     internal_iter->AddPut("c", "200");
     internal_iter->Finish();
-    std::unique_ptr<Iterator> db_iter(NewDBIterator(
-        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        200, options.max_sequential_skip_in_iterations, 0));
+    std::unique_ptr<Iterator> db_iter(
+        NewDBIterator(env_, ImmutableCFOptions(options),
+                      BytewiseComparator(), internal_iter, 200,
+                      options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -805,8 +809,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->Finish();
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
+          env_, ImmutableCFOptions(options),
+          BytewiseComparator(), internal_iter, i + 2,
+          options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -839,8 +844,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->Finish();
 
       std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options), BytewiseComparator(),
-          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
+          env_, ImmutableCFOptions(options),
+          BytewiseComparator(), internal_iter, i + 2,
+          options.max_sequential_skip_in_iterations));
       db_iter->SeekToLast();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -881,7 +887,7 @@ TEST_F(DBIteratorTest, DBIterator1) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 1,
-      options.max_sequential_skip_in_iterations, 0));
+      options.max_sequential_skip_in_iterations));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -907,7 +913,7 @@ TEST_F(DBIteratorTest, DBIterator2) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 0,
-      options.max_sequential_skip_in_iterations, 0));
+      options.max_sequential_skip_in_iterations));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -930,7 +936,7 @@ TEST_F(DBIteratorTest, DBIterator3) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 2,
-      options.max_sequential_skip_in_iterations, 0));
+      options.max_sequential_skip_in_iterations));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -952,7 +958,7 @@ TEST_F(DBIteratorTest, DBIterator4) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 4,
-      options.max_sequential_skip_in_iterations, 0));
+      options.max_sequential_skip_in_iterations));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -981,7 +987,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations, 0));
+        0, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1003,7 +1009,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        1, options.max_sequential_skip_in_iterations, 0));
+        1, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1025,7 +1031,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        2, options.max_sequential_skip_in_iterations, 0));
+        2, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1047,7 +1053,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        3, options.max_sequential_skip_in_iterations, 0));
+        3, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1069,7 +1075,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations, 0));
+        4, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1091,7 +1097,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        5, options.max_sequential_skip_in_iterations, 0));
+        5, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1113,7 +1119,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        6, options.max_sequential_skip_in_iterations, 0));
+        6, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1139,7 +1145,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations, 0));
+        0, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1161,7 +1167,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        1, options.max_sequential_skip_in_iterations, 0));
+        1, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1183,7 +1189,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        2, options.max_sequential_skip_in_iterations, 0));
+        2, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1205,7 +1211,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        3, options.max_sequential_skip_in_iterations, 0));
+        3, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(!db_iter->Valid());
   }
@@ -1223,7 +1229,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations, 0));
+        4, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1245,7 +1251,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        5, options.max_sequential_skip_in_iterations, 0));
+        5, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1267,7 +1273,7 @@ TEST_F(DBIteratorTest, DBIterator6) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        6, options.max_sequential_skip_in_iterations, 0));
+        6, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1305,7 +1311,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations, 0));
+        0, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
     ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1339,7 +1345,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        2, options.max_sequential_skip_in_iterations, 0));
+        2, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1379,7 +1385,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations, 0));
+        4, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1419,7 +1425,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        5, options.max_sequential_skip_in_iterations, 0));
+        5, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1464,7 +1470,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        6, options.max_sequential_skip_in_iterations, 0));
+        6, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1510,7 +1516,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        7, options.max_sequential_skip_in_iterations, 0));
+        7, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1550,7 +1556,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        9, options.max_sequential_skip_in_iterations, 0));
+        9, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1596,7 +1602,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        13, options.max_sequential_skip_in_iterations, 0));
+        13, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1643,7 +1649,7 @@ TEST_F(DBIteratorTest, DBIterator7) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        14, options.max_sequential_skip_in_iterations, 0));
+        14, options.max_sequential_skip_in_iterations));
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
 
@@ -1672,7 +1678,7 @@ TEST_F(DBIteratorTest, DBIterator8) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-      10, options.max_sequential_skip_in_iterations, 0));
+      10, options.max_sequential_skip_in_iterations));
   db_iter->SeekToLast();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -1701,7 +1707,7 @@ TEST_F(DBIteratorTest, DBIterator9) {
 
     std::unique_ptr<Iterator> db_iter(NewDBIterator(
         env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, 0));
+        10, options.max_sequential_skip_in_iterations));
 
     db_iter->SeekToLast();
     ASSERT_TRUE(db_iter->Valid());
@@ -1748,7 +1754,7 @@ TEST_F(DBIteratorTest, DBIterator10) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-      10, options.max_sequential_skip_in_iterations, 0));
+      10, options.max_sequential_skip_in_iterations));
 
   db_iter->Seek("c");
   ASSERT_TRUE(db_iter->Valid());
@@ -1772,9 +1778,9 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {
   internal_iter->AddPut("b", "2");
   internal_iter->Finish();
 
-  std::unique_ptr<Iterator> db_iter(
-      NewDBIterator(env_, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0 /* force seek */, 0));
+  std::unique_ptr<Iterator> db_iter(NewDBIterator(
+      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
+      10, 0 /* force seek */));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1801,7 +1807,7 @@ TEST_F(DBIteratorTest, DBIterator11) {
 
   std::unique_ptr<Iterator> db_iter(NewDBIterator(
       env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 1,
-      options.max_sequential_skip_in_iterations, 0));
+      options.max_sequential_skip_in_iterations));
   db_iter->SeekToFirst();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1826,7 +1832,7 @@ TEST_F(DBIteratorTest, DBIterator12) {
 
   std::unique_ptr<Iterator> db_iter(
       NewDBIterator(env_, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0, 0));
+                    internal_iter, 10, 0));
   db_iter->SeekToLast();
   ASSERT_TRUE(db_iter->Valid());
   ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -1868,7 +1874,7 @@ class DBIterWithMergeIterTest : public testing::Test {
     db_iter_.reset(NewDBIterator(env_, ImmutableCFOptions(options_),
                                  BytewiseComparator(), merge_iter,
                                  8 /* read data earlier than seqId 8 */,
-                                 3 /* max iterators before reseek */, 0));
+                                 3 /* max iterators before reseek */));
   }
 
   Env* env_;

db/db_log_iter_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/db_table_properties_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -215,34 +215,6 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfTablesInRange) {
     TestGetPropertiesOfTablesInRange(std::move(ranges));
   }
 }
-
-TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) {
-  std::string kExtraCfName = "pikachu";
-  CreateAndReopenWithCF({kExtraCfName}, CurrentOptions());
-
-  // Create one table per CF, then verify it was created with the column family
-  // name property.
-  for (int cf = 0; cf < 2; ++cf) {
-    Put(cf, "key", "val");
-    Flush(cf);
-
-    TablePropertiesCollection fname_to_props;
-    ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props));
-    ASSERT_EQ(1U, fname_to_props.size());
-
-    std::string expected_cf_name;
-    if (cf > 0) {
-      expected_cf_name = kExtraCfName;
-    } else {
-      expected_cf_name = kDefaultColumnFamilyName;
-    }
-    ASSERT_EQ(expected_cf_name,
-              fname_to_props.begin()->second->column_family_name);
-    ASSERT_EQ(cf, static_cast<uint32_t>(
-                      fname_to_props.begin()->second->column_family_id));
-  }
-}
-
 }  // namespace rocksdb
 
 #endif  // ROCKSDB_LITE

db/db_tailing_iter_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -654,51 +654,6 @@ TEST_F(DBTestTailingIterator, ManagedTailingIteratorSeekToSame) {
   ASSERT_EQ(found, iter->key().ToString());
 }
 
-TEST_F(DBTestTailingIterator, ForwardIteratorVersionProperty) {
-  Options options = CurrentOptions();
-  options.write_buffer_size = 1000;
-
-  ReadOptions read_options;
-  read_options.tailing = true;
-
-  Put("foo", "bar");
-
-  uint64_t v1, v2, v3, v4;
-  {
-    std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
-    iter->Seek("foo");
-    std::string prop_value;
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
-                                &prop_value));
-    v1 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-    Put("foo1", "bar1");
-    Flush();
-
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
-                                &prop_value));
-    v2 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-    iter->Seek("f");
-
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
-                                &prop_value));
-    v3 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-    ASSERT_EQ(v1, v2);
-    ASSERT_GT(v3, v2);
-  }
-
-  {
-    std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
-    iter->Seek("foo");
-    std::string prop_value;
-    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
-                                &prop_value));
-    v4 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-  }
-  ASSERT_EQ(v3, v4);
-}
 }  // namespace rocksdb
 
 #endif  // !defined(ROCKSDB_LITE)

db/db_test2.cc

@@ -1,864 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include <cstdlib>
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-#include "rocksdb/wal_filter.h"
-
-namespace rocksdb {
-
-static uint64_t TestGetTickerCount(const Options& options,
-                                   Tickers ticker_type) {
-  return options.statistics->getTickerCount(ticker_type);
-}
-
-class DBTest2 : public DBTestBase {
- public:
-  DBTest2() : DBTestBase("/db_test2") {}
-};
-
-class PrefixFullBloomWithReverseComparator
-    : public DBTestBase,
-      public ::testing::WithParamInterface<bool> {
- public:
-  PrefixFullBloomWithReverseComparator()
-      : DBTestBase("/prefix_bloom_reverse") {}
-  virtual void SetUp() override { if_cache_filter_ = GetParam(); }
-  bool if_cache_filter_;
-};
-
-TEST_P(PrefixFullBloomWithReverseComparator,
-       PrefixFullBloomWithReverseComparator) {
-  Options options = last_options_;
-  options.comparator = ReverseBytewiseComparator();
-  options.prefix_extractor.reset(NewCappedPrefixTransform(3));
-  options.statistics = rocksdb::CreateDBStatistics();
-  BlockBasedTableOptions bbto;
-  if (if_cache_filter_) {
-    bbto.no_block_cache = false;
-    bbto.cache_index_and_filter_blocks = true;
-    bbto.block_cache = NewLRUCache(1);
-  }
-  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
-  bbto.whole_key_filtering = false;
-  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-  DestroyAndReopen(options);
-
-  ASSERT_OK(dbfull()->Put(WriteOptions(), "bar123", "foo"));
-  ASSERT_OK(dbfull()->Put(WriteOptions(), "bar234", "foo2"));
-  ASSERT_OK(dbfull()->Put(WriteOptions(), "foo123", "foo3"));
-
-  dbfull()->Flush(FlushOptions());
-
-  if (bbto.block_cache) {
-    bbto.block_cache->EraseUnRefEntries();
-  }
-
-  unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
-  iter->Seek("bar345");
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ("bar234", iter->key().ToString());
-  ASSERT_EQ("foo2", iter->value().ToString());
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ("bar123", iter->key().ToString());
-  ASSERT_EQ("foo", iter->value().ToString());
-
-  iter->Seek("foo234");
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ("foo123", iter->key().ToString());
-  ASSERT_EQ("foo3", iter->value().ToString());
-
-  iter->Seek("bar");
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(!iter->Valid());
-}
-
-INSTANTIATE_TEST_CASE_P(PrefixFullBloomWithReverseComparator,
-                        PrefixFullBloomWithReverseComparator, testing::Bool());
-
-TEST_F(DBTest2, IteratorPropertyVersionNumber) {
-  Put("", "");
-  Iterator* iter1 = db_->NewIterator(ReadOptions());
-  std::string prop_value;
-  ASSERT_OK(
-      iter1->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
-  uint64_t version_number1 =
-      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-  Put("", "");
-  Flush();
-
-  Iterator* iter2 = db_->NewIterator(ReadOptions());
-  ASSERT_OK(
-      iter2->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
-  uint64_t version_number2 =
-      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-  ASSERT_GT(version_number2, version_number1);
-
-  Put("", "");
-
-  Iterator* iter3 = db_->NewIterator(ReadOptions());
-  ASSERT_OK(
-      iter3->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
-  uint64_t version_number3 =
-      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-
-  ASSERT_EQ(version_number2, version_number3);
-
-  iter1->SeekToFirst();
-  ASSERT_OK(
-      iter1->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
-  uint64_t version_number1_new =
-      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
-  ASSERT_EQ(version_number1, version_number1_new);
-
-  delete iter1;
-  delete iter2;
-  delete iter3;
-}
-
-TEST_F(DBTest2, CacheIndexAndFilterWithDBRestart) {
-  Options options = CurrentOptions();
-  options.create_if_missing = true;
-  options.statistics = rocksdb::CreateDBStatistics();
-  BlockBasedTableOptions table_options;
-  table_options.cache_index_and_filter_blocks = true;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  Put(1, "a", "begin");
-  Put(1, "z", "end");
-  ASSERT_OK(Flush(1));
-  TryReopenWithColumnFamilies({"default", "pikachu"}, options);
-
-  std::string value;
-  value = Get(1, "a");
-}
-
-#ifndef ROCKSDB_LITE
-namespace {
-  void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
-    const std::vector<Slice>& keys_must_not_exist) {
-    // Ensure that expected keys exist
-    std::vector<std::string> values;
-    if (keys_must_exist.size() > 0) {
-      std::vector<Status> status_list =
-        db->MultiGet(ReadOptions(), keys_must_exist, &values);
-      for (size_t i = 0; i < keys_must_exist.size(); i++) {
-        ASSERT_OK(status_list[i]);
-      }
-    }
-
-    // Ensure that given keys don't exist
-    if (keys_must_not_exist.size() > 0) {
-      std::vector<Status> status_list =
-        db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
-      for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
-        ASSERT_TRUE(status_list[i].IsNotFound());
-      }
-    }
-  }
-
-}  // namespace
-
-TEST_F(DBTest2, WalFilterTest) {
-  class TestWalFilter : public WalFilter {
-  private:
-    // Processing option that is requested to be applied at the given index
-    WalFilter::WalProcessingOption wal_processing_option_;
-    // Index at which to apply wal_processing_option_
-    // At other indexes default wal_processing_option::kContinueProcessing is
-    // returned.
-    size_t apply_option_at_record_index_;
-    // Current record index, incremented with each record encountered.
-    size_t current_record_index_;
-
-  public:
-    TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
-      size_t apply_option_for_record_index)
-      : wal_processing_option_(wal_processing_option),
-      apply_option_at_record_index_(apply_option_for_record_index),
-      current_record_index_(0) {}
-
-    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
-      WriteBatch* new_batch,
-      bool* batch_changed) const override {
-      WalFilter::WalProcessingOption option_to_return;
-
-      if (current_record_index_ == apply_option_at_record_index_) {
-        option_to_return = wal_processing_option_;
-      }
-      else {
-        option_to_return = WalProcessingOption::kContinueProcessing;
-      }
-
-      // Filter is passed as a const object for RocksDB to not modify the
-      // object, however we modify it for our own purpose here and hence
-      // cast the constness away.
-      (const_cast<TestWalFilter*>(this)->current_record_index_)++;
-
-      return option_to_return;
-    }
-
-    virtual const char* Name() const override { return "TestWalFilter"; }
-  };
-
-  // Create 3 batches with two keys each
-  std::vector<std::vector<std::string>> batch_keys(3);
-
-  batch_keys[0].push_back("key1");
-  batch_keys[0].push_back("key2");
-  batch_keys[1].push_back("key3");
-  batch_keys[1].push_back("key4");
-  batch_keys[2].push_back("key5");
-  batch_keys[2].push_back("key6");
-
-  // Test with all WAL processing options
-  for (int option = 0;
-    option < static_cast<int>(
-    WalFilter::WalProcessingOption::kWalProcessingOptionMax);
-  option++) {
-    Options options = OptionsForLogIterTest();
-    DestroyAndReopen(options);
-    CreateAndReopenWithCF({ "pikachu" }, options);
-
-    // Write given keys in given batches
-    for (size_t i = 0; i < batch_keys.size(); i++) {
-      WriteBatch batch;
-      for (size_t j = 0; j < batch_keys[i].size(); j++) {
-        batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
-      }
-      dbfull()->Write(WriteOptions(), &batch);
-    }
-
-    WalFilter::WalProcessingOption wal_processing_option =
-      static_cast<WalFilter::WalProcessingOption>(option);
-
-    // Create a test filter that would apply wal_processing_option at the first
-    // record
-    size_t apply_option_for_record_index = 1;
-    TestWalFilter test_wal_filter(wal_processing_option,
-      apply_option_for_record_index);
-
-    // Reopen database with option to use WAL filter
-    options = OptionsForLogIterTest();
-    options.wal_filter = &test_wal_filter;
-    Status status =
-      TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
-    if (wal_processing_option ==
-      WalFilter::WalProcessingOption::kCorruptedRecord) {
-      assert(!status.ok());
-      // In case of corruption we can turn off paranoid_checks to reopen
-      // databse
-      options.paranoid_checks = false;
-      ReopenWithColumnFamilies({ "default", "pikachu" }, options);
-    }
-    else {
-      assert(status.ok());
-    }
-
-    // Compute which keys we expect to be found
-    // and which we expect not to be found after recovery.
-    std::vector<Slice> keys_must_exist;
-    std::vector<Slice> keys_must_not_exist;
-    switch (wal_processing_option) {
-    case WalFilter::WalProcessingOption::kCorruptedRecord:
-    case WalFilter::WalProcessingOption::kContinueProcessing: {
-      fprintf(stderr, "Testing with complete WAL processing\n");
-      // we expect all records to be processed
-      for (size_t i = 0; i < batch_keys.size(); i++) {
-        for (size_t j = 0; j < batch_keys[i].size(); j++) {
-          keys_must_exist.push_back(Slice(batch_keys[i][j]));
-        }
-      }
-      break;
-    }
-    case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
-      fprintf(stderr,
-        "Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
-        apply_option_for_record_index);
-      // We expect the record with apply_option_for_record_index to be not
-      // found.
-      for (size_t i = 0; i < batch_keys.size(); i++) {
-        for (size_t j = 0; j < batch_keys[i].size(); j++) {
-          if (i == apply_option_for_record_index) {
-            keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
-          }
-          else {
-            keys_must_exist.push_back(Slice(batch_keys[i][j]));
-          }
-        }
-      }
-      break;
-    }
-    case WalFilter::WalProcessingOption::kStopReplay: {
-      fprintf(stderr,
-        "Testing with stopping replay from record %" ROCKSDB_PRIszt
-        "\n",
-        apply_option_for_record_index);
-      // We expect records beyond apply_option_for_record_index to be not
-      // found.
-      for (size_t i = 0; i < batch_keys.size(); i++) {
-        for (size_t j = 0; j < batch_keys[i].size(); j++) {
-          if (i >= apply_option_for_record_index) {
-            keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
-          }
-          else {
-            keys_must_exist.push_back(Slice(batch_keys[i][j]));
-          }
-        }
-      }
-      break;
-    }
-    default:
-      assert(false);  // unhandled case
-    }
-
-    bool checked_after_reopen = false;
-
-    while (true) {
-      // Ensure that expected keys exists
-      // and not expected keys don't exist after recovery
-      ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
-
-      if (checked_after_reopen) {
-        break;
-      }
-
-      // reopen database again to make sure previous log(s) are not used
-      //(even if they were skipped)
-      // reopn database with option to use WAL filter
-      options = OptionsForLogIterTest();
-      ReopenWithColumnFamilies({ "default", "pikachu" }, options);
-
-      checked_after_reopen = true;
-    }
-  }
-}
-
-TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
-  class ChangeBatchHandler : public WriteBatch::Handler {
-  private:
-    // Batch to insert keys in
-    WriteBatch* new_write_batch_;
-    // Number of keys to add in the new batch
-    size_t num_keys_to_add_in_new_batch_;
-    // Number of keys added to new batch
-    size_t num_keys_added_;
-
-  public:
-    ChangeBatchHandler(WriteBatch* new_write_batch,
-      size_t num_keys_to_add_in_new_batch)
-      : new_write_batch_(new_write_batch),
-      num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
-      num_keys_added_(0) {}
-    virtual void Put(const Slice& key, const Slice& value) override {
-      if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
-        new_write_batch_->Put(key, value);
-        ++num_keys_added_;
-      }
-    }
-  };
-
-  class TestWalFilterWithChangeBatch : public WalFilter {
-  private:
-    // Index at which to start changing records
-    size_t change_records_from_index_;
-    // Number of keys to add in the new batch
-    size_t num_keys_to_add_in_new_batch_;
-    // Current record index, incremented with each record encountered.
-    size_t current_record_index_;
-
-  public:
-    TestWalFilterWithChangeBatch(size_t change_records_from_index,
-      size_t num_keys_to_add_in_new_batch)
-      : change_records_from_index_(change_records_from_index),
-      num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
-      current_record_index_(0) {}
-
-    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
-      WriteBatch* new_batch,
-      bool* batch_changed) const override {
-      if (current_record_index_ >= change_records_from_index_) {
-        ChangeBatchHandler handler(new_batch, num_keys_to_add_in_new_batch_);
-        batch.Iterate(&handler);
-        *batch_changed = true;
-      }
-
-      // Filter is passed as a const object for RocksDB to not modify the
-      // object, however we modify it for our own purpose here and hence
-      // cast the constness away.
-      (const_cast<TestWalFilterWithChangeBatch*>(this)
-        ->current_record_index_)++;
-
-      return WalProcessingOption::kContinueProcessing;
-    }
-
-    virtual const char* Name() const override {
-      return "TestWalFilterWithChangeBatch";
-    }
-  };
-
-  std::vector<std::vector<std::string>> batch_keys(3);
-
-  batch_keys[0].push_back("key1");
-  batch_keys[0].push_back("key2");
-  batch_keys[1].push_back("key3");
-  batch_keys[1].push_back("key4");
-  batch_keys[2].push_back("key5");
-  batch_keys[2].push_back("key6");
-
-  Options options = OptionsForLogIterTest();
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({ "pikachu" }, options);
-
-  // Write given keys in given batches
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    WriteBatch batch;
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
-    }
-    dbfull()->Write(WriteOptions(), &batch);
-  }
-
-  // Create a test filter that would apply wal_processing_option at the first
-  // record
-  size_t change_records_from_index = 1;
-  size_t num_keys_to_add_in_new_batch = 1;
-  TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
-    change_records_from_index, num_keys_to_add_in_new_batch);
-
-  // Reopen database with option to use WAL filter
-  options = OptionsForLogIterTest();
-  options.wal_filter = &test_wal_filter_with_change_batch;
-  ReopenWithColumnFamilies({ "default", "pikachu" }, options);
-
-  // Ensure that all keys exist before change_records_from_index_
-  // And after that index only single key exists
-  // as our filter adds only single key for each batch
-  std::vector<Slice> keys_must_exist;
-  std::vector<Slice> keys_must_not_exist;
-
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      if (i >= change_records_from_index && j >= 
-        num_keys_to_add_in_new_batch) {
-        keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
-      }
-      else {
-        keys_must_exist.push_back(Slice(batch_keys[i][j]));
-      }
-    }
-  }
-
-  bool checked_after_reopen = false;
-
-  while (true) {
-    // Ensure that expected keys exists
-    // and not expected keys don't exist after recovery
-    ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
-
-    if (checked_after_reopen) {
-      break;
-    }
-
-    // reopen database again to make sure previous log(s) are not used
-    //(even if they were skipped)
-    // reopn database with option to use WAL filter
-    options = OptionsForLogIterTest();
-    ReopenWithColumnFamilies({ "default", "pikachu" }, options);
-
-    checked_after_reopen = true;
-  }
-}
-
-TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
-  class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
-  public:
-    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
-      WriteBatch* new_batch,
-      bool* batch_changed) const override {
-      *new_batch = batch;
-      new_batch->Put("key_extra", "value_extra");
-      *batch_changed = true;
-      return WalProcessingOption::kContinueProcessing;
-    }
-
-    virtual const char* Name() const override {
-      return "WalFilterTestWithChangeBatchExtraKeys";
-    }
-  };
-
-  std::vector<std::vector<std::string>> batch_keys(3);
-
-  batch_keys[0].push_back("key1");
-  batch_keys[0].push_back("key2");
-  batch_keys[1].push_back("key3");
-  batch_keys[1].push_back("key4");
-  batch_keys[2].push_back("key5");
-  batch_keys[2].push_back("key6");
-
-  Options options = OptionsForLogIterTest();
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({ "pikachu" }, options);
-
-  // Write given keys in given batches
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    WriteBatch batch;
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
-    }
-    dbfull()->Write(WriteOptions(), &batch);
-  }
-
-  // Create a test filter that would add extra keys
-  TestWalFilterWithChangeBatchAddExtraKeys test_wal_filter_extra_keys;
-
-  // Reopen database with option to use WAL filter
-  options = OptionsForLogIterTest();
-  options.wal_filter = &test_wal_filter_extra_keys;
-  Status status = TryReopenWithColumnFamilies({ "default", "pikachu" }, 
-    options);
-  ASSERT_TRUE(status.IsNotSupported());
-
-  // Reopen without filter, now reopen should succeed - previous
-  // attempt to open must not have altered the db.
-  options = OptionsForLogIterTest();
-  ReopenWithColumnFamilies({ "default", "pikachu" }, options);
-
-  std::vector<Slice> keys_must_exist;
-  std::vector<Slice> keys_must_not_exist;  // empty vector
-
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      keys_must_exist.push_back(Slice(batch_keys[i][j]));
-    }
-  }
-
-  ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
-}
-
-TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
-  class TestWalFilterWithColumnFamilies : public WalFilter {
-  private:
-    // column_family_id -> log_number map (provided to WALFilter)
-    std::map<uint32_t, uint64_t> cf_log_number_map_;
-    // column_family_name -> column_family_id map (provided to WALFilter)
-    std::map<std::string, uint32_t> cf_name_id_map_;
-    // column_family_name -> keys_found_in_wal map
-    // We store keys that are applicable to the column_family
-    // during recovery (i.e. aren't already flushed to SST file(s))
-    // for verification against the keys we expect.
-    std::map<uint32_t, std::vector<std::string>> cf_wal_keys_;
-  public:
-    virtual void ColumnFamilyLogNumberMap(
-      const std::map<uint32_t, uint64_t>& cf_lognumber_map,
-      const std::map<std::string, uint32_t>& cf_name_id_map) override {
-      cf_log_number_map_ = cf_lognumber_map;
-      cf_name_id_map_ = cf_name_id_map;
-    }
-
-    virtual WalProcessingOption LogRecordFound(unsigned long long log_number,
-      const std::string& log_file_name,
-      const WriteBatch& batch,
-      WriteBatch* new_batch,
-      bool* batch_changed) override {
-      class LogRecordBatchHandler : public WriteBatch::Handler {
-      private:
-        const std::map<uint32_t, uint64_t> & cf_log_number_map_;
-        std::map<uint32_t, std::vector<std::string>> & cf_wal_keys_;
-        unsigned long long log_number_;
-      public:
-        LogRecordBatchHandler(unsigned long long current_log_number,
-          const std::map<uint32_t, uint64_t> & cf_log_number_map,
-          std::map<uint32_t, std::vector<std::string>> & cf_wal_keys) :
-          cf_log_number_map_(cf_log_number_map),
-          cf_wal_keys_(cf_wal_keys),
-          log_number_(current_log_number){}
-
-        virtual Status PutCF(uint32_t column_family_id, const Slice& key,
-          const Slice& /*value*/) override {
-          auto it = cf_log_number_map_.find(column_family_id);
-          assert(it != cf_log_number_map_.end());
-          unsigned long long log_number_for_cf = it->second;
-          // If the current record is applicable for column_family_id
-          // (i.e. isn't flushed to SST file(s) for column_family_id)
-          // add it to the cf_wal_keys_ map for verification.
-          if (log_number_ >= log_number_for_cf) {
-            cf_wal_keys_[column_family_id].push_back(std::string(key.data(),
-              key.size()));
-          }
-          return Status::OK();
-        }
-      } handler(log_number, cf_log_number_map_, cf_wal_keys_);
-
-      batch.Iterate(&handler);
-
-      return WalProcessingOption::kContinueProcessing;
-    }
-
-    virtual const char* Name() const override {
-      return "WalFilterTestWithColumnFamilies";
-    }
-
-    const std::map<uint32_t, std::vector<std::string>> & 
-    GetColumnFamilyKeys() {
-      return cf_wal_keys_;
-    }
-
-    const std::map<std::string, uint32_t> & GetColumnFamilyNameIdMap() {
-      return cf_name_id_map_;
-    }
-  };
-
-  std::vector<std::vector<std::string>> batch_keys_pre_flush(3);
-
-  batch_keys_pre_flush[0].push_back("key1");
-  batch_keys_pre_flush[0].push_back("key2");
-  batch_keys_pre_flush[1].push_back("key3");
-  batch_keys_pre_flush[1].push_back("key4");
-  batch_keys_pre_flush[2].push_back("key5");
-  batch_keys_pre_flush[2].push_back("key6");
-
-  Options options = OptionsForLogIterTest();
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({ "pikachu" }, options);
-
-  // Write given keys in given batches
-  for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
-    WriteBatch batch;
-    for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
-      batch.Put(handles_[0], batch_keys_pre_flush[i][j], DummyString(1024));
-      batch.Put(handles_[1], batch_keys_pre_flush[i][j], DummyString(1024));
-    }
-    dbfull()->Write(WriteOptions(), &batch);
-  }
-
-  //Flush default column-family
-  db_->Flush(FlushOptions(), handles_[0]);
-
-  // Do some more writes
-  std::vector<std::vector<std::string>> batch_keys_post_flush(3);
-
-  batch_keys_post_flush[0].push_back("key7");
-  batch_keys_post_flush[0].push_back("key8");
-  batch_keys_post_flush[1].push_back("key9");
-  batch_keys_post_flush[1].push_back("key10");
-  batch_keys_post_flush[2].push_back("key11");
-  batch_keys_post_flush[2].push_back("key12");
-
-  // Write given keys in given batches
-  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
-    WriteBatch batch;
-    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
-      batch.Put(handles_[0], batch_keys_post_flush[i][j], DummyString(1024));
-      batch.Put(handles_[1], batch_keys_post_flush[i][j], DummyString(1024));
-    }
-    dbfull()->Write(WriteOptions(), &batch);
-  }
-
-  // On Recovery we should only find the second batch applicable to default CF
-  // But both batches applicable to pikachu CF
-
-  // Create a test filter that would add extra keys
-  TestWalFilterWithColumnFamilies test_wal_filter_column_families;
-
-  // Reopen database with option to use WAL filter
-  options = OptionsForLogIterTest();
-  options.wal_filter = &test_wal_filter_column_families;
-  Status status =
-    TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
-  ASSERT_TRUE(status.ok());
-
-  // verify that handles_[0] only has post_flush keys
-  // while handles_[1] has pre and post flush keys
-  auto cf_wal_keys = test_wal_filter_column_families.GetColumnFamilyKeys();
-  auto name_id_map = 
-    test_wal_filter_column_families.GetColumnFamilyNameIdMap();
-  size_t index = 0;
-  auto keys_cf = cf_wal_keys[name_id_map[kDefaultColumnFamilyName]];
-  //default column-family, only post_flush keys are expected
-  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
-    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
-      Slice key_from_the_log(keys_cf[index++]);
-      Slice batch_key(batch_keys_post_flush[i][j]);
-      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
-    }
-  }
-  ASSERT_TRUE(index == keys_cf.size());
-
-  index = 0;
-  keys_cf = cf_wal_keys[name_id_map["pikachu"]];
-  //pikachu column-family, all keys are expected
-  for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
-    for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
-      Slice key_from_the_log(keys_cf[index++]);
-      Slice batch_key(batch_keys_pre_flush[i][j]);
-      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
-    }
-  }
-
-  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
-    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
-      Slice key_from_the_log(keys_cf[index++]);
-      Slice batch_key(batch_keys_post_flush[i][j]);
-      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
-    }
-  }
-  ASSERT_TRUE(index == keys_cf.size());
-}
-#endif  // ROCKSDB_LITE
-
-TEST_F(DBTest2, DISABLED_FirstSnapshotTest) {
-  Options options;
-  options.write_buffer_size = 100000;  // Small write buffer
-  options = CurrentOptions(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  // This snapshot will have sequence number 0.  When compaction encounters
-  // this snapshot, CompactionIterator::findEarliestVisibleSnapshot() will
-  // assert as it expects non-zero snapshots.
-  //
-  // One fix would be to simply remove this assert.  However, a better fix
-  // might
-  // be to always have db sequence numbers start from 1 so that no code is
-  // ever
-  // confused by 0.
-  const Snapshot* s1 = db_->GetSnapshot();
-
-  Put(1, "k1", std::string(100000, 'x'));  // Fill memtable
-  Put(1, "k2", std::string(100000, 'y'));  // Trigger flush
-
-  db_->ReleaseSnapshot(s1);
-}
-
-class PinL0IndexAndFilterBlocksTest : public DBTestBase,
-                                      public testing::WithParamInterface<bool> {
- public:
-  PinL0IndexAndFilterBlocksTest() : DBTestBase("/db_pin_l0_index_bloom_test") {}
-  virtual void SetUp() override { infinite_max_files_ = GetParam(); }
-
-  bool infinite_max_files_;
-};
-
-TEST_P(PinL0IndexAndFilterBlocksTest,
-       IndexAndFilterBlocksOfNewTableAddedToCacheWithPinning) {
-  Options options = CurrentOptions();
-  if (infinite_max_files_) {
-    options.max_open_files = -1;
-  }
-  options.create_if_missing = true;
-  options.statistics = rocksdb::CreateDBStatistics();
-  BlockBasedTableOptions table_options;
-  table_options.cache_index_and_filter_blocks = true;
-  table_options.pin_l0_filter_and_index_blocks_in_cache = true;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  ASSERT_OK(Put(1, "key", "val"));
-  // Create a new table.
-  ASSERT_OK(Flush(1));
-
-  // index/filter blocks added to block cache right after table creation.
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
-
-  // only index/filter were added
-  ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS));
-
-  std::string value;
-  // Miss and hit count should remain the same, they're all pinned.
-  db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value);
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
-
-  // Miss and hit count should remain the same, they're all pinned.
-  value = Get(1, "key");
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
-  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
-}
-
-TEST_P(PinL0IndexAndFilterBlocksTest,
-       MultiLevelIndexAndFilterBlocksCachedWithPinning) {
-  Options options = CurrentOptions();
-  if (infinite_max_files_) {
-    options.max_open_files = -1;
-  }
-  options.create_if_missing = true;
-  options.statistics = rocksdb::CreateDBStatistics();
-  BlockBasedTableOptions table_options;
-  table_options.cache_index_and_filter_blocks = true;
-  table_options.pin_l0_filter_and_index_blocks_in_cache = true;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
-  options.table_factory.reset(new BlockBasedTableFactory(table_options));
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  Put(1, "a", "begin");
-  Put(1, "z", "end");
-  ASSERT_OK(Flush(1));
-  // move this table to L1
-  dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
-
-  // reset block cache
-  table_options.block_cache = NewLRUCache(64 * 1024);
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  TryReopenWithColumnFamilies({"default", "pikachu"}, options);
-  // create new table at L0
-  Put(1, "a2", "begin2");
-  Put(1, "z2", "end2");
-  ASSERT_OK(Flush(1));
-
-  table_options.block_cache->EraseUnRefEntries();
-
-  // get base cache values
-  uint64_t fm = TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS);
-  uint64_t fh = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
-  uint64_t im = TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS);
-  uint64_t ih = TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT);
-
-  std::string value;
-  // this should be read from L0
-  // so cache values don't change
-  value = Get(1, "a2");
-  ASSERT_EQ(fm, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-  ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
-  ASSERT_EQ(im, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-  ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
-
-  // this should be read from L1
-  // the file is opened, prefetching results in a cache filter miss
-  // the block is loaded and added to the cache,
-  // then the get results in a cache hit for L1
-  value = Get(1, "a");
-  ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
-  ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
-}
-
-INSTANTIATE_TEST_CASE_P(PinL0IndexAndFilterBlocksTest,
-                        PinL0IndexAndFilterBlocksTest, ::testing::Bool());
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  rocksdb::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/db_test_util.cc

@@ -1,4 +1,4 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+// Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
@@ -73,69 +73,64 @@ DBTestBase::~DBTestBase() {
   delete env_;
 }
 
-bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) {
-#ifdef ROCKSDB_LITE
-    // These options are not supported in ROCKSDB_LITE
-  if (option_config == kHashSkipList ||
-      option_config == kPlainTableFirstBytePrefix ||
-      option_config == kPlainTableCappedPrefix ||
-      option_config == kPlainTableCappedPrefixNonMmap ||
-      option_config == kPlainTableAllBytesPrefix ||
-      option_config == kVectorRep || option_config == kHashLinkList ||
-      option_config == kHashCuckoo || option_config == kUniversalCompaction ||
-      option_config == kUniversalCompactionMultiLevel ||
-      option_config == kUniversalSubcompactions ||
-      option_config == kFIFOCompaction ||
-      option_config == kConcurrentSkipList) {
-    return true;
-    }
-#endif
-
-    if ((skip_mask & kSkipDeletesFilterFirst) &&
-        option_config == kDeletesFilterFirst) {
-      return true;
-    }
-    if ((skip_mask & kSkipUniversalCompaction) &&
-        (option_config == kUniversalCompaction ||
-         option_config == kUniversalCompactionMultiLevel)) {
-      return true;
-    }
-    if ((skip_mask & kSkipMergePut) && option_config == kMergePut) {
-      return true;
-    }
-    if ((skip_mask & kSkipNoSeekToLast) &&
-        (option_config == kHashLinkList || option_config == kHashSkipList)) {
-      return true;
-    }
-    if ((skip_mask & kSkipPlainTable) &&
-        (option_config == kPlainTableAllBytesPrefix ||
-         option_config == kPlainTableFirstBytePrefix ||
-         option_config == kPlainTableCappedPrefix ||
-         option_config == kPlainTableCappedPrefixNonMmap)) {
-      return true;
-    }
-    if ((skip_mask & kSkipHashIndex) &&
-        (option_config == kBlockBasedTableWithPrefixHashIndex ||
-         option_config == kBlockBasedTableWithWholeKeyHashIndex)) {
-      return true;
-    }
-    if ((skip_mask & kSkipHashCuckoo) && (option_config == kHashCuckoo)) {
-      return true;
-    }
-    if ((skip_mask & kSkipFIFOCompaction) && option_config == kFIFOCompaction) {
-      return true;
-    }
-    if ((skip_mask & kSkipMmapReads) && option_config == kWalDirAndMmapReads) {
-      return true;
-    }
-    return false;
-}
-
 // Switch to a fresh database with the next option configuration to
 // test.  Return false if there are no more configurations to test.
 bool DBTestBase::ChangeOptions(int skip_mask) {
   for (option_config_++; option_config_ < kEnd; option_config_++) {
-    if (ShouldSkipOptions(option_config_, skip_mask)) {
+#ifdef ROCKSDB_LITE
+    // These options are not supported in ROCKSDB_LITE
+    if (option_config_ == kHashSkipList ||
+        option_config_ == kPlainTableFirstBytePrefix ||
+        option_config_ == kPlainTableCappedPrefix ||
+        option_config_ == kPlainTableCappedPrefixNonMmap ||
+        option_config_ == kPlainTableAllBytesPrefix ||
+        option_config_ == kVectorRep || option_config_ == kHashLinkList ||
+        option_config_ == kHashCuckoo ||
+        option_config_ == kUniversalCompaction ||
+        option_config_ == kUniversalCompactionMultiLevel ||
+        option_config_ == kUniversalSubcompactions ||
+        option_config_ == kFIFOCompaction ||
+        option_config_ == kConcurrentSkipList) {
+      continue;
+    }
+#endif
+
+    if ((skip_mask & kSkipDeletesFilterFirst) &&
+        option_config_ == kDeletesFilterFirst) {
+      continue;
+    }
+    if ((skip_mask & kSkipUniversalCompaction) &&
+        (option_config_ == kUniversalCompaction ||
+         option_config_ == kUniversalCompactionMultiLevel)) {
+      continue;
+    }
+    if ((skip_mask & kSkipMergePut) && option_config_ == kMergePut) {
+      continue;
+    }
+    if ((skip_mask & kSkipNoSeekToLast) &&
+        (option_config_ == kHashLinkList || option_config_ == kHashSkipList)) {
+      continue;
+    }
+    if ((skip_mask & kSkipPlainTable) &&
+        (option_config_ == kPlainTableAllBytesPrefix ||
+         option_config_ == kPlainTableFirstBytePrefix ||
+         option_config_ == kPlainTableCappedPrefix ||
+         option_config_ == kPlainTableCappedPrefixNonMmap)) {
+      continue;
+    }
+    if ((skip_mask & kSkipHashIndex) &&
+        (option_config_ == kBlockBasedTableWithPrefixHashIndex ||
+         option_config_ == kBlockBasedTableWithWholeKeyHashIndex)) {
+      continue;
+    }
+    if ((skip_mask & kSkipHashCuckoo) && (option_config_ == kHashCuckoo)) {
+      continue;
+    }
+    if ((skip_mask & kSkipFIFOCompaction) &&
+        option_config_ == kFIFOCompaction) {
+      continue;
+    }
+    if ((skip_mask & kSkipMmapReads) && option_config_ == kWalDirAndMmapReads) {
       continue;
     }
     break;
@@ -210,8 +205,6 @@ Options DBTestBase::CurrentOptions(
     const anon::OptionsOverride& options_override) {
   Options options;
   options.write_buffer_size = 4090 * 4096;
-  options.target_file_size_base = 2 * 1024 * 1024;
-  options.max_bytes_for_level_base = 10 * 1024 * 1024;
   return CurrentOptions(options, options_override);
 }
 
@@ -303,7 +296,6 @@ Options DBTestBase::CurrentOptions(
     case kPerfOptions:
       options.soft_rate_limit = 2.0;
       options.delayed_write_rate = 8 * 1024 * 1024;
-      options.report_bg_io_stats = true;
       // TODO(3.13) -- test more options
       break;
     case kDeletesFilterFirst:
@@ -342,10 +334,6 @@ Options DBTestBase::CurrentOptions(
       options.prefix_extractor.reset(NewNoopTransform());
       break;
     }
-    case kBlockBasedTableWithIndexRestartInterval: {
-      table_options.index_block_restart_interval = 8;
-      break;
-    }
     case kOptimizeFiltersForHits: {
       options.optimize_filters_for_hits = true;
       set_block_based_table_factory = true;
@@ -1018,29 +1006,4 @@ void DBTestBase::CopyFile(const std::string& source,
   ASSERT_OK(destfile->Close());
 }
 
-std::unordered_map<std::string, uint64_t> DBTestBase::GetAllSSTFiles(
-    uint64_t* total_size) {
-  std::unordered_map<std::string, uint64_t> res;
-
-  if (total_size) {
-    *total_size = 0;
-  }
-  std::vector<std::string> files;
-  env_->GetChildren(dbname_, &files);
-  for (auto& file_name : files) {
-    uint64_t number;
-    FileType type;
-    std::string file_path = dbname_ + "/" + file_name;
-    if (ParseFileName(file_name, &number, &type) && type == kTableFile) {
-      uint64_t file_size = 0;
-      env_->GetFileSize(file_path, &file_size);
-      res[file_path] = file_size;
-      if (total_size) {
-        *total_size += file_size;
-      }
-    }
-  }
-  return res;
-}
-
 }  // namespace rocksdb

db/db_test_util.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+// Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
@@ -19,7 +19,6 @@
 #endif
 
 #include <algorithm>
-#include <map>
 #include <set>
 #include <string>
 #include <thread>
@@ -28,6 +27,7 @@
 #include <vector>
 
 #include "db/db_impl.h"
+#include "db/db_test_util.h"
 #include "db/dbformat.h"
 #include "db/filename.h"
 #include "memtable/hash_linklist_rep.h"
@@ -529,7 +529,6 @@ class DBTestBase : public testing::Test {
     kEnd = 31,
     kLevelSubcompactions = 31,
     kUniversalSubcompactions = 32,
-    kBlockBasedTableWithIndexRestartInterval = 33,
   };
   int option_config_;
 
@@ -575,8 +574,6 @@ class DBTestBase : public testing::Test {
     return std::string(buf);
   }
 
-  static bool ShouldSkipOptions(int option_config, int skip_mask = kNoSkip);
-
   // Switch to a fresh database with the next option configuration to
   // test.  Return false if there are no more configurations to test.
   bool ChangeOptions(int skip_mask = kNoSkip);
@@ -753,9 +750,6 @@ class DBTestBase : public testing::Test {
 
   void CopyFile(const std::string& source, const std::string& destination,
                 uint64_t size = 0);
-
-  std::unordered_map<std::string, uint64_t> GetAllSSTFiles(
-      uint64_t* total_size = nullptr);
 };
 
 }  // namespace rocksdb

db/db_universal_compaction_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -124,7 +124,8 @@ class DelayFilterFactory : public CompactionFilterFactory {
 // Make sure we don't trigger a problem if the trigger conditon is given
 // to be 0, which is invalid.
 TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
-  Options options = CurrentOptions();
+  Options options;
+  options = CurrentOptions(options);
 
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = num_levels_;
@@ -159,7 +160,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
 }
 
 TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
-  Options options = CurrentOptions();
+  Options options;
+  options = CurrentOptions(options);
   options.compaction_style = kCompactionStyleUniversal;
   options.compaction_options_universal.size_ratio = 5;
   options.num_levels = num_levels_;
@@ -185,16 +187,14 @@ TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
   env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
                  Env::Priority::LOW);
 
+  Put("", "");
   for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
     Put(Key(num * 10), "val");
-    if (num) {
-      dbfull()->TEST_WaitForFlushMemTable();
-    }
     Put(Key(30 + num * 10), "val");
     Put(Key(60 + num * 10), "val");
+
+    dbfull()->TEST_WaitForFlushMemTable();
   }
-  Put("", "");
-  dbfull()->TEST_WaitForFlushMemTable();
 
   // Query set of non existing keys
   for (int i = 5; i < 90; i += 10) {
@@ -205,13 +205,6 @@ TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
   ASSERT_GT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
   auto prev_counter = TestGetTickerCount(options, BLOOM_FILTER_USEFUL);
 
-  // Make sure bloom filter is used for all but the last L0 file when looking
-  // up a non-existent key that's in the range of all L0 files.
-  ASSERT_EQ(Get(Key(35)), "NOT_FOUND");
-  ASSERT_EQ(prev_counter + NumTableFilesAtLevel(0) - 1,
-            TestGetTickerCount(options, BLOOM_FILTER_USEFUL));
-  prev_counter = TestGetTickerCount(options, BLOOM_FILTER_USEFUL);
-
   // Unblock compaction and wait it for happening.
   sleeping_task_low.WakeUp();
   dbfull()->TEST_WaitForCompact();
@@ -335,12 +328,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
 }
 
 TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = num_levels_;
   options.write_buffer_size = 100 << 10;     // 100KB
   options.target_file_size_base = 32 << 10;  // 32KB
   options.level0_file_num_compaction_trigger = 3;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -385,12 +379,12 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
   ChangeCompactOptions();
   Options options;
   options.create_if_missing = true;
+  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
   options.compaction_style = kCompactionStyleLevel;
   options.num_levels = 1;
   options.target_file_size_base = options.write_buffer_size;
   options.compression = kNoCompression;
   options = CurrentOptions(options);
-  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
   CreateAndReopenWithCF({"pikachu"}, options);
   ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
   Random rnd(301);
@@ -448,11 +442,12 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
 }
 
 TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.write_buffer_size = 100 << 10;     // 100KB
   options.num_levels = 7;
   options.disable_auto_compactions = true;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   // Generate 3 overlapping files
@@ -492,13 +487,14 @@ class DBTestUniversalCompactionMultiLevels
 };
 
 TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = num_levels_;
   options.write_buffer_size = 100 << 10;  // 100KB
   options.level0_file_num_compaction_trigger = 8;
   options.max_background_compactions = 3;
   options.target_file_size_base = 32 * 1024;
+  options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
   // Trigger compaction if size amplification exceeds 110%
@@ -534,7 +530,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
       });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.compaction_options_universal.allow_trivial_move = true;
   options.num_levels = 3;
@@ -542,6 +538,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
   options.level0_file_num_compaction_trigger = 3;
   options.max_background_compactions = 2;
   options.target_file_size_base = 32 * 1024;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -580,7 +577,7 @@ class DBTestUniversalCompactionParallel :
 };
 
 TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = num_levels_;
   options.write_buffer_size = 1 << 10;  // 1KB
@@ -589,6 +586,7 @@ TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
   options.max_background_flushes = 3;
   options.target_file_size_base = 1 * 1024;
   options.compaction_options_universal.max_size_amplification_percent = 110;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -645,7 +643,7 @@ INSTANTIATE_TEST_CASE_P(DBTestUniversalCompactionParallel,
                                            ::testing::Bool()));
 
 TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.write_buffer_size = 105 << 10;    // 105KB
   options.arena_block_size = 4 << 10;       // 4KB
@@ -653,6 +651,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
   options.level0_file_num_compaction_trigger = 4;
   options.num_levels = num_levels_;
   options.compaction_options_universal.compression_size_percent = -1;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -765,13 +764,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
     return;
   }
 
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.write_buffer_size = 100 << 10;     // 100KB
   options.target_file_size_base = 32 << 10;  // 32KB
   options.level0_file_num_compaction_trigger = 2;
   options.num_levels = num_levels_;
   options.compaction_options_universal.compression_size_percent = 70;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   Random rnd(301);
@@ -832,13 +832,14 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) {
   if (!Snappy_Supported()) {
     return;
   }
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.write_buffer_size = 100 << 10;     // 100KB
   options.target_file_size_base = 32 << 10;  // 32KB
   options.level0_file_num_compaction_trigger = 2;
   options.num_levels = num_levels_;
   options.compaction_options_universal.compression_size_percent = 95;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
 
   Random rnd(301);
@@ -874,7 +875,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
       });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.compaction_options_universal.allow_trivial_move = true;
   options.num_levels = 2;
@@ -882,6 +883,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
   options.level0_file_num_compaction_trigger = 3;
   options.max_background_compactions = 1;
   options.target_file_size_base = 32 * 1024;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -918,7 +920,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
 
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Options options = CurrentOptions();
+  Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.compaction_options_universal.allow_trivial_move = true;
   options.num_levels = 15;
@@ -926,6 +928,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
   options.level0_file_num_compaction_trigger = 8;
   options.max_background_compactions = 4;
   options.target_file_size_base = 64 * 1024;
+  options = CurrentOptions(options);
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -951,7 +954,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
 }
 
 TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
-  Options options = CurrentOptions();
+  Options options;
   options.db_paths.emplace_back(dbname_, 300 * 1024);
   options.db_paths.emplace_back(dbname_ + "_2", 300 * 1024);
   options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
@@ -964,6 +967,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
   options.arena_block_size = 4 << 10;
   options.level0_file_num_compaction_trigger = 2;
   options.num_levels = 1;
+  options = CurrentOptions(options);
 
   std::vector<std::string> filenames;
   env_->GetChildren(options.db_paths[1].path, &filenames);
@@ -1090,11 +1094,16 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
   for (int i = 0; i <= max_key1; i++) {
     // each value is 10K
     ASSERT_OK(Put(1, Key(i), RandomString(&rnd, 10000)));
-    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
   }
   ASSERT_OK(Flush(1));
   dbfull()->TEST_WaitForCompact();
 
+  int non_level0_num_files = 0;
+  for (int i = 1; i < options.num_levels; i++) {
+    non_level0_num_files += NumTableFilesAtLevel(i, 1);
+  }
+  ASSERT_EQ(non_level0_num_files, 0);
+
   // Stage 2: reopen with universal compaction, num_levels=4
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = 4;
@@ -1107,7 +1116,6 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
   for (int i = max_key1 + 1; i <= max_key2; i++) {
     // each value is 10K
     ASSERT_OK(Put(1, Key(i), RandomString(&rnd, 10000)));
-    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
   }
   ASSERT_OK(Flush(1));
   dbfull()->TEST_WaitForCompact();
@@ -1138,7 +1146,6 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
   for (int i = max_key2 + 1; i <= max_key3; i++) {
     // each value is 10K
     ASSERT_OK(Put(1, Key(i), RandomString(&rnd, 10000)));
-    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
   }
   ASSERT_OK(Flush(1));
   dbfull()->TEST_WaitForCompact();
@@ -1150,7 +1157,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
   if (!Snappy_Supported()) {
     return;
   }
-  Options options = CurrentOptions();
+  Options options;
   options.db_paths.emplace_back(dbname_, 500 * 1024);
   options.db_paths.emplace_back(dbname_ + "_2", 1024 * 1024 * 1024);
   options.compaction_style = kCompactionStyleUniversal;
@@ -1162,6 +1169,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
   options.num_levels = 1;
   options.memtable_factory.reset(
       new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+  options = CurrentOptions(options);
 
   std::vector<std::string> filenames;
   env_->GetChildren(options.db_paths[1].path, &filenames);

db/db_wal_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/dbformat.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/dbformat.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -465,12 +465,6 @@ class InternalKeySliceTransform : public SliceTransform {
   const SliceTransform* const transform_;
 };
 
-// Read the key of a record from a write batch.
-// if this record represent the default column family then cf_record
-// must be passed as false, otherwise it must be passed as true.
-extern bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key,
-                                       bool cf_record);
-
 // Read record from a write batch piece from input.
 // tag, column_family, key, value and blob are return values. Callers own the
 // Slice they point to.

db/dbformat_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/deletefile_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -37,7 +37,6 @@ class DeleteFileTest : public testing::Test {
   DeleteFileTest() {
     db_ = nullptr;
     env_ = Env::Default();
-    options_.delete_obsolete_files_period_micros = 0;  // always do full purge
     options_.enable_thread_tracking = true;
     options_.write_buffer_size = 1024*1024*1000;
     options_.target_file_size_base = 1024*1024*1000;

db/event_helpers.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/event_helpers.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/experimental.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/fault_injection_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2015, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/file_indexer.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/file_indexer.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/file_indexer_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/filename.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/filename.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/filename_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/flush_job.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -66,7 +66,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
                    JobContext* job_context, LogBuffer* log_buffer,
                    Directory* db_directory, Directory* output_file_directory,
                    CompressionType output_compression, Statistics* stats,
-                   EventLogger* event_logger, bool measure_io_stats)
+                   EventLogger* event_logger)
     : dbname_(dbname),
       cfd_(cfd),
       db_options_(db_options),
@@ -83,8 +83,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
       output_file_directory_(output_file_directory),
       output_compression_(output_compression),
       stats_(stats),
-      event_logger_(event_logger),
-      measure_io_stats_(measure_io_stats) {
+      event_logger_(event_logger) {
   // Update the thread status to indicate flush.
   ReportStartedFlush();
   TEST_SYNC_POINT("FlushJob::FlushJob()");
@@ -95,8 +94,7 @@ FlushJob::~FlushJob() {
 }
 
 void FlushJob::ReportStartedFlush() {
-  ThreadStatusUtil::SetColumnFamily(cfd_, cfd_->ioptions()->env,
-                                    cfd_->options()->enable_thread_tracking);
+  ThreadStatusUtil::SetColumnFamily(cfd_);
   ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH);
   ThreadStatusUtil::SetThreadOperationProperty(
       ThreadStatus::COMPACTION_JOB_ID,
@@ -122,21 +120,6 @@ void FlushJob::RecordFlushIOStats() {
 Status FlushJob::Run(FileMetaData* file_meta) {
   AutoThreadOperationStageUpdater stage_run(
       ThreadStatus::STAGE_FLUSH_RUN);
-  // I/O measurement variables
-  PerfLevel prev_perf_level = PerfLevel::kEnableTime;
-  uint64_t prev_write_nanos = 0;
-  uint64_t prev_fsync_nanos = 0;
-  uint64_t prev_range_sync_nanos = 0;
-  uint64_t prev_prepare_write_nanos = 0;
-  if (measure_io_stats_) {
-    prev_perf_level = GetPerfLevel();
-    SetPerfLevel(PerfLevel::kEnableTime);
-    prev_write_nanos = IOSTATS(write_nanos);
-    prev_fsync_nanos = IOSTATS(fsync_nanos);
-    prev_range_sync_nanos = IOSTATS(range_sync_nanos);
-    prev_prepare_write_nanos = IOSTATS(prepare_write_nanos);
-  }
-
   // Save the contents of the earliest memtable as a new Table
   FileMetaData meta;
   autovector<MemTable*> mems;
@@ -196,18 +179,6 @@ Status FlushJob::Run(FileMetaData* file_meta) {
   }
   stream.EndArray();
 
-  if (measure_io_stats_) {
-    if (prev_perf_level != PerfLevel::kEnableTime) {
-      SetPerfLevel(prev_perf_level);
-    }
-    stream << "file_write_nanos" << (IOSTATS(write_nanos) - prev_write_nanos);
-    stream << "file_range_sync_nanos"
-           << (IOSTATS(range_sync_nanos) - prev_range_sync_nanos);
-    stream << "file_fsync_nanos" << (IOSTATS(fsync_nanos) - prev_fsync_nanos);
-    stream << "file_prepare_write_nanos"
-           << (IOSTATS(prepare_write_nanos) - prev_prepare_write_nanos);
-  }
-
   return s;
 }
 
@@ -262,15 +233,14 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
 
       TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:output_compression",
                                &output_compression_);
-      s = BuildTable(
-          dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
-          cfd_->table_cache(), iter.get(), meta, cfd_->internal_comparator(),
-          cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
-          cfd_->GetName(), existing_snapshots_,
-          earliest_write_conflict_snapshot_, output_compression_,
-          cfd_->ioptions()->compression_opts,
-          mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
-          Env::IO_HIGH, &table_properties_, 0 /* level */);
+      s = BuildTable(dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
+                     cfd_->table_cache(), iter.get(), meta,
+                     cfd_->internal_comparator(),
+                     cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
+                     existing_snapshots_, earliest_write_conflict_snapshot_,
+                     output_compression_, cfd_->ioptions()->compression_opts,
+                     mutable_cf_options_.paranoid_file_checks,
+                     cfd_->internal_stats(), Env::IO_HIGH, &table_properties_);
       info.table_properties = table_properties_;
       LogFlush(db_options_.info_log);
     }
@@ -300,7 +270,6 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
     if (!db_options_.disableDataSync && output_file_directory_ != nullptr) {
       output_file_directory_->Fsync();
     }
-    TEST_SYNC_POINT("FlushJob::WriteLevel0Table");
     db_mutex_->Lock();
   }
   base->Unref();

db/flush_job.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -62,7 +62,7 @@ class FlushJob {
            JobContext* job_context, LogBuffer* log_buffer,
            Directory* db_directory, Directory* output_file_directory,
            CompressionType output_compression, Statistics* stats,
-           EventLogger* event_logger, bool measure_io_stats);
+           EventLogger* event_logger);
 
   ~FlushJob();
 
@@ -93,7 +93,6 @@ class FlushJob {
   Statistics* stats_;
   EventLogger* event_logger_;
   TableProperties table_properties_;
-  bool measure_io_stats_;
 };
 
 }  // namespace rocksdb

db/flush_job_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -93,7 +93,7 @@ TEST_F(FlushJobTest, Empty) {
                      db_options_, *cfd->GetLatestMutableCFOptions(),
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
                      {}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
-                     nullptr, kNoCompression, nullptr, &event_logger, false);
+                     nullptr, kNoCompression, nullptr, &event_logger);
   ASSERT_OK(flush_job.Run());
   job_context.Clean();
 }
@@ -132,7 +132,7 @@ TEST_F(FlushJobTest, NonEmpty) {
                      db_options_, *cfd->GetLatestMutableCFOptions(),
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
                      {}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
-                     nullptr, kNoCompression, nullptr, &event_logger, true);
+                     nullptr, kNoCompression, nullptr, &event_logger);
   FileMetaData fd;
   mutex_.Lock();
   ASSERT_OK(flush_job.Run(&fd));
@@ -192,11 +192,11 @@ TEST_F(FlushJobTest, Snapshots) {
   }
 
   EventLogger event_logger(db_options_.info_log.get());
-  FlushJob flush_job(
-      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
-      *cfd->GetLatestMutableCFOptions(), env_options_, versions_.get(), &mutex_,
-      &shutting_down_, snapshots, kMaxSequenceNumber, &job_context, nullptr,
-      nullptr, nullptr, kNoCompression, nullptr, &event_logger, true);
+  FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
+                     db_options_, *cfd->GetLatestMutableCFOptions(),
+                     env_options_, versions_.get(), &mutex_, &shutting_down_,
+                     snapshots, kMaxSequenceNumber, &job_context, nullptr,
+                     nullptr, nullptr, kNoCompression, nullptr, &event_logger);
   mutex_.Lock();
   ASSERT_OK(flush_job.Run());
   mutex_.Unlock();

db/flush_scheduler.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/flush_scheduler.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/forward_iterator.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -10,16 +10,15 @@
 #include <string>
 #include <utility>
 
-#include "db/column_family.h"
+#include "db/job_context.h"
 #include "db/db_impl.h"
 #include "db/db_iter.h"
-#include "db/dbformat.h"
-#include "db/job_context.h"
+#include "db/column_family.h"
 #include "rocksdb/env.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/slice_transform.h"
 #include "table/merger.h"
-#include "util/string_util.h"
+#include "db/dbformat.h"
 #include "util/sync_point.h"
 
 namespace rocksdb {
@@ -472,15 +471,6 @@ Status ForwardIterator::status() const {
   return immutable_status_;
 }
 
-Status ForwardIterator::GetProperty(std::string prop_name, std::string* prop) {
-  assert(prop != nullptr);
-  if (prop_name == "rocksdb.iterator.super-version-number") {
-    *prop = ToString(sv_->version_number);
-    return Status::OK();
-  }
-  return Status::InvalidArgument();
-}
-
 void ForwardIterator::RebuildIterators(bool refresh_sv) {
   // Clean up
   Cleanup(refresh_sv);

db/forward_iterator.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -71,8 +71,6 @@ class ForwardIterator : public InternalIterator {
   virtual Slice key() const override;
   virtual Slice value() const override;
   virtual Status status() const override;
-  virtual Status GetProperty(std::string prop_name, std::string* prop) override;
-
   bool TEST_CheckDeletedIters(int* deleted_iters, int* num_iters);
 
  private:

db/forward_iterator_bench.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/inlineskiplist.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional
 //  grant of patent rights can be found in the PATENTS file in the same

db/inlineskiplist_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/internal_stats.cc

@@ -15,7 +15,6 @@
 #include <inttypes.h>
 #include <string>
 #include <algorithm>
-#include <utility>
 #include <vector>
 #include "db/column_family.h"
 
@@ -82,21 +81,7 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
            stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count,
            num_input_records.c_str(), num_dropped_records.c_str());
 }
-
-// Assumes that trailing numbers represent an optional argument. This requires
-// property names to not end with numbers.
-std::pair<Slice, Slice> GetPropertyNameAndArg(const Slice& property) {
-  Slice name = property, arg = property;
-  size_t sfx_len = 0;
-  while (sfx_len < property.size() &&
-         isdigit(property[property.size() - sfx_len - 1])) {
-    ++sfx_len;
-  }
-  name.remove_suffix(sfx_len);
-  arg.remove_prefix(property.size() - sfx_len);
-  return {name, arg};
 }
-}  // anonymous namespace
 
 static const std::string rocksdb_prefix = "rocksdb.";
 
@@ -114,8 +99,9 @@ static const std::string compaction_pending = "compaction-pending";
 static const std::string background_errors = "background-errors";
 static const std::string cur_size_active_mem_table =
                           "cur-size-active-mem-table";
-static const std::string cur_size_all_mem_tables = "cur-size-all-mem-tables";
-static const std::string size_all_mem_tables = "size-all-mem-tables";
+static const std::string cur_size_unflushed_mem_tables =
+    "cur-size-all-mem-tables";
+static const std::string cur_size_all_mem_tables = "size-all-mem-tables";
 static const std::string num_entries_active_mem_table =
                           "num-entries-active-mem-table";
 static const std::string num_entries_imm_mem_tables =
@@ -132,8 +118,6 @@ static const std::string is_file_deletions_enabled =
 static const std::string num_snapshots = "num-snapshots";
 static const std::string oldest_snapshot_time = "oldest-snapshot-time";
 static const std::string num_live_versions = "num-live-versions";
-static const std::string current_version_number =
-    "current-super-version-number";
 static const std::string estimate_live_data_size = "estimate-live-data-size";
 static const std::string base_level = "base-level";
 static const std::string total_sst_files_size = "total-sst-files-size";
@@ -152,11 +136,8 @@ const std::string DB::Properties::kStats = rocksdb_prefix + allstats;
 const std::string DB::Properties::kSSTables = rocksdb_prefix + sstables;
 const std::string DB::Properties::kCFStats = rocksdb_prefix + cfstats;
 const std::string DB::Properties::kDBStats = rocksdb_prefix + dbstats;
-const std::string DB::Properties::kLevelStats = rocksdb_prefix + levelstats;
 const std::string DB::Properties::kNumImmutableMemTable =
                       rocksdb_prefix + num_immutable_mem_table;
-const std::string DB::Properties::kNumImmutableMemTableFlushed =
-    rocksdb_prefix + num_immutable_mem_table_flushed;
 const std::string DB::Properties::kMemTableFlushPending =
                       rocksdb_prefix + mem_table_flush_pending;
 const std::string DB::Properties::kCompactionPending =
@@ -170,9 +151,9 @@ const std::string DB::Properties::kBackgroundErrors =
 const std::string DB::Properties::kCurSizeActiveMemTable =
                       rocksdb_prefix + cur_size_active_mem_table;
 const std::string DB::Properties::kCurSizeAllMemTables =
-    rocksdb_prefix + cur_size_all_mem_tables;
+    rocksdb_prefix + cur_size_unflushed_mem_tables;
 const std::string DB::Properties::kSizeAllMemTables =
-    rocksdb_prefix + size_all_mem_tables;
+    rocksdb_prefix + cur_size_all_mem_tables;
 const std::string DB::Properties::kNumEntriesActiveMemTable =
                       rocksdb_prefix + num_entries_active_mem_table;
 const std::string DB::Properties::kNumEntriesImmMemTables =
@@ -193,13 +174,10 @@ const std::string DB::Properties::kOldestSnapshotTime =
                       rocksdb_prefix + oldest_snapshot_time;
 const std::string DB::Properties::kNumLiveVersions =
                       rocksdb_prefix + num_live_versions;
-const std::string DB::Properties::kCurrentSuperVersionNumber =
-    rocksdb_prefix + current_version_number;
 const std::string DB::Properties::kEstimateLiveDataSize =
                       rocksdb_prefix + estimate_live_data_size;
 const std::string DB::Properties::kTotalSstFilesSize =
                       rocksdb_prefix + total_sst_files_size;
-const std::string DB::Properties::kBaseLevel = rocksdb_prefix + base_level;
 const std::string DB::Properties::kEstimatePendingCompactionBytes =
     rocksdb_prefix + estimate_pending_comp_bytes;
 const std::string DB::Properties::kAggregatedTableProperties =
@@ -207,368 +185,294 @@ const std::string DB::Properties::kAggregatedTableProperties =
 const std::string DB::Properties::kAggregatedTablePropertiesAtLevel =
     rocksdb_prefix + aggregated_table_properties_at_level;
 
-const std::unordered_map<std::string,
-                         DBPropertyInfo> InternalStats::ppt_name_to_info = {
-    {DB::Properties::kNumFilesAtLevelPrefix,
-     {false, &InternalStats::HandleNumFilesAtLevel, nullptr}},
-    {DB::Properties::kLevelStats,
-     {false, &InternalStats::HandleLevelStats, nullptr}},
-    {DB::Properties::kStats, {false, &InternalStats::HandleStats, nullptr}},
-    {DB::Properties::kCFStats, {false, &InternalStats::HandleCFStats, nullptr}},
-    {DB::Properties::kDBStats, {false, &InternalStats::HandleDBStats, nullptr}},
-    {DB::Properties::kSSTables,
-     {false, &InternalStats::HandleSsTables, nullptr}},
-    {DB::Properties::kAggregatedTableProperties,
-     {false, &InternalStats::HandleAggregatedTableProperties, nullptr}},
-    {DB::Properties::kAggregatedTablePropertiesAtLevel,
-     {false, &InternalStats::HandleAggregatedTablePropertiesAtLevel, nullptr}},
-    {DB::Properties::kNumImmutableMemTable,
-     {false, nullptr, &InternalStats::HandleNumImmutableMemTable}},
-    {DB::Properties::kNumImmutableMemTableFlushed,
-     {false, nullptr, &InternalStats::HandleNumImmutableMemTableFlushed}},
-    {DB::Properties::kMemTableFlushPending,
-     {false, nullptr, &InternalStats::HandleMemTableFlushPending}},
-    {DB::Properties::kCompactionPending,
-     {false, nullptr, &InternalStats::HandleCompactionPending}},
-    {DB::Properties::kBackgroundErrors,
-     {false, nullptr, &InternalStats::HandleBackgroundErrors}},
-    {DB::Properties::kCurSizeActiveMemTable,
-     {false, nullptr, &InternalStats::HandleCurSizeActiveMemTable}},
-    {DB::Properties::kCurSizeAllMemTables,
-     {false, nullptr, &InternalStats::HandleCurSizeAllMemTables}},
-    {DB::Properties::kSizeAllMemTables,
-     {false, nullptr, &InternalStats::HandleSizeAllMemTables}},
-    {DB::Properties::kNumEntriesActiveMemTable,
-     {false, nullptr, &InternalStats::HandleNumEntriesActiveMemTable}},
-    {DB::Properties::kNumEntriesImmMemTables,
-     {false, nullptr, &InternalStats::HandleNumEntriesImmMemTables}},
-    {DB::Properties::kNumDeletesActiveMemTable,
-     {false, nullptr, &InternalStats::HandleNumDeletesActiveMemTable}},
-    {DB::Properties::kNumDeletesImmMemTables,
-     {false, nullptr, &InternalStats::HandleNumDeletesImmMemTables}},
-    {DB::Properties::kEstimateNumKeys,
-     {false, nullptr, &InternalStats::HandleEstimateNumKeys}},
-    {DB::Properties::kEstimateTableReadersMem,
-     {true, nullptr, &InternalStats::HandleEstimateTableReadersMem}},
-    {DB::Properties::kIsFileDeletionsEnabled,
-     {false, nullptr, &InternalStats::HandleIsFileDeletionsEnabled}},
-    {DB::Properties::kNumSnapshots,
-     {false, nullptr, &InternalStats::HandleNumSnapshots}},
-    {DB::Properties::kOldestSnapshotTime,
-     {false, nullptr, &InternalStats::HandleOldestSnapshotTime}},
-    {DB::Properties::kNumLiveVersions,
-     {false, nullptr, &InternalStats::HandleNumLiveVersions}},
-    {DB::Properties::kCurrentSuperVersionNumber,
-     {false, nullptr, &InternalStats::HandleCurrentSuperVersionNumber}},
-    {DB::Properties::kEstimateLiveDataSize,
-     {true, nullptr, &InternalStats::HandleEstimateLiveDataSize}},
-    {DB::Properties::kBaseLevel,
-     {false, nullptr, &InternalStats::HandleBaseLevel}},
-    {DB::Properties::kTotalSstFilesSize,
-     {false, nullptr, &InternalStats::HandleTotalSstFilesSize}},
-    {DB::Properties::kEstimatePendingCompactionBytes,
-     {false, nullptr, &InternalStats::HandleEstimatePendingCompactionBytes}},
-    {DB::Properties::kNumRunningFlushes,
-     {false, nullptr, &InternalStats::HandleNumRunningFlushes}},
-    {DB::Properties::kNumRunningCompactions,
-     {false, nullptr, &InternalStats::HandleNumRunningCompactions}},
-};
-
-const DBPropertyInfo* GetPropertyInfo(const Slice& property) {
-  std::string ppt_name = GetPropertyNameAndArg(property).first.ToString();
-  auto ppt_info_iter = InternalStats::ppt_name_to_info.find(ppt_name);
-  if (ppt_info_iter == InternalStats::ppt_name_to_info.end()) {
-    return nullptr;
+DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
+                               bool* need_out_of_mutex) {
+  assert(is_int_property != nullptr);
+  assert(need_out_of_mutex != nullptr);
+  Slice in = property;
+  Slice prefix(rocksdb_prefix);
+  *need_out_of_mutex = false;
+  *is_int_property = false;
+  if (!in.starts_with(prefix)) {
+    return kUnknown;
   }
-  return &ppt_info_iter->second;
+  in.remove_prefix(prefix.size());
+
+  if (in.starts_with(num_files_at_level_prefix)) {
+    return kNumFilesAtLevel;
+  } else if (in == levelstats) {
+    return kLevelStats;
+  } else if (in == allstats) {
+    return kStats;
+  } else if (in == cfstats) {
+    return kCFStats;
+  } else if (in == dbstats) {
+    return kDBStats;
+  } else if (in == sstables) {
+    return kSsTables;
+  } else if (in == aggregated_table_properties) {
+    return kAggregatedTableProperties;
+  } else if (in.starts_with(aggregated_table_properties_at_level)) {
+    return kAggregatedTablePropertiesAtLevel;
+  }
+
+  *is_int_property = true;
+  if (in == num_immutable_mem_table) {
+    return kNumImmutableMemTable;
+  } else if (in == num_immutable_mem_table_flushed) {
+    return kNumImmutableMemTableFlushed;
+  } else if (in == mem_table_flush_pending) {
+    return kMemtableFlushPending;
+  } else if (in == compaction_pending) {
+    return kCompactionPending;
+  } else if (in == background_errors) {
+    return kBackgroundErrors;
+  } else if (in == cur_size_active_mem_table) {
+    return kCurSizeActiveMemTable;
+  } else if (in == cur_size_unflushed_mem_tables) {
+    return kCurSizeAllMemTables;
+  } else if (in == cur_size_all_mem_tables) {
+    return kSizeAllMemTables;
+  } else if (in == num_entries_active_mem_table) {
+    return kNumEntriesInMutableMemtable;
+  } else if (in == num_entries_imm_mem_tables) {
+    return kNumEntriesInImmutableMemtable;
+  } else if (in == num_deletes_active_mem_table) {
+    return kNumDeletesInMutableMemtable;
+  } else if (in == num_deletes_imm_mem_tables) {
+    return kNumDeletesInImmutableMemtable;
+  } else if (in == estimate_num_keys) {
+    return kEstimatedNumKeys;
+  } else if (in == estimate_table_readers_mem) {
+    *need_out_of_mutex = true;
+    return kEstimatedUsageByTableReaders;
+  } else if (in == is_file_deletions_enabled) {
+    return kIsFileDeletionEnabled;
+  } else if (in == num_snapshots) {
+    return kNumSnapshots;
+  } else if (in == oldest_snapshot_time) {
+    return kOldestSnapshotTime;
+  } else if (in == num_live_versions) {
+    return kNumLiveVersions;
+  } else if (in == estimate_live_data_size) {
+    *need_out_of_mutex = true;
+    return kEstimateLiveDataSize;
+  } else if (in == base_level) {
+    return kBaseLevel;
+  } else if (in == total_sst_files_size) {
+    return kTotalSstFilesSize;
+  } else if (in == estimate_pending_comp_bytes) {
+    return kEstimatePendingCompactionBytes;
+  } else if (in == num_running_flushes) {
+    return kNumRunningFlushes;
+  } else if (in == num_running_compactions) {
+    return kNumRunningCompactions;
+  }
+  return kUnknown;
 }
 
-bool InternalStats::GetStringProperty(const DBPropertyInfo& property_info,
+bool InternalStats::GetIntPropertyOutOfMutex(DBPropertyType property_type,
+                                             Version* version,
+                                             uint64_t* value) const {
+  assert(value != nullptr);
+  const auto* vstorage = cfd_->current()->storage_info();
+
+  switch (property_type) {
+    case kEstimatedUsageByTableReaders:
+      *value = (version == nullptr) ?
+        0 : version->GetMemoryUsageByTableReaders();
+      return true;
+    case kEstimateLiveDataSize:
+      *value = vstorage->EstimateLiveDataSize();
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool InternalStats::GetStringProperty(DBPropertyType property_type,
                                       const Slice& property,
                                       std::string* value) {
   assert(value != nullptr);
-  assert(property_info.handle_string != nullptr);
-  Slice arg = GetPropertyNameAndArg(property).second;
-  return (this->*(property_info.handle_string))(value, arg);
-}
-
-bool InternalStats::GetIntProperty(const DBPropertyInfo& property_info,
-                                   uint64_t* value, DBImpl* db) {
-  assert(value != nullptr);
-  assert(property_info.handle_int != nullptr &&
-         !property_info.need_out_of_mutex);
-  db->mutex_.AssertHeld();
-  return (this->*(property_info.handle_int))(value, db, nullptr /* version */);
-}
-
-bool InternalStats::GetIntPropertyOutOfMutex(
-    const DBPropertyInfo& property_info, Version* version, uint64_t* value) {
-  assert(value != nullptr);
-  assert(property_info.handle_int != nullptr &&
-         property_info.need_out_of_mutex);
-  return (this->*(property_info.handle_int))(value, nullptr /* db */, version);
-}
-
-bool InternalStats::HandleNumFilesAtLevel(std::string* value, Slice suffix) {
-  uint64_t level;
-  const auto* vstorage = cfd_->current()->storage_info();
-  bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
-  if (!ok || static_cast<int>(level) >= number_levels_) {
-    return false;
-  } else {
-    char buf[100];
-    snprintf(buf, sizeof(buf), "%d",
-             vstorage->NumLevelFiles(static_cast<int>(level)));
-    *value = buf;
-    return true;
-  }
-}
-
-bool InternalStats::HandleLevelStats(std::string* value, Slice suffix) {
-  char buf[1000];
-  const auto* vstorage = cfd_->current()->storage_info();
-  snprintf(buf, sizeof(buf),
-           "Level Files Size(MB)\n"
-           "--------------------\n");
-  value->append(buf);
-
-  for (int level = 0; level < number_levels_; level++) {
-    snprintf(buf, sizeof(buf), "%3d %8d %8.0f\n", level,
-             vstorage->NumLevelFiles(level),
-             vstorage->NumLevelBytes(level) / kMB);
-    value->append(buf);
-  }
-  return true;
-}
-
-bool InternalStats::HandleStats(std::string* value, Slice suffix) {
-  if (!HandleCFStats(value, suffix)) {
-    return false;
-  }
-  if (!HandleDBStats(value, suffix)) {
-    return false;
-  }
-  return true;
-}
-
-bool InternalStats::HandleCFStats(std::string* value, Slice suffix) {
-  DumpCFStats(value);
-  return true;
-}
-
-bool InternalStats::HandleDBStats(std::string* value, Slice suffix) {
-  DumpDBStats(value);
-  return true;
-}
-
-bool InternalStats::HandleSsTables(std::string* value, Slice suffix) {
   auto* current = cfd_->current();
-  *value = current->DebugString();
-  return true;
-}
+  const auto* vstorage = current->storage_info();
+  Slice in = property;
 
-bool InternalStats::HandleAggregatedTableProperties(std::string* value,
-                                                    Slice suffix) {
-  std::shared_ptr<const TableProperties> tp;
-  auto s = cfd_->current()->GetAggregatedTableProperties(&tp);
-  if (!s.ok()) {
-    return false;
+  switch (property_type) {
+    case kNumFilesAtLevel: {
+      in.remove_prefix(strlen("rocksdb.num-files-at-level"));
+      uint64_t level;
+      bool ok = ConsumeDecimalNumber(&in, &level) && in.empty();
+      if (!ok || (int)level >= number_levels_) {
+        return false;
+      } else {
+        char buf[100];
+        snprintf(buf, sizeof(buf), "%d",
+                 vstorage->NumLevelFiles(static_cast<int>(level)));
+        *value = buf;
+        return true;
+      }
+    }
+    case kLevelStats: {
+      char buf[1000];
+      snprintf(buf, sizeof(buf),
+               "Level Files Size(MB)\n"
+               "--------------------\n");
+      value->append(buf);
+
+      for (int level = 0; level < number_levels_; level++) {
+        snprintf(buf, sizeof(buf), "%3d %8d %8.0f\n", level,
+                 vstorage->NumLevelFiles(level),
+                 vstorage->NumLevelBytes(level) / kMB);
+        value->append(buf);
+      }
+      return true;
+    }
+    case kStats: {
+      if (!GetStringProperty(kCFStats, DB::Properties::kCFStats, value)) {
+        return false;
+      }
+      if (!GetStringProperty(kDBStats, DB::Properties::kDBStats, value)) {
+        return false;
+      }
+      return true;
+    }
+    case kCFStats: {
+      DumpCFStats(value);
+      return true;
+    }
+    case kDBStats: {
+      DumpDBStats(value);
+      return true;
+    }
+    case kSsTables:
+      *value = current->DebugString();
+      return true;
+    case kAggregatedTableProperties: {
+      std::shared_ptr<const TableProperties> tp;
+      auto s = cfd_->current()->GetAggregatedTableProperties(&tp);
+      if (!s.ok()) {
+        return false;
+      }
+      *value = tp->ToString();
+      return true;
+    }
+    case kAggregatedTablePropertiesAtLevel: {
+      in.remove_prefix(
+          DB::Properties::kAggregatedTablePropertiesAtLevel.length());
+      uint64_t level;
+      bool ok = ConsumeDecimalNumber(&in, &level) && in.empty();
+      if (!ok || static_cast<int>(level) >= number_levels_) {
+        return false;
+      }
+      std::shared_ptr<const TableProperties> tp;
+      auto s = cfd_->current()->GetAggregatedTableProperties(
+          &tp, static_cast<int>(level));
+      if (!s.ok()) {
+        return false;
+      }
+      *value = tp->ToString();
+      return true;
+    }
+    default:
+      return false;
   }
-  *value = tp->ToString();
-  return true;
 }
 
-bool InternalStats::HandleAggregatedTablePropertiesAtLevel(std::string* value,
-                                                           Slice suffix) {
-  uint64_t level;
-  bool ok = ConsumeDecimalNumber(&suffix, &level) && suffix.empty();
-  if (!ok || static_cast<int>(level) >= number_levels_) {
-    return false;
+bool InternalStats::GetIntProperty(DBPropertyType property_type,
+                                   uint64_t* value, DBImpl* db) const {
+  db->mutex_.AssertHeld();
+  const auto* vstorage = cfd_->current()->storage_info();
+
+  switch (property_type) {
+    case kNumImmutableMemTable:
+      *value = cfd_->imm()->NumNotFlushed();
+      return true;
+    case kNumImmutableMemTableFlushed:
+      *value = cfd_->imm()->NumFlushed();
+      return true;
+    case kMemtableFlushPending:
+      // Return number of mem tables that are ready to flush (made immutable)
+      *value = (cfd_->imm()->IsFlushPending() ? 1 : 0);
+      return true;
+    case kNumRunningFlushes:
+      *value = db->num_running_flushes();
+      return true;
+    case kCompactionPending:
+      // 1 if the system already determines at least one compaction is needed.
+      // 0 otherwise,
+      *value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0);
+      return true;
+    case kNumRunningCompactions:
+      *value = db->num_running_compactions_;
+      return true;
+    case kBackgroundErrors:
+      // Accumulated number of  errors in background flushes or compactions.
+      *value = GetBackgroundErrorCount();
+      return true;
+    case kCurSizeActiveMemTable:
+      // Current size of the active memtable
+      *value = cfd_->mem()->ApproximateMemoryUsage();
+      return true;
+    case kCurSizeAllMemTables:
+      // Current size of the active memtable + immutable memtables
+      *value = cfd_->mem()->ApproximateMemoryUsage() +
+               cfd_->imm()->ApproximateUnflushedMemTablesMemoryUsage();
+      return true;
+    case kSizeAllMemTables:
+      *value = cfd_->mem()->ApproximateMemoryUsage() +
+               cfd_->imm()->ApproximateMemoryUsage();
+      return true;
+    case kNumEntriesInMutableMemtable:
+      // Current number of entires in the active memtable
+      *value = cfd_->mem()->num_entries();
+      return true;
+    case kNumEntriesInImmutableMemtable:
+      // Current number of entries in the immutable memtables
+      *value = cfd_->imm()->current()->GetTotalNumEntries();
+      return true;
+    case kNumDeletesInMutableMemtable:
+      // Current number of entires in the active memtable
+      *value = cfd_->mem()->num_deletes();
+      return true;
+    case kNumDeletesInImmutableMemtable:
+      // Current number of entries in the immutable memtables
+      *value = cfd_->imm()->current()->GetTotalNumDeletes();
+      return true;
+    case kEstimatedNumKeys:
+      // Estimate number of entries in the column family:
+      // Use estimated entries in tables + total entries in memtables.
+      *value = cfd_->mem()->num_entries() +
+               cfd_->imm()->current()->GetTotalNumEntries() -
+               (cfd_->mem()->num_deletes() +
+                cfd_->imm()->current()->GetTotalNumDeletes()) *
+                   2 +
+               vstorage->GetEstimatedActiveKeys();
+      return true;
+    case kNumSnapshots:
+      *value = db->snapshots().count();
+      return true;
+    case kOldestSnapshotTime:
+      *value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotTime());
+      return true;
+    case kNumLiveVersions:
+      *value = cfd_->GetNumLiveVersions();
+      return true;
+    case kIsFileDeletionEnabled:
+      *value = db->IsFileDeletionsEnabled();
+      return true;
+    case kBaseLevel:
+      *value = vstorage->base_level();
+      return true;
+    case kTotalSstFilesSize:
+      *value = cfd_->GetTotalSstFilesSize();
+      return true;
+    case kEstimatePendingCompactionBytes:
+      *value = vstorage->estimated_compaction_needed_bytes();
+      return true;
+    default:
+      return false;
   }
-  std::shared_ptr<const TableProperties> tp;
-  auto s = cfd_->current()->GetAggregatedTableProperties(
-      &tp, static_cast<int>(level));
-  if (!s.ok()) {
-    return false;
-  }
-  *value = tp->ToString();
-  return true;
-}
-
-bool InternalStats::HandleNumImmutableMemTable(uint64_t* value, DBImpl* db,
-                                               Version* version) {
-  *value = cfd_->imm()->NumNotFlushed();
-  return true;
-}
-
-bool InternalStats::HandleNumImmutableMemTableFlushed(uint64_t* value,
-                                                      DBImpl* db,
-                                                      Version* version) {
-  *value = cfd_->imm()->NumFlushed();
-  return true;
-}
-
-bool InternalStats::HandleMemTableFlushPending(uint64_t* value, DBImpl* db,
-                                               Version* version) {
-  // Return number of mem tables that are ready to flush (made immutable)
-  *value = (cfd_->imm()->IsFlushPending() ? 1 : 0);
-  return true;
-}
-
-bool InternalStats::HandleNumRunningFlushes(uint64_t* value, DBImpl* db,
-                                            Version* version) {
-  *value = db->num_running_flushes();
-  return true;
-}
-
-bool InternalStats::HandleCompactionPending(uint64_t* value, DBImpl* db,
-                                            Version* version) {
-  // 1 if the system already determines at least one compaction is needed.
-  // 0 otherwise,
-  const auto* vstorage = cfd_->current()->storage_info();
-  *value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0);
-  return true;
-}
-
-bool InternalStats::HandleNumRunningCompactions(uint64_t* value, DBImpl* db,
-                                                Version* version) {
-  *value = db->num_running_compactions_;
-  return true;
-}
-
-bool InternalStats::HandleBackgroundErrors(uint64_t* value, DBImpl* db,
-                                           Version* version) {
-  // Accumulated number of  errors in background flushes or compactions.
-  *value = GetBackgroundErrorCount();
-  return true;
-}
-
-bool InternalStats::HandleCurSizeActiveMemTable(uint64_t* value, DBImpl* db,
-                                                Version* version) {
-  // Current size of the active memtable
-  *value = cfd_->mem()->ApproximateMemoryUsage();
-  return true;
-}
-
-bool InternalStats::HandleCurSizeAllMemTables(uint64_t* value, DBImpl* db,
-                                              Version* version) {
-  // Current size of the active memtable + immutable memtables
-  *value = cfd_->mem()->ApproximateMemoryUsage() +
-           cfd_->imm()->ApproximateUnflushedMemTablesMemoryUsage();
-  return true;
-}
-
-bool InternalStats::HandleSizeAllMemTables(uint64_t* value, DBImpl* db,
-                                           Version* version) {
-  *value = cfd_->mem()->ApproximateMemoryUsage() +
-           cfd_->imm()->ApproximateMemoryUsage();
-  return true;
-}
-
-bool InternalStats::HandleNumEntriesActiveMemTable(uint64_t* value, DBImpl* db,
-                                                   Version* version) {
-  // Current number of entires in the active memtable
-  *value = cfd_->mem()->num_entries();
-  return true;
-}
-
-bool InternalStats::HandleNumEntriesImmMemTables(uint64_t* value, DBImpl* db,
-                                                 Version* version) {
-  // Current number of entries in the immutable memtables
-  *value = cfd_->imm()->current()->GetTotalNumEntries();
-  return true;
-}
-
-bool InternalStats::HandleNumDeletesActiveMemTable(uint64_t* value, DBImpl* db,
-                                                   Version* version) {
-  // Current number of entires in the active memtable
-  *value = cfd_->mem()->num_deletes();
-  return true;
-}
-
-bool InternalStats::HandleNumDeletesImmMemTables(uint64_t* value, DBImpl* db,
-                                                 Version* version) {
-  // Current number of entries in the immutable memtables
-  *value = cfd_->imm()->current()->GetTotalNumDeletes();
-  return true;
-}
-
-bool InternalStats::HandleEstimateNumKeys(uint64_t* value, DBImpl* db,
-                                          Version* version) {
-  // Estimate number of entries in the column family:
-  // Use estimated entries in tables + total entries in memtables.
-  const auto* vstorage = cfd_->current()->storage_info();
-  *value = cfd_->mem()->num_entries() +
-           cfd_->imm()->current()->GetTotalNumEntries() -
-           (cfd_->mem()->num_deletes() +
-            cfd_->imm()->current()->GetTotalNumDeletes()) *
-               2 +
-           vstorage->GetEstimatedActiveKeys();
-  return true;
-}
-
-bool InternalStats::HandleNumSnapshots(uint64_t* value, DBImpl* db,
-                                       Version* version) {
-  *value = db->snapshots().count();
-  return true;
-}
-
-bool InternalStats::HandleOldestSnapshotTime(uint64_t* value, DBImpl* db,
-                                             Version* version) {
-  *value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotTime());
-  return true;
-}
-
-bool InternalStats::HandleNumLiveVersions(uint64_t* value, DBImpl* db,
-                                          Version* version) {
-  *value = cfd_->GetNumLiveVersions();
-  return true;
-}
-
-bool InternalStats::HandleCurrentSuperVersionNumber(uint64_t* value, DBImpl* db,
-                                                    Version* version) {
-  *value = cfd_->GetSuperVersionNumber();
-  return true;
-}
-
-bool InternalStats::HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db,
-                                                 Version* version) {
-  *value = db->IsFileDeletionsEnabled();
-  return true;
-}
-
-bool InternalStats::HandleBaseLevel(uint64_t* value, DBImpl* db,
-                                    Version* version) {
-  const auto* vstorage = cfd_->current()->storage_info();
-  *value = vstorage->base_level();
-  return true;
-}
-
-bool InternalStats::HandleTotalSstFilesSize(uint64_t* value, DBImpl* db,
-                                            Version* version) {
-  *value = cfd_->GetTotalSstFilesSize();
-  return true;
-}
-
-bool InternalStats::HandleEstimatePendingCompactionBytes(uint64_t* value,
-                                                         DBImpl* db,
-                                                         Version* version) {
-  const auto* vstorage = cfd_->current()->storage_info();
-  *value = vstorage->estimated_compaction_needed_bytes();
-  return true;
-}
-
-bool InternalStats::HandleEstimateTableReadersMem(uint64_t* value, DBImpl* db,
-                                                  Version* version) {
-  *value = (version == nullptr) ? 0 : version->GetMemoryUsageByTableReaders();
-  return true;
-}
-
-bool InternalStats::HandleEstimateLiveDataSize(uint64_t* value, DBImpl* db,
-                                               Version* version) {
-  const auto* vstorage = cfd_->current()->storage_info();
-  *value = vstorage->EstimateLiveDataSize();
-  return true;
 }
 
 void InternalStats::DumpDBStats(std::string* value) {
@@ -853,7 +757,10 @@ void InternalStats::DumpCFStats(std::string* value) {
 
 #else
 
-const DBPropertyInfo* GetPropertyInfo(const Slice& property) { return nullptr; }
+DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
+                               bool* need_out_of_mutex) {
+  return kUnknown;
+}
 
 #endif  // !ROCKSDB_LITE
 

db/internal_stats.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -21,29 +21,63 @@ namespace rocksdb {
 class MemTableList;
 class DBImpl;
 
-// Config for retrieving a property's value.
-struct DBPropertyInfo {
-  bool need_out_of_mutex;
-
-  // gcc had an internal error for initializing union of pointer-to-member-
-  // functions. Workaround is to populate exactly one of the following function
-  // pointers with a non-nullptr value.
-
-  // @param value Value-result argument for storing the property's string value
-  // @param suffix Argument portion of the property. For example, suffix would
-  //      be "5" for the property "rocksdb.num-files-at-level5". So far, only
-  //      certain string properties take an argument.
-  bool (InternalStats::*handle_string)(std::string* value, Slice suffix);
-
-  // @param value Value-result argument for storing the property's uint64 value
-  // @param db Many of the int properties rely on DBImpl methods.
-  // @param version Version is needed in case the property is retrieved without
-  //      holding db mutex, which is only supported for int properties.
-  bool (InternalStats::*handle_int)(uint64_t* value, DBImpl* db,
-                                    Version* version);
+// IMPORTANT: If you add a new property here, also add it to the list in
+//            include/rocksdb/db.h
+enum DBPropertyType : uint32_t {
+  kUnknown,
+  kNumFilesAtLevel,  // Number of files at a specific level
+  kLevelStats,       // Return number of files and total sizes of each level
+  kCFStats,          // Return general statitistics of CF
+  kDBStats,          // Return general statitistics of DB
+  kStats,            // Return general statitistics of both DB and CF
+  kSsTables,         // Return a human readable string of current SST files
+  kStartIntTypes,    // ---- Dummy value to indicate the start of integer values
+  kNumImmutableMemTable,         // Return number of immutable mem tables that
+                                 // have not been flushed.
+  kNumImmutableMemTableFlushed,  // Return number of immutable mem tables
+                                 // in memory that have already been flushed
+  kMemtableFlushPending,         // Return 1 if mem table flushing is pending,
+                                 // otherwise 0.
+  kNumRunningFlushes,      // Return the number of currently running flushes.
+  kCompactionPending,      // Return 1 if a compaction is pending. Otherwise 0.
+  kNumRunningCompactions,  // Return the number of currently running
+                           // compactions.
+  kBackgroundErrors,       // Return accumulated background errors encountered.
+  kCurSizeActiveMemTable,  // Return current size of the active memtable
+  kCurSizeAllMemTables,    // Return current size of unflushed
+                           // (active + immutable) memtables
+  kSizeAllMemTables,       // Return current size of all (active + immutable
+                           // + pinned) memtables
+  kNumEntriesInMutableMemtable,    // Return number of deletes in the mutable
+                                   // memtable.
+  kNumEntriesInImmutableMemtable,  // Return sum of number of entries in all
+                                   // the immutable mem tables.
+  kNumDeletesInMutableMemtable,    // Return number of deletion entries in the
+                                   // mutable memtable.
+  kNumDeletesInImmutableMemtable,  // Return the total number of deletion
+                                   // entries in all the immutable mem tables.
+  kEstimatedNumKeys,  // Estimated total number of keys in the database.
+  kEstimatedUsageByTableReaders,  // Estimated memory by table readers.
+  kIsFileDeletionEnabled,         // Equals disable_delete_obsolete_files_,
+                                  // 0 means file deletions enabled
+  kNumSnapshots,                  // Number of snapshots in the system
+  kOldestSnapshotTime,            // Unix timestamp of the first snapshot
+  kNumLiveVersions,
+  kEstimateLiveDataSize,            // Estimated amount of live data in bytes
+  kTotalSstFilesSize,               // Total size of all sst files.
+  kBaseLevel,                       // The level that L0 data is compacted to
+  kEstimatePendingCompactionBytes,  // Estimated bytes to compaction
+  kAggregatedTableProperties,  // Return a string that contains the aggregated
+                               // table properties.
+  kAggregatedTablePropertiesAtLevel,  // Return a string that contains the
+                                      // aggregated
+  // table properties at the specified level.
 };
 
-extern const DBPropertyInfo* GetPropertyInfo(const Slice& property);
+extern DBPropertyType GetPropertyType(const Slice& property,
+                                      bool* is_int_property,
+                                      bool* need_out_of_mutex);
+
 
 #ifndef ROCKSDB_LITE
 class InternalStats {
@@ -214,18 +248,14 @@ class InternalStats {
 
   uint64_t BumpAndGetBackgroundErrorCount() { return ++bg_error_count_; }
 
-  bool GetStringProperty(const DBPropertyInfo& property_info,
-                         const Slice& property, std::string* value);
+  bool GetStringProperty(DBPropertyType property_type, const Slice& property,
+                         std::string* value);
 
-  bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
-                      DBImpl* db);
+  bool GetIntProperty(DBPropertyType property_type, uint64_t* value,
+                      DBImpl* db) const;
 
-  bool GetIntPropertyOutOfMutex(const DBPropertyInfo& property_info,
-                                Version* version, uint64_t* value);
-
-  // Store a mapping from the user-facing DB::Properties string to our
-  // DBPropertyInfo struct used internally for retrieving properties.
-  static const std::unordered_map<std::string, DBPropertyInfo> ppt_name_to_info;
+  bool GetIntPropertyOutOfMutex(DBPropertyType property_type, Version* version,
+                                uint64_t* value) const;
 
  private:
   void DumpDBStats(std::string* value);
@@ -291,56 +321,6 @@ class InternalStats {
           seconds_up(0) {}
   } db_stats_snapshot_;
 
-  // Handler functions for getting property values. They use "value" as a value-
-  // result argument, and return true upon successfully setting "value".
-  bool HandleNumFilesAtLevel(std::string* value, Slice suffix);
-  bool HandleLevelStats(std::string* value, Slice suffix);
-  bool HandleStats(std::string* value, Slice suffix);
-  bool HandleCFStats(std::string* value, Slice suffix);
-  bool HandleDBStats(std::string* value, Slice suffix);
-  bool HandleSsTables(std::string* value, Slice suffix);
-  bool HandleAggregatedTableProperties(std::string* value, Slice suffix);
-  bool HandleAggregatedTablePropertiesAtLevel(std::string* value, Slice suffix);
-  bool HandleNumImmutableMemTable(uint64_t* value, DBImpl* db,
-                                  Version* version);
-  bool HandleNumImmutableMemTableFlushed(uint64_t* value, DBImpl* db,
-                                         Version* version);
-  bool HandleMemTableFlushPending(uint64_t* value, DBImpl* db,
-                                  Version* version);
-  bool HandleNumRunningFlushes(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleCompactionPending(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleNumRunningCompactions(uint64_t* value, DBImpl* db,
-                                   Version* version);
-  bool HandleBackgroundErrors(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleCurSizeActiveMemTable(uint64_t* value, DBImpl* db,
-                                   Version* version);
-  bool HandleCurSizeAllMemTables(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleSizeAllMemTables(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleNumEntriesActiveMemTable(uint64_t* value, DBImpl* db,
-                                      Version* version);
-  bool HandleNumEntriesImmMemTables(uint64_t* value, DBImpl* db,
-                                    Version* version);
-  bool HandleNumDeletesActiveMemTable(uint64_t* value, DBImpl* db,
-                                      Version* version);
-  bool HandleNumDeletesImmMemTables(uint64_t* value, DBImpl* db,
-                                    Version* version);
-  bool HandleEstimateNumKeys(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleNumSnapshots(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleOldestSnapshotTime(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleNumLiveVersions(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleCurrentSuperVersionNumber(uint64_t* value, DBImpl* db,
-                                       Version* version);
-  bool HandleIsFileDeletionsEnabled(uint64_t* value, DBImpl* db,
-                                    Version* version);
-  bool HandleBaseLevel(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleTotalSstFilesSize(uint64_t* value, DBImpl* db, Version* version);
-  bool HandleEstimatePendingCompactionBytes(uint64_t* value, DBImpl* db,
-                                            Version* version);
-  bool HandleEstimateTableReadersMem(uint64_t* value, DBImpl* db,
-                                     Version* version);
-  bool HandleEstimateLiveDataSize(uint64_t* value, DBImpl* db,
-                                  Version* version);
-
   // Total number of background errors encountered. Every time a flush task
   // or compaction task fails, this counter is incremented. The failure can
   // be caused by any possible reason, including file system errors, out of
@@ -422,20 +402,14 @@ class InternalStats {
 
   uint64_t BumpAndGetBackgroundErrorCount() { return 0; }
 
-  bool GetStringProperty(const DBPropertyInfo& property_info,
-                         const Slice& property, std::string* value) {
-    return false;
-  }
+  bool GetStringProperty(DBPropertyType property_type, const Slice& property,
+                         std::string* value) { return false; }
 
-  bool GetIntProperty(const DBPropertyInfo& property_info, uint64_t* value,
-                      DBImpl* db) const {
-    return false;
-  }
+  bool GetIntProperty(DBPropertyType property_type, uint64_t* value,
+                      DBImpl* db) const { return false; }
 
-  bool GetIntPropertyOutOfMutex(const DBPropertyInfo& property_info,
-                                Version* version, uint64_t* value) const {
-    return false;
-  }
+  bool GetIntPropertyOutOfMutex(DBPropertyType property_type, Version* version,
+                                uint64_t* value) const { return false; }
 };
 #endif  // !ROCKSDB_LITE
 

db/job_context.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -22,9 +22,9 @@ class MemTable;
 struct JobContext {
   inline bool HaveSomethingToDelete() const {
     return full_scan_candidate_files.size() || sst_delete_files.size() ||
-           log_delete_files.size() || manifest_delete_files.size() ||
-           new_superversion != nullptr || superversions_to_free.size() > 0 ||
-           memtables_to_free.size() > 0 || logs_to_free.size() > 0;
+           log_delete_files.size() || new_superversion != nullptr ||
+           superversions_to_free.size() > 0 || memtables_to_free.size() > 0 ||
+           logs_to_free.size() > 0;
   }
 
   // Structure to store information for candidate files to delete.
@@ -56,9 +56,6 @@ struct JobContext {
   // a list of log files that we need to delete
   std::vector<uint64_t> log_delete_files;
 
-  // a list of manifest files that we need to delete
-  std::vector<std::string> manifest_delete_files;
-
   // a list of memtables to be free
   autovector<MemTable*> memtables_to_free;
 

db/listener_test.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/log_format.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

db/log_reader.cc

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.