Merge pull request #764 from dmittendorf/fix-java-static-packaging

Fix crossbuild jar packaging Conflicts: Makefile java/crossbuild/build-linux-centos.sh
Merge pull request #759 from jwlent55/statically-load-compression-libraries
2026-11-06 11:02:36 -05:00 · 2015-10-15 15:05:20 -07:00 · 2015-10-15 15:04:53 -07:00 · 2015-09-30 11:45:32 -07:00 · 2015-09-30 11:37:27 -07:00 · 2015-09-30 11:15:35 -07:00
168 changed files with 2796 additions and 9488 deletions
@@ -38,7 +38,7 @@ coverage/COVERAGE_REPORT
 .gdbhistory
 package/
 .phutil_module_cache
-unity.a
+unity
 tags
 rocksdb_dump
 rocksdb_undump
@@ -52,8 +52,6 @@ java/include/org_rocksdb_*.h
 .idea/
 *.iml

-rocksdb.cc
-rocksdb.h
 unity.cc
 java/crossbuild/.vagrant
 .vagrant/
@@ -33,7 +33,7 @@ before_script:
 # Lousy hack to disable use and testing of fallocate, which doesn't behave quite
 # as EnvPosixTest::AllocateTest expects within the Travis OpenVZ environment.
 script:
-  - if [[ "${TRAVIS_OS_NAME}" == 'linux' ]]; then OPT=-DTRAVIS CLANG_FORMAT_DIFF=/tmp/clang-format-diff.py make format || true; fi
+  - if [[ "${TRAVIS_OS_NAME}" == 'linux' ]]; then OPT=-DTRAVIS CLANG_FORMAT_DIFF=/tmp/clang-format-diff.py make format; fi
  - OPT=-DTRAVIS V=1 make -j4 check && OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest

 notifications:
@@ -13,10 +13,10 @@
 #        cd build
 # 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
 #    See thirdparty.inc for more information.
-#        sample command: cmake -G "Visual Studio 12 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 ..
+#        cmake -G "Visual Studio 12 Win64" .. <more options>
 # 4. Then build the project in debug mode (you may want to add /m:<N> flag to run msbuild in <N> parallel threads)
 #        msbuild ALL_BUILD.vcxproj
-# 5. And release mode (/m[:<N>] is also supported)
+# 5. And release mode (/m:<N> is also supported)
 #        msbuild ALL_BUILD.vcxproj /p:Configuration=Release
 #

@@ -45,34 +45,19 @@ add_custom_command(OUTPUT ${BUILD_VERSION_CC}

 add_custom_target(GenerateBuildVersion DEPENDS ${BUILD_VERSION_CC})

-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo  /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /WX /wd4018 /wd4100 /wd4101 /wd4127 /wd4189 /wd4200 /wd4244 /wd4267 /wd4296 /wd4305 /wd4307 /wd4309 /wd4512 /wd4701 /wd4702 /wd4800 /wd4804 /wd4996")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /W3 /WX /EHsc /GS /fp:precise /Zc:wchar_t /Zc:forScope /Gd /TP /errorReport:queue")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /wd4018 /wd4100 /wd4101 /wd4127 /wd4189 /wd4200 /wd4244 /wd4267 /wd4296 /wd4305 /wd4307 /wd4309 /wd4512 /wd4701 /wd4702 /wd4800 /wd4804 /wd4996")

-# Used to run CI build and tests so we can run faster
-set(OPTIMIZE_DEBUG_DEFAULT 0)        # Debug build is unoptimized by default use -DOPTDBG=1 to optimize
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd")
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /O2 /Oi /Gm- /Gy /MD")

-if(DEFINED OPTDBG)
-   set(OPTIMIZE_DEBUG ${OPTDBG})
-else()
-   set(OPTIMIZE_DEBUG ${OPTIMIZE_DEBUG_DEFAULT})
-endif()
-
-if((${OPTIMIZE_DEBUG} EQUAL 1))
-   message(STATUS "Debug optimization is enabled")
-   set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /MDd")
-else()
-   set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd")
-endif()
-
-set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /MD")
-
-set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")

 add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64)

 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/include)
+include_directories(${PROJECT_SOURCE_DIR}/port)
 include_directories(${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)

 set(ROCKSDB_LIBS rocksdblib${ARTIFACT_SUFFIX})
@@ -153,7 +138,6 @@ set(SOURCES
        table/get_context.cc
        table/iterator.cc
        table/merger.cc
-        table/sst_file_writer.cc
        table/meta_blocks.cc
        table/mock_table.cc
        table/plain_table_builder.cc
@@ -163,7 +147,6 @@ set(SOURCES
        table/plain_table_reader.cc
        table/table_properties.cc
        table/two_level_iterator.cc
-        tools/dump/db_dump_tool.cc
        util/arena.cc
        util/auto_roll_logger.cc
        util/bloom.cc
@@ -201,7 +184,6 @@ set(SOURCES
        util/options.cc
        util/options_builder.cc
        util/options_helper.cc
-        util/options_parser.cc
        util/perf_context.cc
        util/perf_level.cc
        util/rate_limiter.cc
@@ -273,9 +255,8 @@ set(APPS
        util/cache_bench.cc
 )

-set(C_TESTS db/c_test.c)
-
 set(TESTS
+        db/c_test.c
        db/column_family_test.cc
        db/compact_files_test.cc
        db/compaction_job_test.cc
@@ -341,7 +322,6 @@ set(TESTS
        util/file_reader_writer_test.cc
        util/heap_test.cc
        util/histogram_test.cc
-        util/ldb_cmd_test.cc
        util/manual_compaction_test.cc
        util/memenv_test.cc
        util/mock_env_test.cc
@@ -370,17 +350,8 @@ set(EXES ${APPS} ${TESTS})

 foreach(sourcefile ${EXES})
    string(REPLACE ".cc" "" exename ${sourcefile})
+    string(REPLACE ".c" "" exename ${exename})
    string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
    add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
    target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${LIBS})
 endforeach(sourcefile ${EXES})
-
-# C executables must link to a shared object
-set(C_EXES ${C_TESTS})
-
-foreach(sourcefile ${C_EXES})
-    string(REPLACE ".c" "" exename ${sourcefile})
-    string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
-    add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
-    target_link_libraries(${exename}${ARTIFACT_SUFFIX} rocksdb${ARTIFACT_SUFFIX})
-endforeach(sourcefile ${C_TESTS})
@@ -1,23 +1,9 @@
 # Rocksdb Change Log

-## 4.1.0 (10/8/2015)
-### New Features
-* Added single delete operation as a more efficient way to delete keys that have not been overwritten.
-* Added experimental AddFile() to DB interface that allow users to add files created by SstFileWriter into an empty Database, see include/rocksdb/sst_file_writer.h and DB::AddFile() for more info.
-* Added support for opening SST files with .ldb suffix which enables opening LevelDB databases.
-* CompactionFilter now supports filtering of merge operands and merge results.
-
-### Public API Changes
-* Added SingleDelete() to the DB interface.
-* Added AddFile() to DB interface.
-* Added SstFileWriter class.
-* CompactionFilter has a new method FilterMergeOperand() that RocksDB applies to every merge operand during compaction to decide whether to filter the operand.
-* We removed CompactionFilterV2 interfaces from include/rocksdb/compaction_filter.h. The functionality was deprecated already in version 3.13.
-
 ## 4.0.0 (9/9/2015)
 ### New Features
 * Added support for transactions.  See include/rocksdb/utilities/transaction.h for more info.
-* DB::GetProperty() now accepts "rocksdb.aggregated-table-properties" and "rocksdb.aggregated-table-properties-at-levelN", in which case it returns aggregated table properties of the target column family, or the aggregated table properties of the specified level N if the "at-level" version is used.
+* DB::GetProperty() now accept "rocksdb.aggregated-table-properties" and "rocksdb.aggregated-table-properties-at-levelN", in which case it returns aggregated table properties of the target column family, or the aggregated table properties of the specified level N if the "at-level" version is used.
 * Add compression option kZSTDNotFinalCompression for people to experiment ZSTD although its format is not finalized.
 * We removed the need for LATEST_BACKUP file in BackupEngine. We still keep writing it when we create new backups (because of backward compatibility), but we don't read it anymore.

@@ -63,14 +63,6 @@ ifeq ($(MAKECMDGOALS),rocksdbjavastatic)
 	DEBUG_LEVEL=0
 endif

-ifeq ($(MAKECMDGOALS),rocksdbjavastaticrelease)
-	DEBUG_LEVEL=0
-endif
-
-ifeq ($(MAKECMDGOALS),rocksdbjavastaticpublish)
-	DEBUG_LEVEL=0
-endif
-
 # compile with -O2 if debug level is not 2
 ifneq ($(DEBUG_LEVEL), 2)
 OPT += -O2 -fno-omit-frame-pointer
@@ -214,13 +206,13 @@ util/build_version.cc: FORCE
 	else mv -f $@-t $@; fi

 LIBOBJECTS = $(LIB_SOURCES:.cc=.o)
-LIBOBJECTS += $(TOOL_SOURCES:.cc=.o)
 MOCKOBJECTS = $(MOCK_SOURCES:.cc=.o)

 GTEST = $(GTEST_DIR)/gtest/gtest-all.o
 TESTUTIL = ./util/testutil.o
 TESTHARNESS = ./util/testharness.o $(TESTUTIL) $(MOCKOBJECTS) $(GTEST)
 VALGRIND_ERROR = 2
+VALGRIND_DIR = build_tools/VALGRIND_LOGS
 VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)

 VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full
@@ -303,7 +295,6 @@ TESTS = \
 	flush_job_test \
 	wal_manager_test \
 	listener_test \
-	compaction_iterator_test \
 	compaction_job_test \
 	thread_list_test \
 	sst_dump_test \
@@ -314,8 +305,7 @@ TESTS = \
 	heap_test \
 	compact_on_deletion_collector_test \
 	compaction_job_stats_test \
-	transaction_test \
-	ldb_cmd_test
+	transaction_test

 SUBSET :=  $(shell echo $(TESTS) |sed s/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/)

@@ -583,12 +573,20 @@ asan_crash_test:
 	$(MAKE) clean

 valgrind_check: $(TESTS)
+	mkdir -p $(VALGRIND_DIR)
+	echo TESTS THAT HAVE VALGRIND ERRORS > $(VALGRIND_DIR)/valgrind_failed_tests; \
+	echo TIMES in seconds TAKEN BY TESTS ON VALGRIND > $(VALGRIND_DIR)/valgrind_tests_times; \
 	for t in $(filter-out skiplist_test,$(TESTS)); do \
+		stime=`date '+%s'`; \
 		$(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \
 		ret_code=$$?; \
-		if [ $$ret_code -ne 0 ]; then \
+		if [ $$ret_code -eq $(VALGRIND_ERROR) ] ; then \
+			echo $$t >> $(VALGRIND_DIR)/valgrind_failed_tests; \
+		elif [ $$ret_code -ne 0 ]; then \
 			exit $$ret_code; \
 		fi; \
+		etime=`date '+%s'`; \
+		echo $$t $$((etime - stime)) >> $(VALGRIND_DIR)/valgrind_tests_times; \
 	done

 analyze: clean
@@ -601,22 +599,14 @@ CLEAN_FILES += unity.cc
 unity.cc: Makefile
 	rm -f $@ $@-t
 	for source_file in $(LIB_SOURCES); do \
-		echo "#include \"$$source_file\"" >> $@-t; \
+		echo "#include <$$source_file>" >> $@-t; \
 	done
+	echo 'int main(int argc, char** argv){ return 0; }' >> $@-t
 	chmod a=r $@-t
 	mv $@-t $@

-unity.a: unity.o
-	$(AM_V_AR)rm -f $@
-	$(AM_V_at)$(AR) $(ARFLAGS) $@ unity.o
-
-# try compiling db_test with unity
-unity_test: db/db_test.o util/db_test_util.o $(TESTHARNESS) unity.a
+unity: unity.o
 	$(AM_LINK)
-	./unity_test
-
-rocksdb.h rocksdb.cc: build_tools/amalgamate.py Makefile $(LIB_SOURCES) unity.cc
-	build_tools/amalgamate.py -I. -i./include unity.cc -x include/rocksdb/c.h -H rocksdb.h -o rocksdb.cc

 clean:
 	rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(LIBRARY) $(SHARED)
@@ -783,9 +773,6 @@ write_batch_with_index_test: utilities/write_batch_with_index/write_batch_with_i
 flush_job_test: db/flush_job_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-compaction_iterator_test: db/compaction_iterator_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 compaction_job_test: db/compaction_job_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -942,9 +929,6 @@ transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TEST
 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 	$(AM_LINK)

-ldb_cmd_test: util/ldb_cmd_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 ldb: tools/ldb.o $(LIBOBJECTS)
 	$(AM_LINK)

@@ -1037,20 +1021,23 @@ liblz4.a:
 	   cd lz4-r127/lib && make CFLAGS='-fPIC' all
 	   cp lz4-r127/lib/liblz4.a .

-# A version of each $(LIBOBJECTS) compiled with -fPIC
-java_libobjects = $(patsubst %,jl/%,$(LIBOBJECTS))
-CLEAN_FILES += jl
+# A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries
+java_static_libobjects = $(patsubst %,jls/%,$(LIBOBJECTS))
+CLEAN_FILES += jls

-$(java_libobjects): jl/%.o: %.cc
-	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) -fPIC -c $< -o $@ $(COVERAGEFLAGS)
+JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4
+JAVA_STATIC_INCLUDES = -I./zlib-1.2.8 -I./bzip2-1.0.6 -I./snappy-1.1.1 -I./lz4-r127/lib

-rocksdbjavastatic: $(java_libobjects) libz.a libbz2.a libsnappy.a liblz4.a
+$(java_static_libobjects): jls/%.o: %.cc libz.a libbz2.a libsnappy.a liblz4.a
+	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -fPIC -c $< -o $@ $(COVERAGEFLAGS)
+
+rocksdbjavastatic: $(java_static_libobjects)
 	cd java;$(MAKE) javalib;
 	rm -f ./java/target/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
-	  $(java_libobjects) $(COVERAGEFLAGS) \
-	  libz.a libbz2.a libsnappy.a liblz4.a $(LDFLAGS)
+	  $(java_static_libobjects) $(COVERAGEFLAGS) \
+	  libz.a libbz2.a libsnappy.a liblz4.a $(JAVA_STATIC_LDFLAGS)
 	cd java/target;strip -S -x $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
@@ -1061,7 +1048,7 @@ rocksdbjavastatic: $(java_libobjects) libz.a libbz2.a libsnappy.a liblz4.a
 rocksdbjavastaticrelease: rocksdbjavastatic
 	cd java/crossbuild && vagrant destroy -f && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64
 	cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
-	cd java;jar -uf target/$(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
+	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class

 rocksdbjavastaticpublish: rocksdbjavastaticrelease
@@ -1072,13 +1059,20 @@ rocksdbjavastaticpublish: rocksdbjavastaticrelease
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar -Dclassifier=osx
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar

+# A version of each $(LIBOBJECTS) compiled with -fPIC
+java_libobjects = $(patsubst %,jl/%,$(LIBOBJECTS))
+CLEAN_FILES += jl
+
+$(java_libobjects): jl/%.o: %.cc
+	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) -fPIC -c $< -o $@ $(COVERAGEFLAGS)
+
 rocksdbjava: $(java_libobjects)
-	$(AM_V_GEN)cd java;$(MAKE) javalib;
-	$(AM_V_at)rm -f ./java/target/$(ROCKSDBJNILIB)
-	$(AM_V_at)$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(java_libobjects) $(JAVA_LDFLAGS) $(COVERAGEFLAGS)
-	$(AM_V_at)cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
-	$(AM_V_at)cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
-	$(AM_V_at)cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
+	cd java;$(MAKE) javalib;
+	rm -f ./java/target/$(ROCKSDBJNILIB)
+	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(java_libobjects) $(JAVA_LDFLAGS) $(COVERAGEFLAGS)
+	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
+	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
+	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class

 jclean:
 	cd java;$(MAKE) clean;
@@ -6,6 +6,5 @@ before_build:
 - cd ..
 build:
  project: build\ALL_BUILD.vcxproj
-  parallel: true
  verbosity: minimal
 test: off
@@ -1,22 +0,0 @@
-version: 1.0.{build}
-before_build:
- md %APPVEYOR_BUILD_FOLDER%\build
- cd %APPVEYOR_BUILD_FOLDER%\build
- cmake -G "Visual Studio 12 Win64" -DOPTDBG=1 ..
- cd ..
-build:
-  project: build\ALL_BUILD.vcxproj
-  parallel: true
-  verbosity: minimal
-test:
-test_script:
- ps: build_tools\run_ci_db_test.ps1
-notifications:
-  - provider: Email
-    to:
-      - svmtrocksdb@microsoft.com
-    subject: "Build {{status}}"
-    message: "{{message}}, {{commitId}}, ..."
-    on_build_success: false
-    on_build_failure: true
-    on_build_status_changed: true
@@ -1,110 +0,0 @@
-#!/usr/bin/python
-
-# amalgamate.py creates an amalgamation from a unity build.
-# It can be run with either Python 2 or 3.
-# An amalgamation consists of a header that includes the contents of all public
-# headers and a source file that includes the contents of all source files and
-# private headers.
-#
-# This script works by starting with the unity build file and recursively expanding
-# #include directives. If the #include is found in a public include directory,
-# that header is expanded into the amalgamation header.
-#
-# A particular header is only expanded once, so this script will
-# break if there are multiple inclusions of the same header that are expected to
-# expand differently. Similarly, this type of code causes issues:
-#
-# #ifdef FOO
-#   #include "bar.h"
-#   // code here
-# #else
-#   #include "bar.h"            // oops, doesn't get expanded
-#   // different code here
-# #endif
-#
-# The solution is to move the include out of the #ifdef.
-
-from __future__ import print_function
-
-import argparse
-from os import path
-import re
-import sys
-
-include_re = re.compile('^[ \t]*#include[ \t]+"(.*)"[ \t]*$')
-included = set()
-excluded = set()
-
-def find_header(name, abs_path, include_paths):
-    samedir = path.join(path.dirname(abs_path), name)
-    if path.exists(samedir):
-        return samedir
-    for include_path in include_paths:
-        include_path = path.join(include_path, name)
-        if path.exists(include_path):
-            return include_path
-    return None
-
-def expand_include(include_path, f, abs_path, source_out, header_out, include_paths, public_include_paths):
-    if include_path in included:
-        return False
-
-    included.add(include_path)
-    with open(include_path) as f:
-        print('#line 1 "{}"'.format(include_path), file=source_out)
-        process_file(f, include_path, source_out, header_out, include_paths, public_include_paths)
-    return True
-
-def process_file(f, abs_path, source_out, header_out, include_paths, public_include_paths):
-    for (line, text) in enumerate(f):
-        m = include_re.match(text)
-        if m:
-            filename = m.groups()[0]
-            # first check private headers
-            include_path = find_header(filename, abs_path, include_paths)
-            if include_path:
-                if include_path in excluded:
-                    source_out.write(text)
-                    expanded = False
-                else:
-                    expanded = expand_include(include_path, f, abs_path, source_out, header_out, include_paths, public_include_paths)
-            else:
-                # now try public headers
-                include_path = find_header(filename, abs_path, public_include_paths)
-                if include_path:
-                    # found public header
-                    expanded = False
-                    if include_path in excluded:
-                        source_out.write(text)
-                    else:
-                        expand_include(include_path, f, abs_path, header_out, None, public_include_paths, [])
-                else:
-                    sys.exit("unable to find {}, included in {} on line {}".format(filename, abs_path, line))
-
-            if expanded:
-                print('#line {} "{}"'.format(line+1, abs_path), file=source_out)
-        elif text != "#pragma once\n":
-            source_out.write(text)
-
-def main():
-    parser = argparse.ArgumentParser(description="Transform a unity build into an amalgamation")
-    parser.add_argument("source", help="source file")
-    parser.add_argument("-I", action="append", dest="include_paths", help="include paths for private headers")
-    parser.add_argument("-i", action="append", dest="public_include_paths", help="include paths for public headers")
-    parser.add_argument("-x", action="append", dest="excluded", help="excluded header files")
-    parser.add_argument("-o", dest="source_out", help="output C++ file", required=True)
-    parser.add_argument("-H", dest="header_out", help="output C++ header file", required=True)
-    args = parser.parse_args()
-
-    include_paths = list(map(path.abspath, args.include_paths or []))
-    public_include_paths = list(map(path.abspath, args.public_include_paths or []))
-    excluded.update(map(path.abspath, args.excluded or []))
-    filename = args.source
-    abs_path = path.abspath(filename)
-    with open(filename) as f, open(args.source_out, 'w') as source_out, open(args.header_out, 'w') as header_out:
-        print('#line 1 "{}"'.format(filename), file=source_out)
-        print('#include "{}"'.format(header_out.name), file=source_out)
-        process_file(f, abs_path, source_out, header_out, include_paths, public_include_paths)
-
-if __name__ == "__main__":
-    main()
@@ -8,6 +8,7 @@
 #   CXX                         C++ Compiler path
 #   PLATFORM_LDFLAGS            Linker flags
 #   JAVA_LDFLAGS                Linker flags for RocksDBJava
+#   JAVA_STATIC_LDFLAGS         Linker flags for RocksDBJava static build
 #   PLATFORM_SHARED_EXT         Extension for shared libraries
 #   PLATFORM_SHARED_LDFLAGS     Flags for building shared library
 #   PLATFORM_SHARED_CFLAGS      Flags for compiling objects for shared library
@@ -181,6 +182,7 @@ esac

 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}"
 JAVA_LDFLAGS="$PLATFORM_LDFLAGS"
+JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS"

 if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
    # Cross-compiling; do not try any compilation tests.
@@ -213,7 +215,7 @@ EOF
    fi

    # Test whether gflags library is installed
-    # http://gflags.github.io/gflags/
+    # http://code.google.com/p/gflags/
    # check if the namespace is gflags
    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
      #include <gflags/gflags.h>
@@ -374,6 +376,7 @@ echo "CXX=$CXX" >> "$OUTPUT"
 echo "PLATFORM=$PLATFORM" >> "$OUTPUT"
 echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT"
 echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT"
+echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT"
 echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT"
 echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT"
 echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT"
@@ -2,62 +2,22 @@
 # This script is executed by Sandcastle
 # to determine next steps to run

-# Usage:
-# EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> rocks_ci.py <test-name>
-#
-# Input         Value
-# -------------------------------------------------------------------------
-# EMAIL         Email address to report on trigger conditions
-# ONCAL         Email address to raise a task on failure
-# TRIGGER       Trigger conditions for email. Valid values are fail, warn, all
-# SUBSCRIBER    Email addresss to add as subscriber for task
-#
-
-#
-# Report configuration
-#
-REPORT_EMAIL=
-if [ ! -z $EMAIL ]; then
-  if [ -z $TRIGGER ]; then
-    TRIGGER="fail"
-  fi
-
-  REPORT_EMAIL="
-  {
-      'type':'email',
-      'triggers': [ '$TRIGGER' ],
-      'emails':['$EMAIL']
-  },"
-fi
-
-CREATE_TASK=
-if [ ! -z $ONCALL ]; then
-  CREATE_TASK="
-  {
-      'type':'task',
-      'triggers':[ 'fail' ],
-      'priority':0,
-      'subscribers':[ '$SUBSCRIBER' ],
-      'tags':[ 'rocksdb', 'ci' ],
-  },"
-fi
-
-REPORT=
-if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then
-  REPORT="'report': [
-    $REPORT_EMAIL
-    $CREATE_TASK
-  ]"
-fi
-
 #
 # Helper variables
 #
+REPORT_EMAIL="'report': [
+    {
+        'type':'email',
+        'triggers':['fail'],
+        'emails':['leveldb@fb.com']
+    }
+]"
+
 CLEANUP_ENV="
 {
-    'name':'Cleanup environment',
-    'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && make clean',
-    'user':'root'
+  'name':'Cleanup environment',
+  'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && make clean',
+  'user':'root'
 }"

 DEBUG="OPT=-g"
@@ -67,24 +27,6 @@ ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
 LITE="OPT=-DROCKSDB_LITE"
 TSAN="COMPILE_WITH_TSAN=1"
-DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
-
-#
-# A mechanism to disable tests temporarily
-#
-DISABLE_COMMANDS="[
-    {
-        'name':'Disable test',
-        'oncall':'$ONCALL',
-        'steps': [
-            {
-              'name':'Job disabled. Please contact test owner',
-              'shell':'exit 1',
-              'user':'root'
-            },
-        ],
-    }
-]"

 #
 # RocksDB unit test in parallel
@@ -94,22 +36,21 @@ DISABLE_COMMANDS="[
 PARALLEL_UNIT_TEST_COMMANDS="[
    {
        'name':'Rocksdb Parallel Unit Test',
-        'oncall':'$ONCALL',
        'steps': [
            $CLEANUP_ENV,
            {
                'name':'Build and test RocksDB debug version',
-                'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
+                'shell':'$SHM $DEBUG make check 2>&1 > /dev/null || cat t/log-*',
                'user':'root'
            },
            $CLEANUP_ENV,
            {
                'name':'Build and test RocksDB debug version under gcc-4.8.1',
-                'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || cat t/log-*',
+                'shell':'$SHM $GCC_481 $DEBUG make check 2>&1 > /dev/null || cat t/log-*',
                'user':'root'
            },
        ],
-        $REPORT
+        $REPORT_EMAIL
    }
 ]"

@@ -119,7 +60,6 @@ PARALLEL_UNIT_TEST_COMMANDS="[
 UNIT_TEST_COMMANDS="[
    {
        'name':'Rocksdb Unit Test',
-        'oncall':'$ONCALL',
        'steps': [
            $CLEANUP_ENV,
            {
@@ -128,7 +68,7 @@ UNIT_TEST_COMMANDS="[
                'user':'root'
            },
        ],
-        $REPORT
+        $REPORT_EMAIL
    }
 ]"

@@ -138,7 +78,6 @@ UNIT_TEST_COMMANDS="[
 UNIT_TEST_COMMANDS_481="[
    {
        'name':'Rocksdb Unit Test on GCC 4.8.1',
-        'oncall':'$ONCALL',
        'steps': [
            $CLEANUP_ENV,
            {
@@ -147,7 +86,7 @@ UNIT_TEST_COMMANDS_481="[
                'user':'root'
            },
        ],
-        $REPORT
+        $REPORT_EMAIL
    }
 ]"

@@ -157,73 +96,15 @@ UNIT_TEST_COMMANDS_481="[
 CLANG_UNIT_TEST_COMMANDS="[
    {
        'name':'Rocksdb Unit Test',
-        'oncall':'$ONCALL',
        'steps': [
            $CLEANUP_ENV,
            {
                'name':'Build and test RocksDB debug',
-                'shell':'$CLANG $SHM $DEBUG make J=1 check',
+                'shell':'$CLANG $SHM $DEBUG make check',
                'user':'root'
            },
        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB analyze
-#
-CLANG_ANALYZE_COMMANDS="[
-    {
-        'name':'Rocksdb analyze',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'RocksDB build and analyze',
-                'shell':'$CLANG $SHM $DEBUG make J=1 analyze',
-                'user':'root'
-            },
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB code coverage
-#
-CODE_COV_COMMANDS="[
-    {
-        'name':'Rocksdb Unit Test Code Coverage',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build, test and collect code coverage info',
-                'shell':'$SHM $DEBUG make J=1 coverage',
-                'user':'root'
-            },
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB unity
-#
-UNITY_COMMANDS="[
-    {
-        'name':'Rocksdb Unity',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build, test unity test',
-                'shell':'$SHM $DEBUG V=1 make J=1 unity_test',
-                'user':'root'
-            },
-        ],
-        $REPORT
+        $REPORT_EMAIL
    }
 ]"

@@ -233,7 +114,6 @@ UNITY_COMMANDS="[
 LITE_BUILD_COMMANDS="[
    {
        'name':'Rocksdb Lite build',
-        'oncall':'$ONCALL',
        'steps': [
            $CLEANUP_ENV,
            {
@@ -242,7 +122,7 @@ LITE_BUILD_COMMANDS="[
                'user':'root'
            },
        ],
-        $REPORT
+        $REPORT_EMAIL
    }
 ]"

@@ -252,8 +132,6 @@ LITE_BUILD_COMMANDS="[
 STRESS_CRASH_TEST_COMMANDS="[
    {
        'name':'Rocksdb Stress/Crash Test',
-        'oncall':'$ONCALL',
-        'timeout': 86400,
        'steps': [
            $CLEANUP_ENV,
            {
@@ -263,22 +141,21 @@ STRESS_CRASH_TEST_COMMANDS="[
            },
            {
                'name':'Build and run RocksDB debug crash tests',
-                'timeout': 86400,
                'shell':'$SHM $DEBUG make J=1 crash_test',
                'user':'root'
            }
        ],
-        $REPORT
+        $REPORT_EMAIL
    }
 ]"

+
 #
 # RocksDB test under address sanitizer
 #
 ASAN_TEST_COMMANDS="[
    {
        'name':'Rocksdb Unit Test under ASAN',
-        'oncall':'$ONCALL',
        'steps': [
            $CLEANUP_ENV,
            {
@@ -287,7 +164,7 @@ ASAN_TEST_COMMANDS="[
                'user':'root'
            }
        ],
-        $REPORT
+        $REPORT_EMAIL
    }
 ]"

@@ -297,18 +174,15 @@ ASAN_TEST_COMMANDS="[
 ASAN_CRASH_TEST_COMMANDS="[
    {
        'name':'Rocksdb crash test under ASAN',
-        'oncall':'$ONCALL',
-        'timeout': 86400,
        'steps': [
            $CLEANUP_ENV,
            {
                'name':'Build and run RocksDB debug asan_crash_test',
-                'timeout': 86400,
                'shell':'$SHM $DEBUG make J=1 asan_crash_test',
                'user':'root'
            },
        ],
-        $REPORT
+        $REPORT_EMAIL
    }
 ]"

@@ -318,26 +192,24 @@ ASAN_CRASH_TEST_COMMANDS="[
 VALGRIND_TEST_COMMANDS="[
    {
        'name':'Rocksdb Unit Test under valgrind',
-        'oncall':'$ONCALL',
        'steps': [
            $CLEANUP_ENV,
            {
                'name':'Run RocksDB debug unit tests',
-                'shell':'$DISABLE_JEMALLOC $SHM $DEBUG make valgrind_check',
+                'shell':'$SHM $DEBUG ./build_tools/valgrind_test.sh',
                'user':'root'
            },
        ],
-        $REPORT
+        $REPORT_EMAIL
    }
 ]"

 #
 # RocksDB test under TSAN
 #
-TSAN_UNIT_TEST_COMMANDS="[
+TSAN_TEST_COMMANDS="[
    {
        'name':'Rocksdb Unit Test under TSAN',
-        'oncall':'$ONCALL',
        'steps': [
            $CLEANUP_ENV,
            {
@@ -346,177 +218,7 @@ TSAN_UNIT_TEST_COMMANDS="[
                'user':'root'
            },
        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB crash test under TSAN
-#
-TSAN_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb Crash Test under TSAN',
-        'oncall':'$ONCALL',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Compile and run',
-                'timeout': 86400,
-                'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 crash_test',
-                'user':'root'
-            },
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB format compatible
-#
-
-run_format_compatible()
-{
-  export TEST_TMPDIR=/dev/shm/rocksdb
-  rm -rf /dev/shm/rocksdb
-  mkdir /dev/shm/rocksdb
-
-  echo '
-  if [ -e "build_tools/build_detect_platform" ]
-  then
-    sed "s/tcmalloc/nothingnothingnothing/g" build_tools/build_detect_platform > $TEST_TMPDIR/temp_build_file
-    rm -rf build_tools/build_detect_platform
-    cp $TEST_TMPDIR/temp_build_file build_tools/build_detect_platform
-    chmod +x build_tools/build_detect_platform
-  fi
-
-  if [ -e "build_detect_platform" ]
-  then
-    sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
-    rm -rf build_detect_platform 
-    cp $TEST_TMPDIR/temp_build_file build_detect_platform
-    chmod +x build_detect_platform 
-  fi
-
-  make ldb -j32
-
-  if [ -e "build_detect_platform" ]
-  then
-    git checkout -- build_detect_platform
-  fi
-
-  if [ -e "build_tools/build_detect_platform" ]
-  then
-    git checkout -- build_tools/build_detect_platform
-  fi
-  ' > temp_build_ldb.sh
-
-  sed "s/make ldb -j32/source temp_build_ldb.sh/g" tools/check_format_compatible.sh > tools/temp_check_format_compatible.sh
-  chmod +x tools/temp_check_format_compatible.sh
-  tools/temp_check_format_compatible.sh
-}
-
-FORMAT_COMPATIBLE_COMMANDS="[
-    {
-        'name':'Rocksdb Format Compatible tests',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Run RocksDB debug unit test',
-                'shell':'build_tools/rocksdb-lego-determinator run_format_compatible',
-                'user':'root'
-            },
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB no compression
-#
-run_no_compression()
-{
-  export TEST_TMPDIR=/dev/shm/rocksdb
-  rm -rf /dev/shm/rocksdb
-  mkdir /dev/shm/rocksdb
-  make clean
-  cat build_tools/fbcode_config.sh | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh
-  mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh
-  cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile
-  mv .tmp.Makefile Makefile
-  make $DEBUG J=1 check
-}
-
-NO_COMPRESSION_COMMANDS="[
-    {
-        'name':'Rocksdb No Compression tests',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Run RocksDB debug unit test',
-                'shell':'build_tools/rocksdb-lego-determinator run_no_compression',
-                'user':'root'
-            },
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB regression
-#
-run_regression()
-{
-  time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d  $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX)
-
-  # ======= report size to ODS ========
-
-  # parameters: $1 -- key, $2 -- value
-  function send_size_to_ods {
-    curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \
-      --connect-timeout 60
-  }
-
-  # === normal build ===
-  make clean
-  make -j$(nproc) static_lib
-  send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a)
-  strip librocksdb.a
-  send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a)
-
-  make -j$(nproc) shared_lib
-  send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`)
-  strip `readlink -f librocksdb.so`
-  send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
-
-  # === lite build ===
-  make clean
-  OPT=-DROCKSDB_LITE make -j$(nproc) static_lib
-  send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a)
-  strip librocksdb.a
-  send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a)
-
-  OPT=-DROCKSDB_LITE make -j$(nproc) shared_lib
-  send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`)
-  strip `readlink -f librocksdb.so`
-  send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
-}
-
-REGRESSION_COMMANDS="[
-    {
-        'name':'Rocksdb regression commands',
-        'oncall':'$ONCALL',
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Make and run script',
-                'shell':'build_tools/rocksdb-lego-determinator run_regression',
-                'user':'root'
-            },
-        ],
-        $REPORT
+        $REPORT_EMAIL
    }
 ]"

@@ -533,15 +235,6 @@ case $1 in
  clang_unit)
    echo $CLANG_UNIT_TEST_COMMANDS
    ;;
-  clang_analyze)
-    echo $CLANG_ANALYZE_COMMANDS
-    ;;
-  code_cov)
-    echo $CODE_COV_COMMANDS
-    ;;
-  unity)
-    echo $UNITY_COMMANDS
-    ;;
  lite)
    echo $LITE_BUILD_COMMANDS
    ;;
@@ -558,28 +251,7 @@ case $1 in
    echo $VALGRIND_TEST_COMMANDS
    ;;
  tsan)
-    echo $TSAN_UNIT_TEST_COMMANDS
-    ;;
-  tsan_crash)
-    echo $TSAN_CRASH_TEST_COMMANDS
-    ;;
-  format_compatible)
-    echo $FORMAT_COMPATIBLE_COMMANDS
-    ;;
-  run_format_compatible)
-    run_format_compatible
-    ;;
-  no_compression)
-    echo $NO_COMPRESSION_COMMANDS
-    ;;
-  run_no_compression)
-    run_no_compression
-    ;;
-  regression)
-    echo $REGRESSION_COMMANDS
-    ;;
-  run_regression)
-    run_regression
+    echo $TSAN_TEST_COMMANDS
    ;;
  *)
    echo "Invalid determinator command"
@@ -1,252 +0,0 @@
-# This script enables you running RocksDB tests by running
-# All the tests in paralell and utilizing all the cores
-# For db_test the script first lists and parses the tests
-# and then fires them up in parallel using async PS Job functionality
-# Run the script from the enlistment
-Param(
-  [switch]$EnableJE = $false,  # Use je executable
-  [string]$WorkFolder = "",  # Direct tests to use that folder
-  [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
-  [string]$Exclude = "", # Expect a comma separated list, no spaces
-  [string]$Run = "db_test"  # Run db_test|tests
-)
-
-# Folders and commands must be fullpath to run assuming
-# the current folder is at the root of the git enlistment
-Get-Date
-
-# If running under Appveyor assume that root
-[string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
-if($Appveyor -ne "") {
-    $RootFolder = $Appveyor
-} else {
-    $RootFolder = $PSScriptRoot -replace '\\build_tools', ''
-}
-
-$LogFolder = -Join($RootFolder, "\db_logs\")
-$BinariesFolder = -Join($RootFolder, "\build\Debug\")
-
-if($WorkFolder -eq "") {
-
-    # If TEST_TMPDIR is set use it    
-    [string]$var = $Env:TEST_TMPDIR
-    if($var -eq "") {
-        $WorkFolder = -Join($RootFolder, "\db_tests\")
-        $Env:TEST_TMPDIR = $WorkFolder
-    } else {
-        $WorkFolder = $var
-    }
-} else {
-# Override from a command line
-  $Env:TEST_TMPDIR = $WorkFolder
-}
-
-# Use JEMALLOC executables
-if($EnableJE) {
-    $db_test = -Join ($BinariesFolder, "db_test_je.exe")
-} else {
-    $db_test = -Join ($BinariesFolder, "db_test.exe")
-}
-
-Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
-Write-Output "Binaries: $BinariesFolder exe: $db_test"
-
-#Exclusions that we do not want to run
-$ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
-
-
-if($Exclude -ne "") {
-    Write-Host "Exclude: $Exclude"
-    $l = $Exclude -split ','
-    ForEach($t in $l) { $ExcludeTests.Add($t) | Out-Null }
-}
-
-# Create test directories in the current folder
-md -Path $WorkFolder -ErrorAction Ignore | Out-Null
-md -Path $LogFolder -ErrorAction Ignore | Out-Null
-
-# Extract the names of its tests by running db_test with --gtest_list_tests.
-# This filter removes the "#"-introduced comments, and expands to
-# fully-qualified names by changing input like this:
-#
-#   DBTest.
-#     Empty
-#     WriteEmptyBatch
-#   MultiThreaded/MultiThreadedDBTest.
-#     MultiThreaded/0  # GetParam() = 0
-#     MultiThreaded/1  # GetParam() = 1
-#
-# into this:
-#
-#   DBTest.Empty
-#   DBTest.WriteEmptyBatch
-#   MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
-#   MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
-# Output into the parameter in a form TestName -> Log File Name
-function Normalize-DbTests($HashTable) {
-
-    $Tests = @()
-# Run db_test to get a list of tests and store it into $a array
-    &$db_test --gtest_list_tests | tee -Variable Tests | Out-Null
-
-    # Current group
-    $Group=""
-
-    ForEach( $l in $Tests) {
-      # Trailing dot is a test group
-      if( $l -match "\.$") {
-        $Group = $l
-      }  else {
-        # Otherwise it is a test name, remove leading space
-        $test = $l -replace '^\s+',''
-        # remove trailing comment if any and create a log name
-        $test = $test -replace '\s+\#.*',''
-        $test = "$Group$test"
-
-        if($ExcludeTests.Contains($test)) {
-            continue
-        }
-
-        $test_log = $test -replace '[\./]','_'
-        $test_log += ".log"
-
-        # Add to a hashtable
-        $HashTable.Add($test, $test_log);
-      }
-    }
-}
-
-# The function scans build\Debug folder to discover
-# Test executables. It then populates a table with
-# Test executable name -> Log file
-function Discover-TestBinaries($HashTable) {
-
-    $Exclusions = @("db_test*", "db_sanity_test*")
-    $p = -join ($BinariesFolder, "*_test*.exe")
-
-    dir -Path $p -Exclude $Exclusions | ForEach-Object {
-       $t = ($_.Name) -replace '.exe$', ''
-       $test_log = -join ($t, ".log")
-       $HashTable.Add($t, $test_log)
-    }
-}
-
-$TestToLog = [ordered]@{}
-
-if($Run -ceq "db_test") {
-    Normalize-DbTests -HashTable $TestToLog
-} elseif($Run -ceq "tests") {
-    Discover-TestBinaries -HashTable $TestToLog
-}
-
-
-Write-Host "Attempting to start: " ($TestToLog.Count) " tests"
-
-# Invoke a test with a filter and redirect all output
-$InvokeTestCase = {
-    param($exe, $test, $log);
-    &$exe --gtest_filter=$test > $log 2>&1
-}
-
-# Invoke all tests and redirect output
-$InvokeTestAsync = {
-    param($exe, $log)
-    &$exe > $log 2>&1
-}
-
-$jobs = @()
-$JobToLog = @{}
-# Test limiting factor here
-$count = 0
-
-ForEach($k in $TestToLog.keys) {
-
-    Write-Host "Starting $k"
-    $log_path = -join ($LogFolder, ($TestToLog.$k))
-
-    if($Run -ceq "db_test") {
-        $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
-    } else {
-        [string]$Exe =  -Join ($BinariesFolder, $k)
-        $job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path)
-    }
-
-    $JobToLog.Add($job, $log_path)
-
-    # Limiting trial runs
-    if(($Limit -gt 0) -and (++$count -ge $Limit)) {
-         break
-    }
-}
-
-[bool]$success = $true;
-
-# Wait for all to finish and get the results
-while($JobToLog.Count -gt 0) {
-
-    $jobs = @()
-    foreach($k in $JobToLog.Keys) { $jobs += $k }
-
-<#
-    if(!$success) {
-        break
-    }
-#>
-
-    $completed = Wait-Job -Job $jobs -Any
-    $log = $JobToLog[$completed]
-    $JobToLog.Remove($completed)
-
-    $message = -join @($completed.Name, " State: ", ($completed.State))
-
-    $log_content = @(Get-Content $log)
-
-    if($completed.State -ne "Completed") {
-        $success = $false
-        Write-Warning $message
-        $log_content | Write-Warning
-    } else {
-        # Scan the log. If we find PASSED and no occurence of FAILED
-        # then it is a success
-        [bool]$pass_found = $false
-        ForEach($l in $log_content) {
-
-            if(($l -match "^\[\s+FAILED") -or
-               ($l -match "Assertion failed:")) {
-                $pass_found = $false
-                break
-            }
-
-            if(($l -match "^\[\s+PASSED") -or
-               ($l -match " : PASSED$") -or
-                ($l -match "^PASSED") -or
-                ($l -match "Passed all tests!") ) {
-                $pass_found = $true
-            }
-        }
-
-        if(!$pass_found) {
-            $success = $false;
-            Write-Warning $message
-            $log_content | Write-Warning
-        } else {
-            Write-Host $message
-        }
-    }
-
-    # Remove cached job info from the system
-    # Should be no output
-    Receive-Job -Job $completed | Out-Null
-}
-
-Get-Date
-
-if(!$success) {
-# This does not succeed killing off jobs quick
-# So we simply exit
-#    Remove-Job -Job $jobs -Force
-# indicate failure using this exit code
-    exit 12345
- }
-
- 
@@ -0,0 +1,15 @@
+#!/bin/bash
+#A shell script for Jenknis to run valgrind on rocksdb tests
+#Returns 0 on success when there are no failed tests 
+
+VALGRIND_DIR=build_tools/VALGRIND_LOGS
+make clean
+make -j$(nproc) valgrind_check
+NUM_FAILED_TESTS=$((`wc -l $VALGRIND_DIR/valgrind_failed_tests | awk '{print $1}'` - 1))
+if [ $NUM_FAILED_TESTS -lt 1 ]; then
+  echo No tests have valgrind errors
+  exit 0
+else
+  cat $VALGRIND_DIR/valgrind_failed_tests
+  exit 1
+fi
@@ -85,11 +85,10 @@ Status BuildTable(
          file_writer.get(), compression, compression_opts);
    }

-    MergeHelper merge(env, internal_comparator.user_comparator(),
-                      ioptions.merge_operator, nullptr, ioptions.info_log,
+    MergeHelper merge(internal_comparator.user_comparator(),
+                      ioptions.merge_operator, ioptions.info_log,
                      ioptions.min_partial_merge_operands,
-                      true /* internal key corruption is not ok */,
-                      snapshots.empty() ? 0 : snapshots.back());
+                      true /* internal key corruption is not ok */);

    CompactionIterator c_iter(iter, internal_comparator.user_comparator(),
                              &merge, kMaxSequenceNumber, &snapshots, env,
@@ -110,15 +109,13 @@ Status BuildTable(
    }

    // Finish and check for builder errors
-    bool empty = builder->NumEntries() == 0;
    s = c_iter.status();
-    if (!s.ok() || empty) {
-      builder->Abandon();
-    } else {
+    if (s.ok()) {
      s = builder->Finish();
+    } else {
+      builder->Abandon();
    }
-
-    if (s.ok() && !empty) {
+    if (s.ok()) {
      meta->fd.file_size = builder->FileSize();
      meta->marked_for_compaction = builder->NeedCompact();
      assert(meta->fd.GetFileSize() > 0);
@@ -129,27 +126,28 @@ Status BuildTable(
    delete builder;

    // Finish and check for file errors
-    if (s.ok() && !empty && !ioptions.disable_data_sync) {
+    if (s.ok() && !ioptions.disable_data_sync) {
      StopWatch sw(env, ioptions.statistics, TABLE_SYNC_MICROS);
      file_writer->Sync(ioptions.use_fsync);
    }
-    if (s.ok() && !empty) {
+    if (s.ok()) {
      s = file_writer->Close();
    }

-    if (s.ok() && !empty) {
+    if (s.ok()) {
      // Verify that the table is usable
-      std::unique_ptr<Iterator> it(table_cache->NewIterator(
+      Iterator* it = table_cache->NewIterator(
          ReadOptions(), env_options, internal_comparator, meta->fd, nullptr,
          (internal_stats == nullptr) ? nullptr
                                      : internal_stats->GetFileReadHist(0),
-          false));
+          false);
      s = it->status();
      if (s.ok() && paranoid_file_checks) {
-        for (it->SeekToFirst(); it->Valid(); it->Next()) {
-        }
+        for (it->SeekToFirst(); it->Valid(); it->Next()) {}
        s = it->status();
      }
+
+      delete it;
    }
  }

@@ -158,7 +156,9 @@ Status BuildTable(
    s = iter->status();
  }

-  if (!s.ok() || meta->fd.GetFileSize() == 0) {
+  if (s.ok() && meta->fd.GetFileSize() > 0) {
+    // Keep it
+  } else {
    env->DeleteFile(fname);
  }
  return s;
@@ -446,33 +446,15 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
    } else if (vstorage->l0_delay_trigger_count() >=
               mutable_cf_options.level0_stop_writes_trigger) {
      write_controller_token_ = write_controller->GetStopToken();
-      internal_stats_->AddCFStats(InternalStats::LEVEL0_NUM_FILES_TOTAL, 1);
-      if (compaction_picker_->IsLevel0CompactionInProgress()) {
-        internal_stats_->AddCFStats(
-            InternalStats::LEVEL0_NUM_FILES_WITH_COMPACTION, 1);
-      }
+      internal_stats_->AddCFStats(InternalStats::LEVEL0_NUM_FILES, 1);
      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
          "[%s] Stopping writes because we have %d level-0 files",
          name_.c_str(), vstorage->l0_delay_trigger_count());
-    } else if (mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
-               vstorage->estimated_compaction_needed_bytes() >=
-                   mutable_cf_options.hard_pending_compaction_bytes_limit) {
-      write_controller_token_ = write_controller->GetStopToken();
-      internal_stats_->AddCFStats(
-          InternalStats::HARD_PENDING_COMPACTION_BYTES_LIMIT, 1);
-      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
-          "[%s] Stopping writes because estimated pending compaction "
-          "bytes exceed %" PRIu64,
-          name_.c_str(), vstorage->estimated_compaction_needed_bytes());
    } else if (mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
               vstorage->l0_delay_trigger_count() >=
                   mutable_cf_options.level0_slowdown_writes_trigger) {
      write_controller_token_ = write_controller->GetDelayToken();
-      internal_stats_->AddCFStats(InternalStats::LEVEL0_SLOWDOWN_TOTAL, 1);
-      if (compaction_picker_->IsLevel0CompactionInProgress()) {
-        internal_stats_->AddCFStats(
-            InternalStats::LEVEL0_SLOWDOWN_WITH_COMPACTION, 1);
-      }
+      internal_stats_->AddCFStats(InternalStats::LEVEL0_SLOWDOWN, 1);
      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
          "[%s] Stalling writes because we have %d level-0 files",
          name_.c_str(), vstorage->l0_delay_trigger_count());
@@ -23,8 +23,6 @@
 #include "util/sync_point.h"
 #include "utilities/merge_operators.h"

-#if !(defined NDEBUG) || !defined(OS_WIN)
-
 namespace rocksdb {

 namespace {
@@ -1262,13 +1260,8 @@ TEST_F(ColumnFamilyTest, FlushAndDropRaceCondition) {
 }

 }  // namespace rocksdb
-#endif

 int main(int argc, char** argv) {
-#if !(defined NDEBUG) || !defined(OS_WIN)
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
-#else
-  return 0;
-#endif
 }
@@ -40,47 +40,6 @@ void Compaction::SetInputVersion(Version* _input_version) {
  edit_.SetColumnFamily(cfd_->GetID());
 }

-void Compaction::GetBoundaryKeys(
-    VersionStorageInfo* vstorage,
-    const std::vector<CompactionInputFiles>& inputs, Slice* smallest_user_key,
-    Slice* largest_user_key) {
-  bool initialized = false;
-  const Comparator* ucmp = vstorage->InternalComparator()->user_comparator();
-  for (uint32_t i = 0; i < inputs.size(); ++i) {
-    if (inputs[i].files.empty()) {
-      continue;
-    }
-    if (inputs[i].level == 0) {
-      // we need to consider all files on level 0
-      for (const auto* f : inputs[i].files) {
-        const Slice& start_user_key = f->smallest.user_key();
-        if (!initialized ||
-            ucmp->Compare(start_user_key, *smallest_user_key) < 0) {
-          *smallest_user_key = start_user_key;
-        }
-        const Slice& end_user_key = f->largest.user_key();
-        if (!initialized ||
-            ucmp->Compare(end_user_key, *largest_user_key) > 0) {
-          *largest_user_key = end_user_key;
-        }
-        initialized = true;
-      }
-    } else {
-      // we only need to consider the first and last file
-      const Slice& start_user_key = inputs[i].files[0]->smallest.user_key();
-      if (!initialized ||
-          ucmp->Compare(start_user_key, *smallest_user_key) < 0) {
-        *smallest_user_key = start_user_key;
-      }
-      const Slice& end_user_key = inputs[i].files.back()->largest.user_key();
-      if (!initialized || ucmp->Compare(end_user_key, *largest_user_key) > 0) {
-        *largest_user_key = end_user_key;
-      }
-      initialized = true;
-    }
-  }
-}
-
 // helper function to determine if compaction is creating files at the
 // bottommost level
 bool Compaction::IsBottommostLevel(
@@ -91,40 +50,15 @@ bool Compaction::IsBottommostLevel(
    return false;
  }

-  Slice smallest_key, largest_key;
-  GetBoundaryKeys(vstorage, inputs, &smallest_key, &largest_key);
-
-  // Checks whether there are files living beyond the output_level.
-  // If lower levels have files, it checks for overlap between files
-  // if the compaction process and those files.
-  // Bottomlevel optimizations can be made if there are no files in
-  // lower levels or if there is no overlap with the files in
-  // the lower levels.
+  // checks whether there are files living beyond the output_level.
  for (int i = output_level + 1; i < vstorage->num_levels(); i++) {
-    // It is not the bottommost level if there are files in higher
-    // levels when the output level is 0 or if there are files in
-    // higher levels which overlap with files to be compacted.
-    // output_level == 0 means that we want it to be considered
-    // s the bottommost level only if the last file on the level
-    // is a part of the files to be compacted - this is verified by
-    // the first if condition in this function
-    if (vstorage->NumLevelFiles(i) > 0 &&
-        (output_level == 0 ||
-         vstorage->OverlapInLevel(i, &smallest_key, &largest_key))) {
+    if (vstorage->NumLevelFiles(i) > 0) {
      return false;
    }
  }
  return true;
 }

-// test function to validate the functionality of IsBottommostLevel()
-// function -- determines if compaction with inputs and storage is bottommost
-bool Compaction::TEST_IsBottommostLevel(
-    int output_level, VersionStorageInfo* vstorage,
-    const std::vector<CompactionInputFiles>& inputs) {
-  return IsBottommostLevel(output_level, vstorage, inputs);
-}
-
 bool Compaction::IsFullCompaction(
    VersionStorageInfo* vstorage,
    const std::vector<CompactionInputFiles>& inputs) {
@@ -204,27 +204,15 @@ class Compaction {
  // Should this compaction be broken up into smaller ones run in parallel?
  bool ShouldFormSubcompactions() const;

-  // test function to validate the functionality of IsBottommostLevel()
-  // function -- determines if compaction with inputs and storage is bottommost
-  static bool TEST_IsBottommostLevel(
-      int output_level, VersionStorageInfo* vstorage,
-      const std::vector<CompactionInputFiles>& inputs);
-
 private:
  // mark (or clear) all files that are being compacted
  void MarkFilesBeingCompacted(bool mark_as_compacted);

-  // get the smallest and largest key present in files to be compacted
-  static void GetBoundaryKeys(VersionStorageInfo* vstorage,
-                              const std::vector<CompactionInputFiles>& inputs,
-                              Slice* smallest_key, Slice* largest_key);
-
  // helper function to determine if compaction with inputs and storage is
  // bottommost
  static bool IsBottommostLevel(
      int output_level, VersionStorageInfo* vstorage,
      const std::vector<CompactionInputFiles>& inputs);
-
  static bool IsFullCompaction(VersionStorageInfo* vstorage,
                               const std::vector<CompactionInputFiles>& inputs);

@@ -12,14 +12,16 @@ namespace rocksdb {
 CompactionIterator::CompactionIterator(
    Iterator* input, const Comparator* cmp, MergeHelper* merge_helper,
    SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
-    Env* env, bool expect_valid_internal_key, Compaction* compaction,
-    const CompactionFilter* compaction_filter, LogBuffer* log_buffer)
+    Env* env, bool expect_valid_internal_key, Statistics* stats,
+    Compaction* compaction, const CompactionFilter* compaction_filter,
+    LogBuffer* log_buffer)
    : input_(input),
      cmp_(cmp),
      merge_helper_(merge_helper),
      snapshots_(snapshots),
      env_(env),
      expect_valid_internal_key_(expect_valid_internal_key),
+      stats_(stats),
      compaction_(compaction),
      compaction_filter_(compaction_filter),
      log_buffer_(log_buffer),
@@ -69,10 +71,6 @@ void CompactionIterator::Next() {
      // MergeUntil stops when it encounters a corrupt key and does not
      // include them in the result, so we expect the keys here to be valid.
      assert(valid_key);
-      // Keep current_key_ in sync.
-      current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-      key_ = current_key_.GetKey();
-      ikey_.user_key = current_key_.GetUserKey();
      valid_ = true;
    } else {
      // MergeHelper moves the iterator to the first record after the merged
@@ -81,11 +79,8 @@ void CompactionIterator::Next() {
      NextFromInput();
    }
  } else {
-    // Only advance the input iterator if there is no merge output and the
-    // iterator is not already at the next record.
-    if (!at_next_) {
-      input_->Next();
-    }
+    // Only advance the input iterator if there is no merge output.
+    input_->Next();
    NextFromInput();
  }

@@ -93,10 +88,9 @@ void CompactionIterator::Next() {
 }

 void CompactionIterator::NextFromInput() {
-  at_next_ = false;
  valid_ = false;

-  while (!valid_ && input_->Valid()) {
+  while (input_->Valid()) {
    key_ = input_->key();
    value_ = input_->value();
    iter_stats_.num_input_records++;
@@ -106,11 +100,10 @@ void CompactionIterator::NextFromInput() {
      // and let the caller decide what to do with it.
      // TODO(noetzli): We should have a more elegant solution for this.
      if (expect_valid_internal_key_) {
-        assert(!"Corrupted internal key not expected.");
-        status_ = Status::Corruption("Corrupted internal key not expected.");
+        assert(!"corrupted internal key is not expected");
        break;
      }
-      key_ = current_key_.SetKey(key_);
+      current_user_key_.Clear();
      has_current_user_key_ = false;
      current_user_key_sequence_ = kMaxSequenceNumber;
      current_user_key_snapshot_ = 0;
@@ -120,20 +113,16 @@ void CompactionIterator::NextFromInput() {
    }

    // Update input statistics
-    if (ikey_.type == kTypeDeletion || ikey_.type == kTypeSingleDeletion) {
+    if (ikey_.type == kTypeDeletion) {
      iter_stats_.num_input_deletion_records++;
    }
    iter_stats_.total_input_raw_key_bytes += key_.size();
    iter_stats_.total_input_raw_value_bytes += value_.size();

-    // Check whether the user key changed. After this if statement current_key_
-    // is a copy of the current input key (maybe converted to a delete by the
-    // compaction filter). ikey_.user_key is pointing to the copy.
    if (!has_current_user_key_ ||
-        !cmp_->Equal(ikey_.user_key, current_user_key_)) {
+        cmp_->Compare(ikey_.user_key, current_user_key_.GetKey()) != 0) {
      // First occurrence of this user key
-      key_ = current_key_.SetKey(key_, &ikey_);
-      current_user_key_ = ikey_.user_key;
+      current_user_key_.SetKey(ikey_.user_key);
      has_current_user_key_ = true;
      current_user_key_sequence_ = kMaxSequenceNumber;
      current_user_key_snapshot_ = 0;
@@ -156,9 +145,13 @@ void CompactionIterator::NextFromInput() {
              env_ != nullptr ? timer.ElapsedNanos() : 0;
        }
        if (to_delete) {
-          // convert the current key to a delete
-          ikey_.type = kTypeDeletion;
-          current_key_.UpdateInternalKey(ikey_.sequence, kTypeDeletion);
+          // make a copy of the original key and convert it to a delete
+          delete_key_.SetInternalKey(ExtractUserKey(key_), ikey_.sequence,
+                                     kTypeDeletion);
+          // anchor the key again
+          key_ = delete_key_.GetKey();
+          // needed because ikey_ is backed by key
+          ParseInternalKey(key_, &ikey_);
          // no value associated with delete
          value_.clear();
          iter_stats_.num_record_drop_user++;
@@ -166,12 +159,6 @@ void CompactionIterator::NextFromInput() {
          value_ = compaction_filter_value_;
        }
      }
-    } else {
-      // Update the current key to reflect the new sequence number/type without
-      // copying the user key.
-      current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-      key_ = current_key_.GetKey();
-      ikey_.user_key = current_key_.GetUserKey();
    }

    // If there are no snapshots, then this kv affect visibility at tip.
@@ -186,58 +173,7 @@ void CompactionIterator::NextFromInput() {
        visible_at_tip_ ? visible_at_tip_ : findEarliestVisibleSnapshot(
                                                ikey_.sequence, &prev_snapshot);

-    if (ikey_.type == kTypeSingleDeletion) {
-      ParsedInternalKey next_ikey;
-      input_->Next();
-
-      // Check whether the current key is valid, not corrupt and the same
-      // as the single delete.
-      if (input_->Valid() && ParseInternalKey(input_->key(), &next_ikey) &&
-          cmp_->Equal(ikey_.user_key, next_ikey.user_key)) {
-        // Mixing single deletes and merges is not supported. Consecutive
-        // single deletes are not valid.
-        if (next_ikey.type != kTypeValue) {
-          assert(false);
-          status_ =
-              Status::InvalidArgument("Put expected after single delete.");
-          break;
-        }
-
-        // Check whether the current key belongs to the same snapshot as the
-        // single delete.
-        if (prev_snapshot == 0 || next_ikey.sequence > prev_snapshot) {
-          // Found the matching value, we can drop the single delete and the
-          // value.
-          ++iter_stats_.num_record_drop_hidden;
-          ++iter_stats_.num_record_drop_obsolete;
-          input_->Next();
-        } else {
-          // We hit the next snapshot without hitting a put, so the iterator
-          // returns the single delete.
-          valid_ = true;
-        }
-      } else {
-        // We are at the end of the input, could not parse the next key, or hit
-        // the next key. The iterator returns the single delete if the key
-        // possibly exists beyond the current output level.  We set
-        // has_current_user_key to false so that if the iterator is at the next
-        // key, we do not compare it again against the previous key at the next
-        // iteration. If the next key is corrupt, we return before the
-        // comparison, so the value of has_current_user_key does not matter.
-        has_current_user_key_ = false;
-        if (compaction_ != nullptr &&
-            compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
-                                                       &level_ptrs_)) {
-          ++iter_stats_.num_record_drop_obsolete;
-        } else {
-          valid_ = true;
-        }
-      }
-
-      if (valid_) {
-        at_next_ = true;
-      }
-    } else if (last_snapshot == current_user_key_snapshot_) {
+    if (last_snapshot == current_user_key_snapshot_) {
      // If the earliest snapshot is which this key is visible in
      // is the same as the visibility of a previous instance of the
      // same key, then this kv is not visible in any snapshot.
@@ -245,7 +181,6 @@ void CompactionIterator::NextFromInput() {
      // TODO: why not > ?
      assert(last_sequence >= current_user_key_sequence_);
      ++iter_stats_.num_record_drop_hidden;  // (A)
-      input_->Next();
    } else if (compaction_ != nullptr && ikey_.type == kTypeDeletion &&
               ikey_.sequence <= earliest_snapshot_ &&
               compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
@@ -262,7 +197,6 @@ void CompactionIterator::NextFromInput() {
      //     few iterations of this loop (by rule (A) above).
      // Therefore this deletion marker is obsolete and can be dropped.
      ++iter_stats_.num_record_drop_obsolete;
-      input_->Next();
    } else if (ikey_.type == kTypeMerge) {
      if (!merge_helper_->HasOperator()) {
        LogToBuffer(log_buffer_, "Options::merge_operator is null.");
@@ -275,33 +209,27 @@ void CompactionIterator::NextFromInput() {
      // have hit (A)
      // We encapsulate the merge related state machine in a different
      // object to minimize change to the existing flow.
-      merge_helper_->MergeUntil(input_, prev_snapshot, bottommost_level_);
+      merge_helper_->MergeUntil(input_, prev_snapshot, bottommost_level_,
+                                stats_, env_);
      merge_out_iter_.SeekToFirst();

-      if (merge_out_iter_.Valid()) {
-        // NOTE: key, value, and ikey_ refer to old entries.
-        //       These will be correctly set below.
-        key_ = merge_out_iter_.key();
-        value_ = merge_out_iter_.value();
-        bool valid_key __attribute__((__unused__)) =
-            ParseInternalKey(key_, &ikey_);
-        // MergeUntil stops when it encounters a corrupt key and does not
-        // include them in the result, so we expect the keys here to valid.
-        assert(valid_key);
-        // Keep current_key_ in sync.
-        current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-        key_ = current_key_.GetKey();
-        ikey_.user_key = current_key_.GetUserKey();
-        valid_ = true;
-      } else {
-        // all merge operands were filtered out. reset the user key, since the
-        // batch consumed by the merge operator should not shadow any keys
-        // coming after the merges
-        has_current_user_key_ = false;
-      }
+      // NOTE: key, value, and ikey_ refer to old entries.
+      //       These will be correctly set below.
+      key_ = merge_out_iter_.key();
+      value_ = merge_out_iter_.value();
+      bool valid_key __attribute__((__unused__)) =
+          ParseInternalKey(key_, &ikey_);
+      // MergeUntil stops when it encounters a corrupt key and does not
+      // include them in the result, so we expect the keys here to valid.
+      assert(valid_key);
+      valid_ = true;
+      break;
    } else {
      valid_ = true;
+      break;
    }
+
+    input_->Next();
  }
 }

@@ -312,9 +240,12 @@ void CompactionIterator::PrepareOutput() {
  // then we can squash the seqno to zero.
  if (bottommost_level_ && valid_ && ikey_.sequence < earliest_snapshot_ &&
      ikey_.type != kTypeMerge) {
-    assert(ikey_.type != kTypeDeletion && ikey_.type != kTypeSingleDeletion);
-    ikey_.sequence = 0;
-    current_key_.UpdateInternalKey(0, ikey_.type);
+    assert(ikey_.type != kTypeDeletion);
+    // make a copy because updating in place would cause problems
+    // with the priority queue that is managing the input key iterator
+    updated_key_.assign(key_.data(), key_.size());
+    UpdateInternalKey(&updated_key_, (uint64_t)0, ikey_.type);
+    key_ = Slice(updated_key_);
  }
 }

@@ -41,6 +41,7 @@ class CompactionIterator {
                     MergeHelper* merge_helper, SequenceNumber last_sequence,
                     std::vector<SequenceNumber>* snapshots, Env* env,
                     bool expect_valid_internal_key,
+                     Statistics* stats = nullptr,
                     Compaction* compaction = nullptr,
                     const CompactionFilter* compaction_filter = nullptr,
                     LogBuffer* log_buffer = nullptr);
@@ -63,7 +64,7 @@ class CompactionIterator {
  const Status& status() const { return status_; }
  const ParsedInternalKey& ikey() const { return ikey_; }
  bool Valid() const { return valid_; }
-  const Slice& user_key() const { return current_user_key_; }
+  Slice user_key() const { return current_user_key_.GetKey(); }
  const CompactionIteratorStats& iter_stats() const { return iter_stats_; }

 private:
@@ -90,6 +91,7 @@ class CompactionIterator {
  const std::vector<SequenceNumber>* snapshots_;
  Env* env_;
  bool expect_valid_internal_key_;
+  Statistics* stats_;
  Compaction* compaction_;
  const CompactionFilter* compaction_filter_;
  LogBuffer* log_buffer_;
@@ -100,32 +102,18 @@ class CompactionIterator {
  SequenceNumber latest_snapshot_;

  // State
-  //
-  // Points to a copy of the current compaction iterator output (current_key_)
-  // if valid_.
  Slice key_;
-  // Points to the value in the underlying iterator that corresponds to the
-  // current output.
  Slice value_;
-  // The status is OK unless compaction iterator encounters a merge operand
-  // while not having a merge operator defined.
  Status status_;
-  // Stores the user key, sequence number and type of the current compaction
-  // iterator output (or current key in the underlying iterator during
-  // NextFromInput()).
  ParsedInternalKey ikey_;
-  // Stores whether ikey_.user_key is valid. If set to false, the user key is
-  // not compared against the current key in the underlying iterator.
  bool has_current_user_key_ = false;
-  bool at_next_ = false;  // If false, the iterator
-  // Holds a copy of the current compaction iterator output (or current key in
-  // the underlying iterator during NextFromInput()).
-  IterKey current_key_;
-  Slice current_user_key_;
+  IterKey current_user_key_;
  SequenceNumber current_user_key_sequence_;
  SequenceNumber current_user_key_snapshot_;
  MergeOutputIterator merge_out_iter_;
+  std::string updated_key_;
  std::string compaction_filter_value_;
+  IterKey delete_key_;
  // "level_ptrs" holds indices that remember which file of an associated
  // level we were last checking during the last call to compaction->
  // KeyNotExistsBeyondOutputLevel(). This allows future calls to the function
@@ -1,71 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#include "db/compaction_iterator.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace rocksdb {
-
-class CompactionIteratorTest : public testing::Test {
- public:
-  CompactionIteratorTest() : cmp_(BytewiseComparator()), snapshots_({}) {}
-
-  void InitIterator(const std::vector<std::string>& ks,
-                    const std::vector<std::string>& vs,
-                    SequenceNumber last_sequence) {
-    merge_helper_.reset(new MergeHelper(Env::Default(), cmp_, nullptr, nullptr,
-                                        nullptr, 0U, false, 0));
-    iter_.reset(new test::VectorIterator(ks, vs));
-    iter_->SeekToFirst();
-    c_iter_.reset(new CompactionIterator(iter_.get(), cmp_, merge_helper_.get(),
-                                         last_sequence, &snapshots_,
-                                         Env::Default(), false));
-  }
-
-  const Comparator* cmp_;
-  std::vector<SequenceNumber> snapshots_;
-  std::unique_ptr<MergeHelper> merge_helper_;
-  std::unique_ptr<test::VectorIterator> iter_;
-  std::unique_ptr<CompactionIterator> c_iter_;
-};
-
-// It is possible that the output of the compaction iterator is empty even if
-// the input is not.
-TEST_F(CompactionIteratorTest, EmptyResult) {
-  InitIterator({test::KeyStr("a", 5, kTypeSingleDeletion),
-                test::KeyStr("a", 3, kTypeValue)},
-               {"", "val"}, 5);
-  c_iter_->SeekToFirst();
-  ASSERT_FALSE(c_iter_->Valid());
-}
-
-// If there is a corruption after a single deletion, the corrupted key should
-// be preserved.
-TEST_F(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
-  InitIterator({test::KeyStr("a", 5, kTypeSingleDeletion),
-                test::KeyStr("a", 3, kTypeValue, true),
-                test::KeyStr("b", 10, kTypeValue)},
-               {"", "val", "val2"}, 10);
-  c_iter_->SeekToFirst();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("a", 5, kTypeSingleDeletion),
-            c_iter_->key().ToString());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("a", 3, kTypeValue, true), c_iter_->key().ToString());
-  c_iter_->Next();
-  ASSERT_TRUE(c_iter_->Valid());
-  ASSERT_EQ(test::KeyStr("b", 10, kTypeValue), c_iter_->key().ToString());
-  c_iter_->Next();
-  ASSERT_FALSE(c_iter_->Valid());
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
@@ -316,8 +316,6 @@ void CompactionJob::Prepare() {
      Slice* end = i == boundaries_.size() ? nullptr : &boundaries_[i];
      compact_->sub_compact_states.emplace_back(c, start, end, sizes_[i]);
    }
-    MeasureTime(stats_, NUM_SUBCOMPACTIONS_SCHEDULED,
-                compact_->sub_compact_states.size());
  } else {
    compact_->sub_compact_states.emplace_back(c, nullptr, nullptr);
  }
@@ -542,9 +540,8 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
  UpdateCompactionJobStats(stats);

  auto stream = event_logger_->LogToBuffer(log_buffer_);
-  stream << "job" << job_id_
-         << "event" << "compaction_finished"
-         << "compaction_time_micros" << compaction_stats_.micros
+  stream << "job" << job_id_ << "event"
+         << "compaction_finished"
         << "output_level" << compact_->compaction->output_level()
         << "num_output_files" << compact_->NumOutputFiles()
         << "total_output_size" << compact_->total_bytes
@@ -597,6 +594,10 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  }

  ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
+  MergeHelper merge(cfd->user_comparator(), cfd->ioptions()->merge_operator,
+                    db_options_.info_log.get(),
+                    cfd->ioptions()->min_partial_merge_operands,
+                    false /* internal key corruption is expected */);
  auto compaction_filter = cfd->ioptions()->compaction_filter;
  std::unique_ptr<CompactionFilter> compaction_filter_from_factory = nullptr;
  if (compaction_filter == nullptr) {
@@ -604,13 +605,6 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
        sub_compact->compaction->CreateCompactionFilter();
    compaction_filter = compaction_filter_from_factory.get();
  }
-  MergeHelper merge(
-      env_, cfd->user_comparator(), cfd->ioptions()->merge_operator,
-      compaction_filter, db_options_.info_log.get(),
-      cfd->ioptions()->min_partial_merge_operands,
-      false /* internal key corruption is expected */,
-      existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
-      compact_->compaction->level(), db_options_.statistics.get());

  TEST_SYNC_POINT("CompactionJob::Run():Inprogress");

@@ -627,8 +621,8 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
  Status status;
  sub_compact->c_iter.reset(new CompactionIterator(
      input.get(), cfd->user_comparator(), &merge, versions_->LastSequence(),
-      &existing_snapshots_, env_, false, sub_compact->compaction,
-      compaction_filter));
+      &existing_snapshots_, env_, false, db_options_.statistics.get(),
+      sub_compact->compaction, compaction_filter));
  auto c_iter = sub_compact->c_iter.get();
  c_iter->SeekToFirst();
  const auto& c_iter_stats = c_iter->iter_stats();
@@ -1029,14 +1029,6 @@ int main(int argc, char** argv) {
  return RUN_ALL_TESTS();
 }

-#else
-#include <stdio.h>
-
-int main(int argc, char** argv) {
-  fprintf(stderr, "SKIPPED, not supported in ROCKSDB_LITE\n");
-  return 0;
-}
-
 #endif  // !ROCKSDB_LITE

 #else
@@ -182,7 +182,7 @@ class CompactionJobTest : public testing::Test {
    return expected_results;
  }

-  void NewDB() {
+  void NewDB(std::shared_ptr<MergeOperator> merge_operator = nullptr) {
    VersionEdit new_db;
    new_db.SetLogNumber(0);
    new_db.SetNextFile(2);
@@ -207,8 +207,7 @@ class CompactionJobTest : public testing::Test {

    std::vector<ColumnFamilyDescriptor> column_families;
    cf_options_.table_factory = mock_table_factory_;
-    cf_options_.merge_operator = merge_op_;
-    cf_options_.compaction_filter = compaction_filter_.get();
+    cf_options_.merge_operator = merge_operator;
    column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);

    EXPECT_OK(versions_->Recover(column_families, false));
@@ -216,8 +215,7 @@ class CompactionJobTest : public testing::Test {
  }

  void RunCompaction(const std::vector<std::vector<FileMetaData*>>& input_files,
-                     const stl_wrappers::KVMap& expected_results,
-                     const std::vector<SequenceNumber>& snapshots = {}) {
+                     const stl_wrappers::KVMap& expected_results) {
    auto cfd = versions_->GetColumnFamilySet()->GetDefault();

    size_t num_input_files = 0;
@@ -243,9 +241,9 @@ class CompactionJobTest : public testing::Test {
    EventLogger event_logger(db_options_.info_log.get());
    CompactionJob compaction_job(0, &compaction, db_options_, env_options_,
                                 versions_.get(), &shutting_down_, &log_buffer,
-                                 nullptr, nullptr, nullptr, snapshots,
-                                 table_cache_, &event_logger, false, false,
-                                 dbname_, &compaction_job_stats_);
+                                 nullptr, nullptr, nullptr, {}, table_cache_,
+                                 &event_logger, false, false, dbname_,
+                                 &compaction_job_stats_);

    VerifyInitializationOfCompactionJobStats(compaction_job_stats_);

@@ -259,16 +257,10 @@ class CompactionJobTest : public testing::Test {
                                     &mutex_));
    mutex_.Unlock();

-    if (expected_results.size() == 0) {
-      ASSERT_GE(compaction_job_stats_.elapsed_micros, 0U);
-      ASSERT_EQ(compaction_job_stats_.num_input_files, num_input_files);
-      ASSERT_EQ(compaction_job_stats_.num_output_files, 0U);
-    } else {
-      ASSERT_GE(compaction_job_stats_.elapsed_micros, 0U);
-      ASSERT_EQ(compaction_job_stats_.num_input_files, num_input_files);
-      ASSERT_EQ(compaction_job_stats_.num_output_files, 1U);
-      mock_table_factory_->AssertLatestFile(expected_results);
-    }
+    ASSERT_GE(compaction_job_stats_.elapsed_micros, 0U);
+    ASSERT_EQ(compaction_job_stats_.num_input_files, num_input_files);
+    ASSERT_EQ(compaction_job_stats_.num_output_files, 1U);
+    mock_table_factory_->AssertLatestFile(expected_results);
  }

  Env* env_;
@@ -286,8 +278,6 @@ class CompactionJobTest : public testing::Test {
  std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
  CompactionJobStats compaction_job_stats_;
  ColumnFamilyData* cfd_;
-  std::unique_ptr<CompactionFilter> compaction_filter_;
-  std::shared_ptr<MergeOperator> merge_op_;
 };

 TEST_F(CompactionJobTest, Simple) {
@@ -306,7 +296,7 @@ TEST_F(CompactionJobTest, SimpleCorrupted) {
  auto expected_results = CreateTwoFiles(true);
  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
  auto files = cfd->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results);
+  RunCompaction({ files }, expected_results);
  ASSERT_EQ(compaction_job_stats_.num_corrupt_keys, 400U);
 }

@@ -326,7 +316,7 @@ TEST_F(CompactionJobTest, SimpleDeletion) {

  SetLastSequence(4U);
  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results);
+  RunCompaction({ files }, expected_results);
 }

 TEST_F(CompactionJobTest, SimpleOverwrite) {
@@ -348,7 +338,7 @@ TEST_F(CompactionJobTest, SimpleOverwrite) {

  SetLastSequence(4U);
  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results);
+  RunCompaction({ files }, expected_results);
 }

 TEST_F(CompactionJobTest, SimpleNonLastLevel) {
@@ -377,12 +367,12 @@ TEST_F(CompactionJobTest, SimpleNonLastLevel) {
  SetLastSequence(6U);
  auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
  auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1);
-  RunCompaction({lvl0_files, lvl1_files}, expected_results);
+  RunCompaction({ lvl0_files, lvl1_files }, expected_results);
 }

 TEST_F(CompactionJobTest, SimpleMerge) {
-  merge_op_ = MergeOperators::CreateStringAppendOperator();
-  NewDB();
+  auto merge_op = MergeOperators::CreateStringAppendOperator();
+  NewDB(merge_op);

  auto file1 = mock::MakeMockFile({
      {KeyStr("a", 5U, kTypeMerge), "5"},
@@ -401,12 +391,12 @@ TEST_F(CompactionJobTest, SimpleMerge) {

  SetLastSequence(5U);
  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results);
+  RunCompaction({ files }, expected_results);
 }

 TEST_F(CompactionJobTest, NonAssocMerge) {
-  merge_op_ = MergeOperators::CreateStringAppendTESTOperator();
-  NewDB();
+  auto merge_op = MergeOperators::CreateStringAppendTESTOperator();
+  NewDB(merge_op);

  auto file1 = mock::MakeMockFile({
      {KeyStr("a", 5U, kTypeMerge), "5"},
@@ -426,267 +416,7 @@ TEST_F(CompactionJobTest, NonAssocMerge) {

  SetLastSequence(5U);
  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results);
-}
-
-// Filters merge operands with value 10.
-TEST_F(CompactionJobTest, MergeOperandFilter) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
-  compaction_filter_.reset(new test::FilterNumber(10U));
-  NewDB();
-
-  auto file1 = mock::MakeMockFile(
-      {{KeyStr("a", 5U, kTypeMerge), test::EncodeInt(5U)},
-       {KeyStr("a", 4U, kTypeMerge), test::EncodeInt(10U)},  // Filtered
-       {KeyStr("a", 3U, kTypeMerge), test::EncodeInt(3U)}});
-  AddMockFile(file1);
-
-  auto file2 = mock::MakeMockFile({
-      {KeyStr("b", 2U, kTypeMerge), test::EncodeInt(2U)},
-      {KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)}  // Filtered
-  });
-  AddMockFile(file2);
-
-  auto expected_results =
-      mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), test::EncodeInt(8U)},
-                          {KeyStr("b", 2U, kTypeMerge), test::EncodeInt(2U)}});
-
-  SetLastSequence(5U);
-  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results);
-}
-
-TEST_F(CompactionJobTest, FilterSomeMergeOperands) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
-  compaction_filter_.reset(new test::FilterNumber(10U));
-  NewDB();
-
-  auto file1 = mock::MakeMockFile(
-      {{KeyStr("a", 5U, kTypeMerge), test::EncodeInt(5U)},
-       {KeyStr("a", 4U, kTypeMerge), test::EncodeInt(10U)},  // Filtered
-       {KeyStr("a", 3U, kTypeValue), test::EncodeInt(5U)},
-       {KeyStr("d", 8U, kTypeMerge), test::EncodeInt(10U)}});
-  AddMockFile(file1);
-
-  auto file2 =
-      mock::MakeMockFile({{KeyStr("b", 2U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("c", 2U, kTypeMerge), test::EncodeInt(3U)},
-                          {KeyStr("c", 1U, kTypeValue), test::EncodeInt(7U)},
-                          {KeyStr("d", 1U, kTypeValue), test::EncodeInt(6U)}});
-  AddMockFile(file2);
-
-  auto file3 =
-      mock::MakeMockFile({{KeyStr("a", 1U, kTypeMerge), test::EncodeInt(3U)}});
-  AddMockFile(file3, 2);
-
-  auto expected_results = mock::MakeMockFile({
-      {KeyStr("a", 5U, kTypeValue), test::EncodeInt(10U)},
-      {KeyStr("c", 2U, kTypeValue), test::EncodeInt(10U)},
-      {KeyStr("d", 1U, kTypeValue), test::EncodeInt(6U)}
-      // b does not appear because the operands are filtered
-  });
-
-  SetLastSequence(5U);
-  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results);
-}
-
-// Test where all operands/merge results are filtered out.
-TEST_F(CompactionJobTest, FilterAllMergeOperands) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
-  compaction_filter_.reset(new test::FilterNumber(10U));
-  NewDB();
-
-  auto file1 =
-      mock::MakeMockFile({{KeyStr("a", 11U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("a", 10U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("a", 9U, kTypeMerge), test::EncodeInt(10U)}});
-  AddMockFile(file1);
-
-  auto file2 =
-      mock::MakeMockFile({{KeyStr("b", 8U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("b", 7U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("b", 6U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("b", 5U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("b", 4U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("b", 3U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("b", 2U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("c", 2U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("c", 1U, kTypeMerge), test::EncodeInt(10U)}});
-  AddMockFile(file2);
-
-  auto file3 =
-      mock::MakeMockFile({{KeyStr("a", 2U, kTypeMerge), test::EncodeInt(10U)},
-                          {KeyStr("b", 1U, kTypeMerge), test::EncodeInt(10U)}});
-  AddMockFile(file3, 2);
-
-  SetLastSequence(11U);
-  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-
-  stl_wrappers::KVMap empty_map;
-  RunCompaction({files}, empty_map);
-}
-
-TEST_F(CompactionJobTest, SimpleSingleDelete) {
-  NewDB();
-
-  auto file1 = mock::MakeMockFile({
-      {KeyStr("a", 5U, kTypeDeletion), ""},
-      {KeyStr("b", 6U, kTypeSingleDeletion), ""},
-  });
-  AddMockFile(file1);
-
-  auto file2 = mock::MakeMockFile({{KeyStr("a", 3U, kTypeValue), "val"},
-                                   {KeyStr("b", 4U, kTypeValue), "val"}});
-  AddMockFile(file2);
-
-  auto file3 = mock::MakeMockFile({
-      {KeyStr("a", 1U, kTypeValue), "val"},
-  });
-  AddMockFile(file3, 2);
-
-  auto expected_results =
-      mock::MakeMockFile({{KeyStr("a", 5U, kTypeDeletion), ""}});
-
-  SetLastSequence(6U);
-  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results);
-}
-
-TEST_F(CompactionJobTest, SingleDeleteSnapshots) {
-  NewDB();
-
-  auto file1 = mock::MakeMockFile({{KeyStr("A", 12U, kTypeSingleDeletion), ""},
-                                   {KeyStr("a", 12U, kTypeSingleDeletion), ""},
-                                   {KeyStr("b", 21U, kTypeSingleDeletion), ""},
-                                   {KeyStr("c", 22U, kTypeSingleDeletion), ""},
-                                   {KeyStr("d", 9U, kTypeSingleDeletion), ""}});
-  AddMockFile(file1);
-
-  auto file2 = mock::MakeMockFile({{KeyStr("0", 2U, kTypeSingleDeletion), ""},
-                                   {KeyStr("a", 11U, kTypeValue), "val1"},
-                                   {KeyStr("b", 11U, kTypeValue), "val2"},
-                                   {KeyStr("c", 21U, kTypeValue), "val3"},
-                                   {KeyStr("d", 8U, kTypeValue), "val4"},
-                                   {KeyStr("e", 2U, kTypeSingleDeletion), ""}});
-  AddMockFile(file2);
-
-  auto file3 = mock::MakeMockFile({{KeyStr("A", 1U, kTypeValue), "val"},
-                                   {KeyStr("e", 1U, kTypeValue), "val"}});
-  AddMockFile(file3, 2);
-
-  auto expected_results =
-      mock::MakeMockFile({{KeyStr("A", 12U, kTypeSingleDeletion), ""},
-                          {KeyStr("b", 21U, kTypeSingleDeletion), ""},
-                          {KeyStr("b", 11U, kTypeValue), "val2"},
-                          {KeyStr("e", 2U, kTypeSingleDeletion), ""}});
-
-  SetLastSequence(22U);
-  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results, {10U, 20U});
-}
-
-TEST_F(CompactionJobTest, SingleDeleteZeroSeq) {
-  NewDB();
-
-  auto file1 = mock::MakeMockFile({
-      {KeyStr("A", 10U, kTypeSingleDeletion), ""},
-      {KeyStr("dummy", 5U, kTypeValue), "val2"},
-  });
-  AddMockFile(file1);
-
-  auto file2 = mock::MakeMockFile({
-      {KeyStr("A", 0U, kTypeValue), "val"},
-  });
-  AddMockFile(file2);
-
-  auto expected_results = mock::MakeMockFile({
-      {KeyStr("dummy", 0U, kTypeValue), "val2"},
-  });
-
-  SetLastSequence(22U);
-  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results, {});
-}
-
-TEST_F(CompactionJobTest, MultiSingleDelete) {
-  // Tests three scenarios involving multiple single delete/put pairs:
-  //
-  // A: Put Snapshot SDel Put SDel -> Put Snapshot SDel
-  // B: Put SDel Put SDel -> (Removed)
-  // C: SDel Put SDel Snapshot Put -> Snapshot Put
-  // D: (Put) SDel Snapshot Put SDel -> (Put) SDel Snapshot
-  NewDB();
-
-  auto file1 = mock::MakeMockFile({
-      {KeyStr("A", 14U, kTypeSingleDeletion), ""},
-      {KeyStr("A", 13U, kTypeValue), "val5"},
-      {KeyStr("A", 12U, kTypeSingleDeletion), ""},
-      {KeyStr("B", 14U, kTypeSingleDeletion), ""},
-      {KeyStr("B", 13U, kTypeValue), "val2"},
-      {KeyStr("C", 14U, kTypeValue), "val3"},
-      {KeyStr("D", 12U, kTypeSingleDeletion), ""},
-      {KeyStr("D", 11U, kTypeValue), "val4"},
-  });
-  AddMockFile(file1);
-
-  auto file2 = mock::MakeMockFile({
-      {KeyStr("A", 10U, kTypeValue), "val"},
-      {KeyStr("B", 12U, kTypeSingleDeletion), ""},
-      {KeyStr("B", 11U, kTypeValue), "val2"},
-      {KeyStr("C", 10U, kTypeSingleDeletion), ""},
-      {KeyStr("C", 9U, kTypeValue), "val6"},
-      {KeyStr("C", 8U, kTypeSingleDeletion), ""},
-      {KeyStr("D", 10U, kTypeSingleDeletion), ""},
-  });
-  AddMockFile(file2);
-
-  auto file3 = mock::MakeMockFile({
-      {KeyStr("D", 11U, kTypeValue), "val"},
-  });
-  AddMockFile(file3, 2);
-
-  auto expected_results = mock::MakeMockFile({
-      {KeyStr("A", 12U, kTypeSingleDeletion), ""},
-      {KeyStr("A", 10U, kTypeValue), "val"},
-      {KeyStr("C", 14U, kTypeValue), "val3"},
-      {KeyStr("D", 10U, kTypeSingleDeletion), ""},
-  });
-
-  SetLastSequence(22U);
-  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results, {10U});
-}
-
-// This test documents the behavior where a corrupt key follows a deletion or a
-// single deletion and the (single) deletion gets removed while the corrupt key
-// gets written out. TODO(noetzli): We probably want a better way to treat
-// corrupt keys.
-TEST_F(CompactionJobTest, CorruptionAfterDeletion) {
-  NewDB();
-
-  auto file1 =
-      mock::MakeMockFile({{test::KeyStr("A", 6U, kTypeValue), "val3"},
-                          {test::KeyStr("a", 5U, kTypeDeletion), ""},
-                          {test::KeyStr("a", 4U, kTypeValue, true), "val"}});
-  AddMockFile(file1);
-
-  auto file2 =
-      mock::MakeMockFile({{test::KeyStr("b", 3U, kTypeSingleDeletion), ""},
-                          {test::KeyStr("b", 2U, kTypeValue, true), "val"},
-                          {test::KeyStr("c", 1U, kTypeValue), "val2"}});
-  AddMockFile(file2);
-
-  auto expected_results =
-      mock::MakeMockFile({{test::KeyStr("A", 0U, kTypeValue), "val3"},
-                          {test::KeyStr("a", 0U, kTypeValue, true), "val"},
-                          {test::KeyStr("b", 0U, kTypeValue, true), "val"},
-                          {test::KeyStr("c", 0U, kTypeValue), "val2"}});
-
-  SetLastSequence(6U);
-  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results);
+  RunCompaction({ files }, expected_results);
 }

 }  // namespace rocksdb
@@ -1028,7 +1028,7 @@ bool LevelCompactionPicker::PickCompactionBySize(VersionStorageInfo* vstorage,

  // Pick the largest file in this level that is not already
  // being compacted
-  const std::vector<int>& file_size = vstorage->FilesByCompactionPri(level);
+  const std::vector<int>& file_size = vstorage->FilesBySize(level);
  const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(level);

  // record the first file that is not yet compacted
@@ -1039,6 +1039,11 @@ bool LevelCompactionPicker::PickCompactionBySize(VersionStorageInfo* vstorage,
    int index = file_size[i];
    auto* f = level_files[index];

+    assert((i == file_size.size() - 1) ||
+           (i >= VersionStorageInfo::kNumberFilesToSort - 1) ||
+           (f->compensated_file_size >=
+            level_files[file_size[i + 1]]->compensated_file_size));
+
    // do not pick a file to compact if it is being compacted
    // from n-1 level.
    if (f->being_compacted) {
@@ -1317,13 +1322,7 @@ Compaction* UniversalCompactionPicker::PickCompaction(
                              &largest_seqno);
      if (is_first) {
        is_first = false;
-      } else if (prev_smallest_seqno > 0) {
-        // A level is considered as the bottommost level if there are
-        // no files in higher levels or if files in higher levels do
-        // not overlap with the files being compacted. Sequence numbers
-        // of files in bottommost level can be set to 0 to help
-        // compression. As a result, the following assert may not hold
-        // if the prev_smallest_seqno is 0.
+      } else {
        assert(prev_smallest_seqno > largest_seqno);
      }
      prev_smallest_seqno = smallest_seqno;
@@ -110,11 +110,6 @@ class CompactionPicker {
  // overlapping.
  bool IsInputNonOverlapping(Compaction* c);

-  // Is there currently a compaction involving level 0 taking place
-  bool IsLevel0CompactionInProgress() const {
-    return !level0_compactions_in_progress_.empty();
-  }
-
 protected:
  int NumberLevels() const { return ioptions_.num_levels; }

@@ -3,12 +3,9 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.

-#include "db/compaction.h"
 #include "db/compaction_picker.h"
 #include <limits>
 #include <string>
-#include <utility>
-
 #include "util/logging.h"
 #include "util/string_util.h"
 #include "util/testharness.h"
@@ -38,11 +35,6 @@ class CompactionPickerTest : public testing::Test {
  CompactionOptionsFIFO fifo_options_;
  std::unique_ptr<VersionStorageInfo> vstorage_;
  std::vector<std::unique_ptr<FileMetaData>> files_;
-  // does not own FileMetaData
-  std::unordered_map<uint32_t, std::pair<FileMetaData*, int>> file_map_;
-  // input files to compaction process.
-  std::vector<CompactionInputFiles> input_files_;
-  int compaction_level_start_;

  CompactionPickerTest()
      : ucmp_(BytewiseComparator()),
@@ -74,8 +66,6 @@ class CompactionPickerTest : public testing::Test {
  void DeleteVersionStorage() {
    vstorage_.reset();
    files_.clear();
-    file_map_.clear();
-    input_files_.clear();
  }

  void Add(int level, uint32_t file_number, const char* smallest,
@@ -93,29 +83,11 @@ class CompactionPickerTest : public testing::Test {
    f->refs = 0;
    vstorage_->AddFile(level, f);
    files_.emplace_back(f);
-    file_map_.insert({file_number, {f, level}});
-  }
-
-  void SetCompactionInputFilesLevels(int level_count, int start_level) {
-    input_files_.resize(level_count);
-    for (int i = 0; i < level_count; ++i) {
-      input_files_[i].level = start_level + i;
-    }
-    compaction_level_start_ = start_level;
-  }
-
-  void AddToCompactionFiles(uint32_t file_number) {
-    auto iter = file_map_.find(file_number);
-    assert(iter != file_map_.end());
-    int level = iter->second.second;
-    assert(level < vstorage_->num_levels());
-    input_files_[level - compaction_level_start_].files.emplace_back(
-        iter->second.first);
  }

  void UpdateVersionStorageInfo() {
    vstorage_->CalculateBaseBytes(ioptions_, mutable_cf_options_);
-    vstorage_->UpdateFilesByCompactionPri(mutable_cf_options_);
+    vstorage_->UpdateFilesBySize();
    vstorage_->UpdateNumNonEmptyLevels();
    vstorage_->GenerateFileIndexer();
    vstorage_->GenerateLevelFilesBrief();
@@ -665,164 +637,6 @@ TEST_F(CompactionPickerTest, EstimateCompactionBytesNeededDynamicLevel) {
            vstorage_->estimated_compaction_needed_bytes());
 }

-TEST_F(CompactionPickerTest, IsBottommostLevelTest) {
-  // case 1: Higher levels are empty
-  NewVersionStorage(6, kCompactionStyleLevel);
-  Add(0, 1U, "a", "m");
-  Add(0, 2U, "c", "z");
-  Add(1, 3U, "d", "e");
-  Add(1, 4U, "l", "p");
-  Add(2, 5U, "g", "i");
-  Add(2, 6U, "x", "z");
-  UpdateVersionStorageInfo();
-  SetCompactionInputFilesLevels(2, 1);
-  AddToCompactionFiles(3U);
-  AddToCompactionFiles(5U);
-  bool result =
-      Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
-  ASSERT_TRUE(result);
-
-  // case 2: Higher levels have no overlap
-  NewVersionStorage(6, kCompactionStyleLevel);
-  Add(0, 1U, "a", "m");
-  Add(0, 2U, "c", "z");
-  Add(1, 3U, "d", "e");
-  Add(1, 4U, "l", "p");
-  Add(2, 5U, "g", "i");
-  Add(2, 6U, "x", "z");
-  Add(3, 7U, "k", "p");
-  Add(3, 8U, "t", "w");
-  Add(4, 9U, "a", "b");
-  Add(5, 10U, "c", "cc");
-  UpdateVersionStorageInfo();
-  SetCompactionInputFilesLevels(2, 1);
-  AddToCompactionFiles(3U);
-  AddToCompactionFiles(5U);
-  result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
-  ASSERT_TRUE(result);
-
-  // case 3.1: Higher levels (level 3) have overlap
-  NewVersionStorage(6, kCompactionStyleLevel);
-  Add(0, 1U, "a", "m");
-  Add(0, 2U, "c", "z");
-  Add(1, 3U, "d", "e");
-  Add(1, 4U, "l", "p");
-  Add(2, 5U, "g", "i");
-  Add(2, 6U, "x", "z");
-  Add(3, 7U, "e", "g");
-  Add(3, 8U, "h", "k");
-  Add(4, 9U, "a", "b");
-  Add(5, 10U, "c", "cc");
-  UpdateVersionStorageInfo();
-  SetCompactionInputFilesLevels(2, 1);
-  AddToCompactionFiles(3U);
-  AddToCompactionFiles(5U);
-  result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
-  ASSERT_FALSE(result);
-
-  // case 3.2: Higher levels (level 5) have overlap
-  DeleteVersionStorage();
-  NewVersionStorage(6, kCompactionStyleLevel);
-  Add(0, 1U, "a", "m");
-  Add(0, 2U, "c", "z");
-  Add(1, 3U, "d", "e");
-  Add(1, 4U, "l", "p");
-  Add(2, 5U, "g", "i");
-  Add(2, 6U, "x", "z");
-  Add(3, 7U, "j", "k");
-  Add(3, 8U, "l", "m");
-  Add(4, 9U, "a", "b");
-  Add(5, 10U, "c", "cc");
-  Add(5, 11U, "h", "k");
-  Add(5, 12U, "y", "yy");
-  Add(5, 13U, "z", "zz");
-  UpdateVersionStorageInfo();
-  SetCompactionInputFilesLevels(2, 1);
-  AddToCompactionFiles(3U);
-  AddToCompactionFiles(5U);
-  result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
-  ASSERT_FALSE(result);
-
-  // case 3.3: Higher levels (level 5) have overlap, but it's only overlapping
-  // one key ("d")
-  NewVersionStorage(6, kCompactionStyleLevel);
-  Add(0, 1U, "a", "m");
-  Add(0, 2U, "c", "z");
-  Add(1, 3U, "d", "e");
-  Add(1, 4U, "l", "p");
-  Add(2, 5U, "g", "i");
-  Add(2, 6U, "x", "z");
-  Add(3, 7U, "j", "k");
-  Add(3, 8U, "l", "m");
-  Add(4, 9U, "a", "b");
-  Add(5, 10U, "c", "cc");
-  Add(5, 11U, "ccc", "d");
-  Add(5, 12U, "y", "yy");
-  Add(5, 13U, "z", "zz");
-  UpdateVersionStorageInfo();
-  SetCompactionInputFilesLevels(2, 1);
-  AddToCompactionFiles(3U);
-  AddToCompactionFiles(5U);
-  result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
-  ASSERT_FALSE(result);
-
-  // Level 0 files overlap
-  NewVersionStorage(6, kCompactionStyleLevel);
-  Add(0, 1U, "s", "t");
-  Add(0, 2U, "a", "m");
-  Add(0, 3U, "b", "z");
-  Add(0, 4U, "e", "f");
-  Add(5, 10U, "y", "z");
-  UpdateVersionStorageInfo();
-  SetCompactionInputFilesLevels(1, 0);
-  AddToCompactionFiles(1U);
-  AddToCompactionFiles(2U);
-  AddToCompactionFiles(3U);
-  AddToCompactionFiles(4U);
-  result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
-  ASSERT_FALSE(result);
-
-  // Level 0 files don't overlap
-  NewVersionStorage(6, kCompactionStyleLevel);
-  Add(0, 1U, "s", "t");
-  Add(0, 2U, "a", "m");
-  Add(0, 3U, "b", "k");
-  Add(0, 4U, "e", "f");
-  Add(5, 10U, "y", "z");
-  UpdateVersionStorageInfo();
-  SetCompactionInputFilesLevels(1, 0);
-  AddToCompactionFiles(1U);
-  AddToCompactionFiles(2U);
-  AddToCompactionFiles(3U);
-  AddToCompactionFiles(4U);
-  result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
-  ASSERT_TRUE(result);
-
-  // Level 1 files overlap
-  NewVersionStorage(6, kCompactionStyleLevel);
-  Add(0, 1U, "s", "t");
-  Add(0, 2U, "a", "m");
-  Add(0, 3U, "b", "k");
-  Add(0, 4U, "e", "f");
-  Add(1, 5U, "a", "m");
-  Add(1, 6U, "n", "o");
-  Add(1, 7U, "w", "y");
-  Add(5, 10U, "y", "z");
-  UpdateVersionStorageInfo();
-  SetCompactionInputFilesLevels(2, 0);
-  AddToCompactionFiles(1U);
-  AddToCompactionFiles(2U);
-  AddToCompactionFiles(3U);
-  AddToCompactionFiles(4U);
-  AddToCompactionFiles(5U);
-  AddToCompactionFiles(6U);
-  AddToCompactionFiles(7U);
-  result = Compaction::TEST_IsBottommostLevel(2, vstorage_.get(), input_files_);
-  ASSERT_FALSE(result);
-
-  DeleteVersionStorage();
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -7,8 +7,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#ifndef ROCKSDB_LITE
-
 #include "rocksdb/db.h"

 #include <errno.h>
@@ -87,14 +85,10 @@ class CorruptionTest : public testing::Test {
    ASSERT_OK(::rocksdb::RepairDB(dbname_, options_));
  }

-  void Build(int n, int flush_every = 0) {
+  void Build(int n) {
    std::string key_space, value_space;
    WriteBatch batch;
    for (int i = 0; i < n; i++) {
-      if (flush_every != 0 && i != 0 && i % flush_every == 0) {
-        DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
-        dbi->TEST_FlushMemTable();
-      }
      //if ((i % 100) == 0) fprintf(stderr, "@ %d of %d\n", i, n);
      Slice key = Key(i, &key_space);
      batch.Clear();
@@ -301,21 +295,13 @@ TEST_F(CorruptionTest, TableFile) {
 }

 TEST_F(CorruptionTest, TableFileIndexData) {
-  Options options;
-  // very big, we'll trigger flushes manually
-  options.write_buffer_size = 100 * 1024 * 1024;
-  Reopen(&options);
-  // build 2 tables, flush at 5000
-  Build(10000, 5000);
+  Build(10000);  // Enough to build multiple Tables
  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  dbi->TEST_FlushMemTable();

-  // corrupt an index block of an entire file
  Corrupt(kTableFile, -2000, 500);
  Reopen();
-  // one full file should be readable, since only one was corrupted
-  // the other file should be fully non-readable, since index was corrupted
-  Check(5000, 5000);
+  Check(5000, 9999);
 }

 TEST_F(CorruptionTest, MissingDescriptor) {
@@ -481,13 +467,3 @@ int main(int argc, char** argv) {
  ::testing::InitGoogleTest(&argc, argv);
  return RUN_ALL_TESTS();
 }
-
-#else
-#include <stdio.h>
-
-int main(int argc, char** argv) {
-  fprintf(stderr, "SKIPPED as RepairDB() is not supported in ROCKSDB_LITE\n");
-  return 0;
-}
-
-#endif  // !ROCKSDB_LITE
@@ -110,8 +110,7 @@ DEFINE_string(benchmarks,
              "uncompress,"
              "acquireload,"
              "fillseekseq,"
-              "randomtransaction,"
-              "randomreplacekeys",
+              "randomtransaction",

              "Comma-separated list of operations to run in the specified"
              " order. Available benchmarks:\n"
@@ -162,8 +161,6 @@ DEFINE_string(benchmarks,
              "them by seeking to each key\n"
              "\trandomtransaction     -- execute N random transactions and "
              "verify correctness\n"
-              "\trandomreplacekeys     -- randomly replaces N keys by deleting "
-              "the old version and putting the new version\n\n"
              "Meta operations:\n"
              "\tcompact     -- Compact the entire DB\n"
              "\tstats       -- Print DB stats\n"
@@ -314,10 +311,6 @@ static rocksdb::CompactionStyle FLAGS_compaction_style_e;
 DEFINE_int32(compaction_style, (int32_t) rocksdb::Options().compaction_style,
             "style of compaction: level-based vs universal");

-static rocksdb::CompactionPri FLAGS_compaction_pri_e;
-DEFINE_int32(compaction_pri, (int32_t)rocksdb::Options().compaction_style,
-             "priority of files to compaction: by size or by data age");
-
 DEFINE_int32(universal_size_ratio, 0,
             "Percentage flexibility while comparing file size"
             " (for universal compaction only).");
@@ -364,10 +357,6 @@ DEFINE_int32(open_files, rocksdb::Options().max_open_files,
             "Maximum number of files to keep open at the same time"
             " (use default if == 0)");

-DEFINE_int32(file_opening_threads, rocksdb::Options().max_file_opening_threads,
-             "If open_files is set to -1, this option set the number of "
-             "threads that will be used to open files during DB::Open()");
-
 DEFINE_int32(new_table_reader_for_compaction_inputs, true,
             "If true, uses a separate file handle for compaction inputs");

@@ -622,10 +611,9 @@ static bool ValidateRateLimit(const char* flagname, double value) {
 }
 DEFINE_double(soft_rate_limit, 0.0, "");

-DEFINE_double(hard_rate_limit, 0.0, "DEPRECATED");
-
-DEFINE_uint64(hard_pending_compaction_bytes_limit, 128u * 1024 * 1024 * 1024,
-              "Stop writes if pending compaction bytes exceed this number");
+DEFINE_double(hard_rate_limit, 0.0, "When not equal to 0 this make threads "
+              "sleep at each stats reporting interval until the compaction"
+              " score for all levels is less than or equal to this value.");

 DEFINE_uint64(delayed_write_rate, 2097152u,
              "Limited bytes allowed to DB when soft_rate_limit or "
@@ -699,13 +687,6 @@ DEFINE_uint64(wal_bytes_per_sync,  rocksdb::Options().wal_bytes_per_sync,
 DEFINE_bool(filter_deletes, false, " On true, deletes use bloom-filter and drop"
            " the delete if key not present");

-DEFINE_bool(use_single_deletes, true,
-            "Use single deletes (used in RandomReplaceKeys only).");
-
-DEFINE_double(stddev, 2000.0,
-              "Standard deviation of normal distribution used for picking keys"
-              " (used in RandomReplaceKeys only).");
-
 DEFINE_int32(max_successive_merges, 0, "Maximum number of successive merge"
             " operations on a key in the memtable");

@@ -917,7 +898,6 @@ class ReportFileOpEnv : public EnvWrapper {
        return rv;
      }

-      Status Truncate(uint64_t size) override { return target_->Truncate(size); }
      Status Close() override { return target_->Close(); }
      Status Flush() override { return target_->Flush(); }
      Status Sync() override { return target_->Sync(); }
@@ -1943,9 +1923,6 @@ class Benchmark {
      } else if (name == "randomtransaction") {
        method = &Benchmark::RandomTransaction;
        post_process_method = &Benchmark::RandomTransactionVerify;
-      } else if (name == "randomreplacekeys") {
-        fresh_db = true;
-        method = &Benchmark::RandomReplaceKeys;
      } else if (name == "stats") {
        PrintStats("rocksdb.stats");
      } else if (name == "levelstats") {
@@ -2256,7 +2233,6 @@ class Benchmark {
    options.max_subcompactions = static_cast<uint32_t>(FLAGS_subcompactions);
    options.max_background_flushes = FLAGS_max_background_flushes;
    options.compaction_style = FLAGS_compaction_style_e;
-    options.compaction_pri = FLAGS_compaction_pri_e;
    if (FLAGS_prefix_size != 0) {
      options.prefix_extractor.reset(
          NewFixedPrefixTransform(FLAGS_prefix_size));
@@ -2271,7 +2247,6 @@ class Benchmark {
    options.memtable_prefix_bloom_bits = FLAGS_memtable_bloom_bits;
    options.bloom_locality = FLAGS_bloom_locality;
    options.max_open_files = FLAGS_open_files;
-    options.max_file_opening_threads = FLAGS_file_opening_threads;
    options.new_table_reader_for_compaction_inputs =
        FLAGS_new_table_reader_for_compaction_inputs;
    options.compaction_readahead_size = FLAGS_compaction_readahead_size;
@@ -2455,8 +2430,6 @@ class Benchmark {
    }
    options.soft_rate_limit = FLAGS_soft_rate_limit;
    options.hard_rate_limit = FLAGS_hard_rate_limit;
-    options.hard_pending_compaction_bytes_limit =
-        FLAGS_hard_pending_compaction_bytes_limit;
    options.delayed_write_rate = FLAGS_delayed_write_rate;
    options.rate_limit_delay_max_milliseconds =
      FLAGS_rate_limit_delay_max_milliseconds;
@@ -3869,66 +3842,6 @@ class Benchmark {
    fprintf(stdout, "RandomTransactionVerify Success!\n");
  }

-  // Writes and deletes random keys without overwriting keys.
-  //
-  // This benchmark is intended to partially replicate the behavior of MyRocks
-  // secondary indices: All data is stored in keys and updates happen by
-  // deleting the old version of the key and inserting the new version.
-  void RandomReplaceKeys(ThreadState* thread) {
-    std::unique_ptr<const char[]> key_guard;
-    Slice key = AllocateKey(&key_guard);
-    std::vector<uint32_t> counters(FLAGS_numdistinct, 0);
-    size_t max_counter = 50;
-    RandomGenerator gen;
-
-    Status s;
-    DB* db = SelectDB(thread);
-    for (int64_t i = 0; i < FLAGS_numdistinct; i++) {
-      GenerateKeyFromInt(i * max_counter, FLAGS_num, &key);
-      s = db->Put(write_options_, key, gen.Generate(value_size_));
-      if (!s.ok()) {
-        fprintf(stderr, "Operation failed: %s\n", s.ToString().c_str());
-        exit(1);
-      }
-    }
-
-    db->GetSnapshot();
-
-    std::default_random_engine generator;
-    std::normal_distribution<double> distribution(FLAGS_numdistinct / 2.0,
-                                                  FLAGS_stddev);
-    Duration duration(FLAGS_duration, FLAGS_num);
-    while (!duration.Done(1)) {
-      int64_t rnd_id = static_cast<int64_t>(distribution(generator));
-      int64_t key_id = std::max(std::min(FLAGS_numdistinct - 1, rnd_id),
-                                static_cast<int64_t>(0));
-      GenerateKeyFromInt(key_id * max_counter + counters[key_id], FLAGS_num,
-                         &key);
-      s = FLAGS_use_single_deletes ? db->SingleDelete(write_options_, key)
-                                   : db->Delete(write_options_, key);
-      if (s.ok()) {
-        counters[key_id] = (counters[key_id] + 1) % max_counter;
-        GenerateKeyFromInt(key_id * max_counter + counters[key_id], FLAGS_num,
-                           &key);
-        s = db->Put(write_options_, key, Slice());
-      }
-
-      if (!s.ok()) {
-        fprintf(stderr, "Operation failed: %s\n", s.ToString().c_str());
-        exit(1);
-      }
-
-      thread->stats.FinishedOps(nullptr, db, 1);
-    }
-
-    char msg[200];
-    snprintf(msg, sizeof(msg),
-             "use single deletes: %d, "
-             "standard deviation: %lf\n",
-             FLAGS_use_single_deletes, FLAGS_stddev);
-    thread->stats.AddMessage(msg);
-  }
-
  void Compact(ThreadState* thread) {
    DB* db = SelectDB(thread);
    db->CompactRange(CompactRangeOptions(), nullptr, nullptr);
@@ -3967,7 +3880,6 @@ int main(int argc, char** argv) {
  if (FLAGS_statistics) {
    dbstats = rocksdb::CreateDBStatistics();
  }
-  FLAGS_compaction_pri_e = (rocksdb::CompactionPri)FLAGS_compaction_pri;

  std::vector<std::string> fanout = rocksdb::StringSplit(
      FLAGS_max_bytes_for_level_multiplier_additional, ',');
@@ -66,6 +66,51 @@ class OnFileDeletionListener : public EventListener {
  std::string expected_file_name_;
 };

+class SleepingBackgroundTask {
+ public:
+  SleepingBackgroundTask()
+      : bg_cv_(&mutex_), should_sleep_(true), done_with_sleep_(false) {}
+  void DoSleep() {
+    MutexLock l(&mutex_);
+    while (should_sleep_) {
+      bg_cv_.Wait();
+    }
+    done_with_sleep_ = true;
+    bg_cv_.SignalAll();
+  }
+  void WakeUp() {
+    MutexLock l(&mutex_);
+    should_sleep_ = false;
+    bg_cv_.SignalAll();
+  }
+  void WaitUntilDone() {
+    MutexLock l(&mutex_);
+    while (!done_with_sleep_) {
+      bg_cv_.Wait();
+    }
+  }
+  bool WokenUp() {
+    MutexLock l(&mutex_);
+    return should_sleep_ == false;
+  }
+
+  void Reset() {
+    MutexLock l(&mutex_);
+    should_sleep_ = true;
+    done_with_sleep_ = false;
+  }
+
+  static void DoSleepTask(void* arg) {
+    reinterpret_cast<SleepingBackgroundTask*>(arg)->DoSleep();
+  }
+
+ private:
+  port::Mutex mutex_;
+  port::CondVar bg_cv_;  // Signalled when background work finishes
+  bool should_sleep_;
+  bool done_with_sleep_;
+};
+
 static const int kCDTValueSize = 1000;
 static const int kCDTKeysPerBuffer = 4;
 static const int kCDTNumLevels = 8;
@@ -1448,8 +1493,8 @@ TEST_P(DBCompactionTestWithParam, PartialCompactionFailure) {
  env_->SetBackgroundThreads(1, Env::HIGH);
  env_->SetBackgroundThreads(1, Env::LOW);
  // stop the compaction thread until we simulate the file creation failure.
-  test::SleepingBackgroundTask sleeping_task_low;
-  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
+  SleepingBackgroundTask sleeping_task_low;
+  env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
                 Env::Priority::LOW);

  options.env = env_;
@@ -1541,8 +1586,8 @@ TEST_P(DBCompactionTestWithParam, DeleteMovedFileAfterCompaction) {
    ASSERT_EQ("0,1", FilesPerLevel(0));

    // block compactions
-    test::SleepingBackgroundTask sleeping_task;
-    env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task,
+    SleepingBackgroundTask sleeping_task;
+    env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &sleeping_task,
                   Env::Priority::LOW);

    options.max_bytes_for_level_base = 1024 * 1024;  // 1 MB
@@ -58,7 +58,6 @@
 #include "rocksdb/delete_scheduler.h"
 #include "rocksdb/env.h"
 #include "rocksdb/merge_operator.h"
-#include "rocksdb/sst_file_writer.h"
 #include "rocksdb/statistics.h"
 #include "rocksdb/status.h"
 #include "rocksdb/table.h"
@@ -260,7 +259,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
      wal_manager_(db_options_, env_options_),
 #endif  // ROCKSDB_LITE
      event_logger_(db_options_.info_log.get()),
-      bg_work_paused_(0),
+      bg_work_gate_closed_(false),
      refitting_level_(false),
      opened_successfully_(false) {
  env_->GetAbsolutePath(dbname, &db_absolute_path_);
@@ -282,6 +281,8 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
  DumpDBFileSummary(db_options_, dbname_);
  db_options_.Dump(db_options_.info_log.get());
  DumpSupportInfo(db_options_.info_log.get());
+
+  LogFlush(db_options_.info_log);
 }

 // Will lock the mutex_,  will wait for completion if wait is true
@@ -771,12 +772,6 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
          "[JOB %d] Delete %s type=%d #%" PRIu64 " -- %s\n", state.job_id,
          fname.c_str(), type, number,
          file_deletion_status.ToString().c_str());
-    } else if (env_->FileExists(fname).IsNotFound()) {
-      Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-          "[JOB %d] Tried to delete a non-existing file %s type=%d #%" PRIu64
-          " -- %s\n",
-          state.job_id, fname.c_str(), type, number,
-          file_deletion_status.ToString().c_str());
    } else {
      Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
          "[JOB %d] Failed to delete %s type=%d #%" PRIu64 " -- %s\n",
@@ -805,16 +800,9 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
          full_path_to_delete.c_str());
      Status s = env_->DeleteFile(full_path_to_delete);
      if (!s.ok()) {
-        if (env_->FileExists(full_path_to_delete).IsNotFound()) {
-          Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-              "[JOB %d] Tried to delete non-existing info log file %s FAILED "
-              "-- %s\n",
-              state.job_id, to_delete.c_str(), s.ToString().c_str());
-        } else {
-          Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
-              "[JOB %d] Delete info log file %s FAILED -- %s\n", state.job_id,
-              to_delete.c_str(), s.ToString().c_str());
-        }
+        Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
+            "[JOB %d] Delete info log file %s FAILED -- %s\n", state.job_id,
+            to_delete.c_str(), s.ToString().c_str());
      }
    }
  }
@@ -1548,13 +1536,7 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
  }

  if (options.change_level) {
-    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-        "[RefitLevel] waiting for background threads to stop");
-    s = PauseBackgroundWork();
-    if (s.ok()) {
-      s = ReFitLevel(cfd, final_output_level, options.target_level);
-    }
-    ContinueBackgroundWork();
+    s = ReFitLevel(cfd, final_output_level, options.target_level);
  }
  LogFlush(db_options_.info_log);

@@ -1753,25 +1735,6 @@ Status DBImpl::CompactFilesImpl(
 }
 #endif  // ROCKSDB_LITE

-Status DBImpl::PauseBackgroundWork() {
-  InstrumentedMutexLock guard_lock(&mutex_);
-  bg_work_paused_++;
-  while (bg_compaction_scheduled_ > 0 || bg_flush_scheduled_ > 0) {
-    bg_cv_.Wait();
-  }
-  return Status::OK();
-}
-
-Status DBImpl::ContinueBackgroundWork() {
-  InstrumentedMutexLock guard_lock(&mutex_);
-  assert(bg_work_paused_ > 0);
-  bg_work_paused_--;
-  if (bg_work_paused_ == 0) {
-    MaybeScheduleFlushOrCompaction();
-  }
-  return Status::OK();
-}
-
 void DBImpl::NotifyOnCompactionCompleted(
    ColumnFamilyData* cfd, Compaction *c, const Status &st,
    const CompactionJobStats& compaction_job_stats,
@@ -1882,18 +1845,14 @@ int DBImpl::FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd,
  return minimum_level;
 }

-// REQUIREMENT: block all background work by calling PauseBackgroundWork()
-// before calling this function
 Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
  assert(level < cfd->NumberLevels());
  if (target_level >= cfd->NumberLevels()) {
    return Status::InvalidArgument("Target level exceeds number of levels");
  }

-  std::unique_ptr<SuperVersion> superversion_to_free;
-  std::unique_ptr<SuperVersion> new_superversion(new SuperVersion());
-
-  Status status;
+  SuperVersion* superversion_to_free = nullptr;
+  SuperVersion* new_superversion = new SuperVersion();

  InstrumentedMutexLock guard_lock(&mutex_);

@@ -1901,26 +1860,40 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
  if (refitting_level_) {
    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
        "[ReFitLevel] another thread is refitting");
+    delete new_superversion;
    return Status::NotSupported("another thread is refitting");
  }
  refitting_level_ = true;

-  const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
+  // wait for all background threads to stop
+  bg_work_gate_closed_ = true;
+  while (bg_compaction_scheduled_ > 0 || bg_flush_scheduled_) {
+    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+        "[RefitLevel] waiting for background threads to stop: %d %d",
+        bg_compaction_scheduled_, bg_flush_scheduled_);
+    bg_cv_.Wait();
+  }
+
+  const MutableCFOptions mutable_cf_options =
+    *cfd->GetLatestMutableCFOptions();
  // move to a smaller level
  int to_level = target_level;
  if (target_level < 0) {
    to_level = FindMinimumEmptyLevelFitting(cfd, mutable_cf_options, level);
  }

+  Status status;
  auto* vstorage = cfd->current()->storage_info();
  if (to_level > level) {
    if (level == 0) {
+      delete new_superversion;
      return Status::NotSupported(
          "Cannot change from level 0 to other levels.");
    }
    // Check levels are empty for a trivial move
    for (int l = level + 1; l <= to_level; l++) {
      if (vstorage->NumLevelFiles(l) > 0) {
+        delete new_superversion;
        return Status::NotSupported(
            "Levels between source and target are not empty for a move.");
      }
@@ -1928,8 +1901,8 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
  }
  if (to_level != level) {
    Log(InfoLogLevel::DEBUG_LEVEL, db_options_.info_log,
-        "[%s] Before refitting:\n%s", cfd->GetName().c_str(),
-        cfd->current()->DebugString().data());
+        "[%s] Before refitting:\n%s",
+        cfd->GetName().c_str(), cfd->current()->DebugString().data());

    VersionEdit edit;
    edit.SetColumnFamily(cfd->GetID());
@@ -1941,13 +1914,14 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
                   f->marked_for_compaction);
    }
    Log(InfoLogLevel::DEBUG_LEVEL, db_options_.info_log,
-        "[%s] Apply version edit:\n%s", cfd->GetName().c_str(),
-        edit.DebugString().data());
+        "[%s] Apply version edit:\n%s",
+        cfd->GetName().c_str(), edit.DebugString().data());

    status = versions_->LogAndApply(cfd, mutable_cf_options, &edit, &mutex_,
                                    directories_.GetDbDir());
-    superversion_to_free.reset(InstallSuperVersionAndScheduleWork(
-        cfd, new_superversion.release(), mutable_cf_options));
+    superversion_to_free = InstallSuperVersionAndScheduleWork(
+        cfd, new_superversion, mutable_cf_options);
+    new_superversion = nullptr;

    Log(InfoLogLevel::DEBUG_LEVEL, db_options_.info_log,
        "[%s] LogAndApply: %s\n", cfd->GetName().c_str(),
@@ -1955,13 +1929,16 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {

    if (status.ok()) {
      Log(InfoLogLevel::DEBUG_LEVEL, db_options_.info_log,
-          "[%s] After refitting:\n%s", cfd->GetName().c_str(),
-          cfd->current()->DebugString().data());
+          "[%s] After refitting:\n%s",
+          cfd->GetName().c_str(), cfd->current()->DebugString().data());
    }
  }

  refitting_level_ = false;
+  bg_work_gate_closed_ = false;

+  delete superversion_to_free;
+  delete new_superversion;
  return status;
 }

@@ -2025,7 +2002,6 @@ Status DBImpl::SyncWAL() {
    need_log_dir_sync = !log_dir_synced_;
  }

-  RecordTick(stats_, WAL_FILE_SYNCED);
  Status status;
  for (log::Writer* log : logs_to_sync) {
    status = log->file()->SyncWithoutFlush(db_options_.use_fsync);
@@ -2214,8 +2190,8 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
    // Compaction may introduce data race to DB open
    return;
  }
-  if (bg_work_paused_ > 0) {
-    // we paused the background work
+  if (bg_work_gate_closed_) {
+    // gate closed for background work
    return;
  } else if (shutting_down_.load(std::memory_order_acquire)) {
    // DB is being deleted; no more background compactions
@@ -3083,221 +3059,6 @@ std::vector<Status> DBImpl::MultiGet(
  return stat_list;
 }

-#ifndef ROCKSDB_LITE
-Status DBImpl::AddFile(ColumnFamilyHandle* column_family,
-                       const std::string& file_path, bool move_file) {
-  Status status;
-  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
-  ColumnFamilyData* cfd = cfh->cfd();
-
-  ExternalSstFileInfo file_info;
-  file_info.file_path = file_path;
-  status = env_->GetFileSize(file_path, &file_info.file_size);
-  if (!status.ok()) {
-    return status;
-  }
-
-  // Access the file using TableReader to extract
-  // version, number of entries, smallest user key, largest user key
-  std::unique_ptr<RandomAccessFile> sst_file;
-  status = env_->NewRandomAccessFile(file_path, &sst_file, env_options_);
-  if (!status.ok()) {
-    return status;
-  }
-  std::unique_ptr<RandomAccessFileReader> sst_file_reader;
-  sst_file_reader.reset(new RandomAccessFileReader(std::move(sst_file)));
-
-  std::unique_ptr<TableReader> table_reader;
-  status = cfd->ioptions()->table_factory->NewTableReader(
-      TableReaderOptions(*cfd->ioptions(), env_options_,
-                         cfd->internal_comparator()),
-      std::move(sst_file_reader), file_info.file_size, &table_reader);
-  if (!status.ok()) {
-    return status;
-  }
-
-  // Get the external sst file version from table properties
-  const UserCollectedProperties& user_collected_properties =
-      table_reader->GetTableProperties()->user_collected_properties;
-  UserCollectedProperties::const_iterator external_sst_file_version_iter =
-      user_collected_properties.find(ExternalSstFilePropertyNames::kVersion);
-  if (external_sst_file_version_iter == user_collected_properties.end()) {
-    return Status::InvalidArgument("Generated table version not found");
-  }
-
-  file_info.version =
-      DecodeFixed32(external_sst_file_version_iter->second.c_str());
-  if (file_info.version == 1) {
-    // version 1 imply that all sequence numbers in table equal 0
-    file_info.sequence_number = 0;
-  } else {
-    return Status::InvalidArgument("Generated table version is not supported");
-  }
-
-  // Get number of entries in table
-  file_info.num_entries = table_reader->GetTableProperties()->num_entries;
-
-  ParsedInternalKey key;
-  std::unique_ptr<Iterator> iter(table_reader->NewIterator(ReadOptions()));
-
-  // Get first (smallest) key from file
-  iter->SeekToFirst();
-  if (!ParseInternalKey(iter->key(), &key)) {
-    return Status::Corruption("Generated table have corrupted keys");
-  }
-  if (key.sequence != 0) {
-    return Status::Corruption("Generated table have non zero sequence number");
-  }
-  file_info.smallest_key = key.user_key.ToString();
-
-  // Get last (largest) key from file
-  iter->SeekToLast();
-  if (!ParseInternalKey(iter->key(), &key)) {
-    return Status::Corruption("Generated table have corrupted keys");
-  }
-  if (key.sequence != 0) {
-    return Status::Corruption("Generated table have non zero sequence number");
-  }
-  file_info.largest_key = key.user_key.ToString();
-
-  return AddFile(column_family, &file_info, move_file);
-}
-
-Status DBImpl::AddFile(ColumnFamilyHandle* column_family,
-                       const ExternalSstFileInfo* file_info, bool move_file) {
-  Status status;
-  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
-  ColumnFamilyData* cfd = cfh->cfd();
-
-  if (cfd->NumberLevels() <= 1) {
-    return Status::NotSupported(
-        "AddFile requires a database with at least 2 levels");
-  }
-  if (file_info->version != 1) {
-    return Status::InvalidArgument("Generated table version is not supported");
-  }
-  // version 1 imply that file have only Put Operations with Sequence Number = 0
-
-  FileMetaData meta;
-  meta.smallest =
-      InternalKey(file_info->smallest_key, file_info->sequence_number,
-                  ValueType::kTypeValue);
-  meta.largest = InternalKey(file_info->largest_key, file_info->sequence_number,
-                             ValueType::kTypeValue);
-  if (!meta.smallest.Valid() || !meta.largest.Valid()) {
-    return Status::Corruption("Generated table have corrupted keys");
-  }
-  meta.smallest_seqno = file_info->sequence_number;
-  meta.largest_seqno = file_info->sequence_number;
-  if (meta.smallest_seqno != 0 || meta.largest_seqno != 0) {
-    return Status::InvalidArgument(
-        "Non zero sequence numbers are not supported");
-  }
-  // Generate a location for the new table
-  meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, file_info->file_size);
-  std::string db_fname = TableFileName(
-      db_options_.db_paths, meta.fd.GetNumber(), meta.fd.GetPathId());
-
-  if (move_file) {
-    status = env_->LinkFile(file_info->file_path, db_fname);
-    if (status.IsNotSupported()) {
-      // Original file is on a different FS, use copy instead of hard linking
-      status = CopyFile(env_, file_info->file_path, db_fname, 0);
-    }
-  } else {
-    status = CopyFile(env_, file_info->file_path, db_fname, 0);
-  }
-  if (!status.ok()) {
-    return status;
-  }
-
-  {
-    InstrumentedMutexLock l(&mutex_);
-    const MutableCFOptions mutable_cf_options =
-        *cfd->GetLatestMutableCFOptions();
-
-    WriteThread::Writer w;
-    write_thread_.EnterUnbatched(&w, &mutex_);
-
-    // Make sure memtables are empty
-    if (!cfd->mem()->IsEmpty() || cfd->imm()->NumNotFlushed() > 0) {
-      // Cannot add the file since the keys in memtable
-      // will hide the keys in file
-      status = Status::NotSupported("Memtable is not empty");
-    }
-
-    // Make sure last sequence number is 0, if there are existing files then
-    // they should have sequence number = 0
-    if (status.ok() && versions_->LastSequence() > 0) {
-      status = Status::NotSupported("Last Sequence number is not zero");
-    }
-
-    auto* vstorage = cfd->current()->storage_info();
-    if (status.ok()) {
-      // Make sure that the key range in the file we will add does not overlap
-      // with previously added files
-      Slice smallest_user_key = meta.smallest.user_key();
-      Slice largest_user_key = meta.largest.user_key();
-      for (int level = 0; level < vstorage->num_non_empty_levels(); level++) {
-        if (vstorage->OverlapInLevel(level, &smallest_user_key,
-                                     &largest_user_key)) {
-          status = Status::NotSupported("Cannot add overlapping files");
-          break;
-        }
-      }
-    }
-
-    if (status.ok()) {
-      // We add the file to the last level
-      int target_level = cfd->NumberLevels() - 1;
-      if (cfd->ioptions()->level_compaction_dynamic_level_bytes == false) {
-        // If we are using dynamic level compaction we add the file to
-        // last level with files
-        target_level = vstorage->num_non_empty_levels() - 1;
-        if (target_level <= 0) {
-          target_level = 1;
-        }
-      }
-      VersionEdit edit;
-      edit.SetColumnFamily(cfd->GetID());
-      edit.AddFile(target_level, meta.fd.GetNumber(), meta.fd.GetPathId(),
-                   meta.fd.GetFileSize(), meta.smallest, meta.largest,
-                   meta.smallest_seqno, meta.largest_seqno,
-                   meta.marked_for_compaction);
-
-      status = versions_->LogAndApply(cfd, mutable_cf_options, &edit, &mutex_,
-                                      directories_.GetDbDir());
-    }
-    write_thread_.ExitUnbatched(&w);
-
-    if (status.ok()) {
-      delete InstallSuperVersionAndScheduleWork(cfd, nullptr,
-                                                mutable_cf_options);
-    }
-  }
-
-  if (!status.ok()) {
-    // We failed to add the file to the database
-    Status s = env_->DeleteFile(db_fname);
-    if (!s.ok()) {
-      Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
-          "AddFile() clean up for file %s failed : %s", db_fname.c_str(),
-          s.ToString().c_str());
-    }
-  } else if (status.ok() && move_file) {
-    // The file was moved and added successfully, remove original file link
-    Status s = env_->DeleteFile(file_info->file_path);
-    if (!s.ok()) {
-      Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
-          "%s was added to DB successfully but failed to remove original file "
-          "link : %s",
-          file_info->file_path.c_str(), s.ToString().c_str());
-    }
-  }
-  return status;
-}
-#endif  // ROCKSDB_LITE
-
 Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options,
                                  const std::string& column_family_name,
                                  ColumnFamilyHandle** handle) {
@@ -3659,12 +3420,6 @@ Status DBImpl::Delete(const WriteOptions& write_options,
  return DB::Delete(write_options, column_family, key);
 }

-Status DBImpl::SingleDelete(const WriteOptions& write_options,
-                            ColumnFamilyHandle* column_family,
-                            const Slice& key) {
-  return DB::SingleDelete(write_options, column_family, key);
-}
-
 Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) {
  return WriteImpl(write_options, my_batch, nullptr);
 }
@@ -3935,10 +3690,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
      default_cf_internal_stats_->AddDBStats(InternalStats::NUMBER_KEYS_WRITTEN,
                                             my_batch_count);
      if (!write_options.disableWAL) {
-        if (write_options.sync) {
-          default_cf_internal_stats_->AddDBStats(InternalStats::WAL_FILE_SYNCED,
-                                                 1);
-        }
+        default_cf_internal_stats_->AddDBStats(
+            InternalStats::WAL_FILE_SYNCED, 1);
        default_cf_internal_stats_->AddDBStats(
            InternalStats::WAL_FILE_BYTES, log_size);
      }
@@ -4481,10 +4234,6 @@ Status DBImpl::CheckConsistency() {

    uint64_t fsize = 0;
    Status s = env_->GetFileSize(file_path, &fsize);
-    if (!s.ok() &&
-        env_->GetFileSize(Rocks2LevelTableFileName(file_path), &fsize).ok()) {
-      s = Status::OK();
-    }
    if (!s.ok()) {
      corruption_messages +=
          "Can't access " + md.name + ": " + s.ToString() + "\n";
@@ -4554,13 +4303,6 @@ Status DB::Delete(const WriteOptions& opt, ColumnFamilyHandle* column_family,
  return Write(opt, &batch);
 }

-Status DB::SingleDelete(const WriteOptions& opt,
-                        ColumnFamilyHandle* column_family, const Slice& key) {
-  WriteBatch batch;
-  batch.SingleDelete(column_family, key);
-  return Write(opt, &batch);
-}
-
 Status DB::Merge(const WriteOptions& opt, ColumnFamilyHandle* column_family,
                 const Slice& key, const Slice& value) {
  WriteBatch batch;
@@ -4752,8 +4494,6 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
  if (s.ok()) {
    Log(InfoLogLevel::INFO_LEVEL, impl->db_options_.info_log, "DB pointer %p",
        impl);
-    LogFlush(impl->db_options_.info_log);
-
    *dbptr = impl;
  } else {
    for (auto* h : *handles) {
@@ -4909,10 +4649,11 @@ void DumpRocksDBBuildVersion(Logger * log) {
 #if !defined(IOS_CROSS_COMPILE)
  // if we compile with Xcode, we don't run build_detect_vesion, so we don't
  // generate util/build_version.cc
-  Header(log, "RocksDB version: %d.%d.%d\n", ROCKSDB_MAJOR, ROCKSDB_MINOR,
-         ROCKSDB_PATCH);
-  Header(log, "Git sha %s", rocksdb_build_git_sha);
-  Header(log, "Compile date %s", rocksdb_build_compile_date);
+  Warn(log,
+      "RocksDB version: %d.%d.%d\n", ROCKSDB_MAJOR, ROCKSDB_MINOR,
+      ROCKSDB_PATCH);
+  Warn(log, "Git sha %s", rocksdb_build_git_sha);
+  Warn(log, "Compile date %s", rocksdb_build_compile_date);
 #endif
 }

@@ -12,6 +12,7 @@
 #include <deque>
 #include <limits>
 #include <list>
+#include <list>
 #include <set>
 #include <string>
 #include <utility>
@@ -39,6 +40,7 @@
 #include "util/autovector.h"
 #include "util/event_logger.h"
 #include "util/hash.h"
+#include "util/hash.h"
 #include "util/instrumented_mutex.h"
 #include "util/scoped_arena_iterator.h"
 #include "util/stop_watch.h"
@@ -51,10 +53,10 @@ class TableCache;
 class Version;
 class VersionEdit;
 class VersionSet;
+class CompactionFilterV2;
 class Arena;
 class WriteCallback;
 struct JobContext;
-struct ExternalSstFileInfo;

 class DBImpl : public DB {
 public:
@@ -74,10 +76,6 @@ class DBImpl : public DB {
  virtual Status Delete(const WriteOptions& options,
                        ColumnFamilyHandle* column_family,
                        const Slice& key) override;
-  using DB::SingleDelete;
-  virtual Status SingleDelete(const WriteOptions& options,
-                              ColumnFamilyHandle* column_family,
-                              const Slice& key) override;
  using DB::Write;
  virtual Status Write(const WriteOptions& options,
                       WriteBatch* updates) override;
@@ -138,9 +136,6 @@ class DBImpl : public DB {
                              const int output_level,
                              const int output_path_id = -1) override;

-  virtual Status PauseBackgroundWork() override;
-  virtual Status ContinueBackgroundWork() override;
-
  using DB::SetOptions;
  Status SetOptions(
      ColumnFamilyHandle* column_family,
@@ -228,13 +223,6 @@ class DBImpl : public DB {
  Status GetLatestSequenceForKeyFromMemtable(SuperVersion* sv, const Slice& key,
                                             SequenceNumber* seq);

-  using DB::AddFile;
-  virtual Status AddFile(ColumnFamilyHandle* column_family,
-                         const ExternalSstFileInfo* file_info,
-                         bool move_file) override;
-  virtual Status AddFile(ColumnFamilyHandle* column_family,
-                         const std::string& file_path, bool move_file) override;
-
 #endif  // ROCKSDB_LITE

  // checks if all live files exist on file system and that their file sizes
@@ -748,8 +736,8 @@ class DBImpl : public DB {
  // Unified interface for logging events
  EventLogger event_logger_;

-  // A value of >0 temporarily disables scheduling of background work
-  int bg_work_paused_;
+  // A value of true temporarily disables scheduling of background work
+  bool bg_work_gate_closed_;

  // Guard against multiple concurrent refitting
  bool refitting_level_;
@@ -53,12 +53,6 @@ class DBImplReadOnly : public DBImpl {
                        const Slice& key) override {
    return Status::NotSupported("Not supported operation in read only mode.");
  }
-  using DBImpl::SingleDelete;
-  virtual Status SingleDelete(const WriteOptions& options,
-                              ColumnFamilyHandle* column_family,
-                              const Slice& key) override {
-    return Status::NotSupported("Not supported operation in read only mode.");
-  }
  virtual Status Write(const WriteOptions& options,
                       WriteBatch* updates) override {
    return Status::NotSupported("Not supported operation in read only mode.");
@@ -100,11 +94,6 @@ class DBImplReadOnly : public DBImpl {
    return Status::NotSupported("Not supported operation in read only mode.");
  }

-  using DBImpl::SyncWAL;
-  virtual Status SyncWAL() override {
-    return Status::NotSupported("Not supported operation in read only mode.");
-  }
-
 private:
  friend class DB;

@@ -189,13 +189,6 @@ void DBIter::Next() {
    return;
  }
  FindNextUserEntry(true /* skipping the current user key */);
-  if (statistics_ != nullptr) {
-    RecordTick(statistics_, NUMBER_DB_NEXT);
-    if (valid_) {
-      RecordTick(statistics_, NUMBER_DB_NEXT_FOUND);
-      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-    }
-  }
 }

 // PRE: saved_key_ has the current user key if skipping
@@ -235,7 +228,6 @@ void DBIter::FindNextUserEntryInternal(bool skipping) {
        } else {
          switch (ikey.type) {
            case kTypeDeletion:
-            case kTypeSingleDeletion:
              // Arrange to skip all upcoming entries for this key since
              // they are hidden by this deletion.
              saved_key_.SetKey(ikey.user_key);
@@ -309,12 +301,16 @@ void DBIter::MergeValuesNewToOld() {
    if (!user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
      // hit the next user key, stop right here
      break;
-    } else if (kTypeDeletion == ikey.type || kTypeSingleDeletion == ikey.type) {
+    }
+
+    if (kTypeDeletion == ikey.type) {
      // hit a delete with the same user key, stop right here
      // iter_ is positioned after delete
      iter_->Next();
      break;
-    } else if (kTypeValue == ikey.type) {
+    }
+
+    if (kTypeValue == ikey.type) {
      // hit a put, merge the put value with operands and store the
      // final result in saved_value_. We are done!
      // ignore corruption if there is any.
@@ -330,13 +326,13 @@ void DBIter::MergeValuesNewToOld() {
      // iter_ is positioned after put
      iter_->Next();
      return;
-    } else if (kTypeMerge == ikey.type) {
+    }
+
+    if (kTypeMerge == ikey.type) {
      // hit a merge, add the value as an operand and run associative merge.
      // when complete, add result to operands and continue.
      const Slice& val = iter_->value();
      operands.push_front(val.ToString());
-    } else {
-      assert(false);
    }
  }

@@ -359,13 +355,6 @@ void DBIter::Prev() {
    ReverseToBackward();
  }
  PrevInternal();
-  if (statistics_ != nullptr) {
-    RecordTick(statistics_, NUMBER_DB_PREV);
-    if (valid_) {
-      RecordTick(statistics_, NUMBER_DB_PREV_FOUND);
-      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-    }
-  }
 }

 void DBIter::ReverseToBackward() {
@@ -430,14 +419,12 @@ void DBIter::PrevInternal() {
 }

 // This function checks, if the entry with biggest sequence_number <= sequence_
-// is non kTypeDeletion or kTypeSingleDeletion. If it's not, we save value in
-// saved_value_
+// is non kTypeDeletion. If it's not, we save value in saved_value_
 bool DBIter::FindValueForCurrentKey() {
  assert(iter_->Valid());
  // Contains operands for merge operator.
  std::deque<std::string> operands;
-  // last entry before merge (could be kTypeDeletion, kTypeSingleDeletion or
-  // kTypeValue)
+  // last entry before merge (could be kTypeDeletion or kTypeValue)
  ValueType last_not_merge_type = kTypeDeletion;
  ValueType last_key_entry_type = kTypeDeletion;

@@ -460,9 +447,8 @@ bool DBIter::FindValueForCurrentKey() {
        last_not_merge_type = kTypeValue;
        break;
      case kTypeDeletion:
-      case kTypeSingleDeletion:
        operands.clear();
-        last_not_merge_type = last_key_entry_type;
+        last_not_merge_type = kTypeDeletion;
        PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
        break;
      case kTypeMerge:
@@ -482,7 +468,6 @@ bool DBIter::FindValueForCurrentKey() {

  switch (last_key_entry_type) {
    case kTypeDeletion:
-    case kTypeSingleDeletion:
      valid_ = false;
      return false;
    case kTypeMerge:
@@ -531,8 +516,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
  ParsedInternalKey ikey;
  FindParseableKey(&ikey, kForward);

-  if (ikey.type == kTypeValue || ikey.type == kTypeDeletion ||
-      ikey.type == kTypeSingleDeletion) {
+  if (ikey.type == kTypeValue || ikey.type == kTypeDeletion) {
    if (ikey.type == kTypeValue) {
      saved_value_ = iter_->value().ToString();
      valid_ = true;
@@ -555,7 +539,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {

  if (!iter_->Valid() ||
      !user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) ||
-      ikey.type == kTypeDeletion || ikey.type == kTypeSingleDeletion) {
+      ikey.type == kTypeDeletion) {
    {
      StopWatchNano timer(env_, statistics_ != nullptr);
      PERF_TIMER_GUARD(merge_operator_time_nanos);
@@ -653,17 +637,10 @@ void DBIter::Seek(const Slice& target) {
    iter_->Seek(saved_key_.GetKey());
  }

-  RecordTick(statistics_, NUMBER_DB_SEEK);
  if (iter_->Valid()) {
    direction_ = kForward;
    ClearSavedValue();
    FindNextUserEntry(false /* not skipping */);
-    if (statistics_ != nullptr) {
-      if (valid_) {
-        RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
-        RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-      }
-    }
  } else {
    valid_ = false;
  }
@@ -683,15 +660,8 @@ void DBIter::SeekToFirst() {
    iter_->SeekToFirst();
  }

-  RecordTick(statistics_, NUMBER_DB_SEEK);
  if (iter_->Valid()) {
    FindNextUserEntry(false /* not skipping */);
-    if (statistics_ != nullptr) {
-      if (valid_) {
-        RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
-        RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-      }
-    }
  } else {
    valid_ = false;
  }
@@ -733,13 +703,6 @@ void DBIter::SeekToLast() {
    }
  }
  PrevInternal();
-  if (statistics_ != nullptr) {
-    RecordTick(statistics_, NUMBER_DB_SEEK);
-    if (valid_) {
-      RecordTick(statistics_, NUMBER_DB_SEEK_FOUND);
-      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-    }
-  }
 }

 Iterator* NewDBIterator(Env* env, const ImmutableCFOptions& ioptions,
@@ -38,20 +38,16 @@ class TestIterator : public Iterator {
        iter_(0),
        cmp(comparator) {}

-  void AddPut(std::string argkey, std::string argvalue) {
-    Add(argkey, kTypeValue, argvalue);
+  void AddMerge(std::string argkey, std::string argvalue) {
+    Add(argkey, kTypeMerge, argvalue);
  }

  void AddDeletion(std::string argkey) {
    Add(argkey, kTypeDeletion, std::string());
  }

-  void AddSingleDeletion(std::string argkey) {
-    Add(argkey, kTypeSingleDeletion, std::string());
-  }
-
-  void AddMerge(std::string argkey, std::string argvalue) {
-    Add(argkey, kTypeMerge, argvalue);
+  void AddPut(std::string argkey, std::string argvalue) {
+    Add(argkey, kTypeValue, argvalue);
  }

  void Add(std::string argkey, ValueType type, std::string argvalue) {
@@ -895,8 +891,6 @@ TEST_F(DBIteratorTest, DBIterator1) {
  db_iter->Next();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "b");
-  db_iter->Next();
-  ASSERT_FALSE(db_iter->Valid());
 }

 TEST_F(DBIteratorTest, DBIterator2) {
@@ -1793,58 +1787,6 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {
  ASSERT_FALSE(db_iter->Valid());
 }

-TEST_F(DBIteratorTest, DBIterator11) {
-  Options options;
-  options.merge_operator = MergeOperators::CreateFromStringId("stringappend");
-
-  TestIterator* internal_iter = new TestIterator(BytewiseComparator());
-  internal_iter->AddPut("a", "0");
-  internal_iter->AddPut("b", "0");
-  internal_iter->AddSingleDeletion("b");
-  internal_iter->AddMerge("a", "1");
-  internal_iter->AddMerge("b", "2");
-  internal_iter->Finish();
-
-  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 1,
-      options.max_sequential_skip_in_iterations));
-  db_iter->SeekToFirst();
-  ASSERT_TRUE(db_iter->Valid());
-  ASSERT_EQ(db_iter->key().ToString(), "a");
-  ASSERT_EQ(db_iter->value().ToString(), "0");
-  db_iter->Next();
-  ASSERT_TRUE(db_iter->Valid());
-  ASSERT_EQ(db_iter->key().ToString(), "b");
-  db_iter->Next();
-  ASSERT_FALSE(db_iter->Valid());
-}
-
-TEST_F(DBIteratorTest, DBIterator12) {
-  Options options;
-  options.merge_operator = nullptr;
-
-  TestIterator* internal_iter = new TestIterator(BytewiseComparator());
-  internal_iter->AddPut("a", "1");
-  internal_iter->AddPut("b", "2");
-  internal_iter->AddPut("c", "3");
-  internal_iter->AddSingleDeletion("b");
-  internal_iter->Finish();
-
-  std::unique_ptr<Iterator> db_iter(
-      NewDBIterator(env_, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0));
-  db_iter->SeekToLast();
-  ASSERT_TRUE(db_iter->Valid());
-  ASSERT_EQ(db_iter->key().ToString(), "c");
-  ASSERT_EQ(db_iter->value().ToString(), "3");
-  db_iter->Prev();
-  ASSERT_TRUE(db_iter->Valid());
-  ASSERT_EQ(db_iter->key().ToString(), "a");
-  ASSERT_EQ(db_iter->value().ToString(), "1");
-  db_iter->Prev();
-  ASSERT_FALSE(db_iter->Valid());
-}
-
 class DBIterWithMergeIterTest : public testing::Test {
 public:
  DBIterWithMergeIterTest()
@@ -1942,8 +1884,6 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIterator2) {
  ASSERT_EQ(db_iter_->value().ToString(), "4");
 }

-#if !(defined NDEBUG) || !defined(OS_WIN)
-
 TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace1) {
  // Test Prev() when one child iterator is at its end but more rows
  // are added.
@@ -2294,7 +2234,6 @@ TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace8) {

  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
-#endif // #if !(defined NDEBUG) || !defined(OS_WIN)
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -26,6 +26,7 @@
 #include "db/filename.h"
 #include "db/dbformat.h"
 #include "db/db_impl.h"
+#include "db/filename.h"
 #include "db/job_context.h"
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
@@ -43,7 +44,6 @@
 #include "rocksdb/slice.h"
 #include "rocksdb/slice_transform.h"
 #include "rocksdb/snapshot.h"
-#include "rocksdb/sst_file_writer.h"
 #include "rocksdb/table.h"
 #include "rocksdb/table_properties.h"
 #include "rocksdb/thread_status.h"
@@ -131,46 +131,6 @@ class DBTestWithParam : public DBTest,
  uint32_t max_subcompactions_;
 };

-class BloomStatsTestWithParam
-    : public DBTest,
-      public testing::WithParamInterface<std::tuple<bool, bool>> {
- public:
-  BloomStatsTestWithParam() {
-    use_block_table_ = std::get<0>(GetParam());
-    use_block_based_builder_ = std::get<1>(GetParam());
-
-    options_.create_if_missing = true;
-    options_.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(4));
-    options_.memtable_prefix_bloom_bits = 8 * 1024;
-    if (use_block_table_) {
-      BlockBasedTableOptions table_options;
-      table_options.hash_index_allow_collision = false;
-      table_options.filter_policy.reset(
-          NewBloomFilterPolicy(10, use_block_based_builder_));
-      options_.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    } else {
-      PlainTableOptions table_options;
-      options_.table_factory.reset(NewPlainTableFactory(table_options));
-    }
-
-    perf_context.Reset();
-    DestroyAndReopen(options_);
-  }
-
-  ~BloomStatsTestWithParam() {
-    perf_context.Reset();
-    Destroy(options_);
-  }
-
-  // Required if inheriting from testing::WithParamInterface<>
-  static void SetUpTestCase() {}
-  static void TearDownTestCase() {}
-
-  bool use_block_table_;
-  bool use_block_based_builder_;
-  Options options_;
-};
-
 TEST_F(DBTest, Empty) {
  do {
    Options options;
@@ -283,7 +243,6 @@ TEST_F(DBTest, ReadOnlyDB) {
  ASSERT_OK(ReadOnlyReopen(options));
  ASSERT_EQ("v3", Get("foo"));
  ASSERT_EQ("v2", Get("bar"));
-  ASSERT_TRUE(db_->SyncWAL().IsNotSupported());
 }

 TEST_F(DBTest, CompactedDB) {
@@ -895,107 +854,6 @@ TEST_F(DBTest, PutDeleteGet) {
  } while (ChangeOptions());
 }

-TEST_F(DBTest, PutSingleDeleteGet) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "foo", "v1"));
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_OK(Put(1, "foo2", "v2"));
-    ASSERT_EQ("v2", Get(1, "foo2"));
-    ASSERT_OK(SingleDelete(1, "foo"));
-    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
-    // Skip HashCuckooRep as it does not support single delete. FIFO and
-    // universal compaction do not apply to the test case. Skip MergePut
-    // because single delete does not get removed when it encounters a merge.
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
-                         kSkipUniversalCompaction | kSkipMergePut));
-}
-
-TEST_F(DBTest, SingleDeleteFlush) {
-  // Test to check whether flushing preserves a single delete hidden
-  // behind a put.
-  do {
-    Random rnd(301);
-
-    Options options = CurrentOptions();
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    // Put values on second level (so that they will not be in the same
-    // compaction as the other operations.
-    Put(1, "foo", "first");
-    Put(1, "bar", "one");
-    ASSERT_OK(Flush(1));
-    MoveFilesToLevel(2, 1);
-
-    // (Single) delete hidden by a put
-    SingleDelete(1, "foo");
-    Put(1, "foo", "second");
-    Delete(1, "bar");
-    Put(1, "bar", "two");
-    ASSERT_OK(Flush(1));
-
-    SingleDelete(1, "foo");
-    Delete(1, "bar");
-    ASSERT_OK(Flush(1));
-
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-
-    ASSERT_EQ("NOT_FOUND", Get(1, "bar"));
-    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
-    // Skip HashCuckooRep as it does not support single delete. FIFO and
-    // universal compaction do not apply to the test case. Skip MergePut
-    // because merges cannot be combined with single deletions.
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
-                         kSkipUniversalCompaction | kSkipMergePut));
-}
-
-TEST_F(DBTest, SingleDeletePutFlush) {
-  // Single deletes that encounter the matching put in a flush should get
-  // removed.
-  do {
-    Random rnd(301);
-
-    Options options = CurrentOptions();
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    Put(1, "foo", Slice());
-    Put(1, "a", Slice());
-    SingleDelete(1, "a");
-    ASSERT_OK(Flush(1));
-
-    ASSERT_EQ("[ ]", AllEntriesFor("a", 1));
-    // Skip HashCuckooRep as it does not support single delete. FIFO and
-    // universal compaction do not apply to the test case. Skip MergePut
-    // because merges cannot be combined with single deletions.
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
-                         kSkipUniversalCompaction | kSkipMergePut));
-}
-
-TEST_F(DBTest, EmptyFlush) {
-  // It is possible to produce empty flushes when using single deletes. Tests
-  // whether empty flushes cause issues.
-  do {
-    Random rnd(301);
-
-    Options options = CurrentOptions();
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    Put(1, "a", Slice());
-    SingleDelete(1, "a");
-    ASSERT_OK(Flush(1));
-
-    ASSERT_EQ("[ ]", AllEntriesFor("a", 1));
-    // Skip HashCuckooRep as it does not support single delete. FIFO and
-    // universal compaction do not apply to the test case. Skip MergePut
-    // because merges cannot be combined with single deletions.
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
-                         kSkipUniversalCompaction | kSkipMergePut));
-}
-
 TEST_F(DBTest, GetFromImmutableLayer) {
  do {
    Options options;
@@ -3690,55 +3548,6 @@ TEST_F(DBTest, CompactBetweenSnapshots) {
  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction));
 }

-TEST_F(DBTest, UnremovableSingleDelete) {
-  // If we compact:
-  //
-  // Put(A, v1) Snapshot SingleDelete(A) Put(A, v2)
-  //
-  // We do not want to end up with:
-  //
-  // Put(A, v1) Snapshot Put(A, v2)
-  //
-  // Because a subsequent SingleDelete(A) would delete the Put(A, v2)
-  // but not Put(A, v1), so Get(A) would return v1.
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    Options options = CurrentOptions(options_override);
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    Put(1, "foo", "first");
-    const Snapshot* snapshot = db_->GetSnapshot();
-    SingleDelete(1, "foo");
-    Put(1, "foo", "second");
-    ASSERT_OK(Flush(1));
-
-    ASSERT_EQ("first", Get(1, "foo", snapshot));
-    ASSERT_EQ("second", Get(1, "foo"));
-
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ("[ second, SDEL, first ]", AllEntriesFor("foo", 1));
-
-    SingleDelete(1, "foo");
-
-    ASSERT_EQ("first", Get(1, "foo", snapshot));
-    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
-
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-
-    ASSERT_EQ("first", Get(1, "foo", snapshot));
-    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
-    db_->ReleaseSnapshot(snapshot);
-    // Skip HashCuckooRep as it does not support single delete.  FIFO and
-    // universal compaction do not apply to the test case.  Skip MergePut
-    // because single delete does not get removed when it encounters a merge.
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
-                         kSkipUniversalCompaction | kSkipMergePut));
-}
-
 TEST_F(DBTest, DeletionMarkers1) {
  Options options = CurrentOptions();
  options.max_background_flushes = 0;
@@ -5552,20 +5361,6 @@ class ModelDB: public DB {
    batch.Put(cf, k, v);
    return Write(o, &batch);
  }
-  using DB::Delete;
-  virtual Status Delete(const WriteOptions& o, ColumnFamilyHandle* cf,
-                        const Slice& key) override {
-    WriteBatch batch;
-    batch.Delete(cf, key);
-    return Write(o, &batch);
-  }
-  using DB::SingleDelete;
-  virtual Status SingleDelete(const WriteOptions& o, ColumnFamilyHandle* cf,
-                              const Slice& key) override {
-    WriteBatch batch;
-    batch.SingleDelete(cf, key);
-    return Write(o, &batch);
-  }
  using DB::Merge;
  virtual Status Merge(const WriteOptions& o, ColumnFamilyHandle* cf,
                       const Slice& k, const Slice& v) override {
@@ -5573,6 +5368,13 @@ class ModelDB: public DB {
    batch.Merge(cf, k, v);
    return Write(o, &batch);
  }
+  using DB::Delete;
+  virtual Status Delete(const WriteOptions& o, ColumnFamilyHandle* cf,
+                        const Slice& key) override {
+    WriteBatch batch;
+    batch.Delete(cf, key);
+    return Write(o, &batch);
+  }
  using DB::Get;
  virtual Status Get(const ReadOptions& options, ColumnFamilyHandle* cf,
                     const Slice& key, std::string* value) override {
@@ -5590,18 +5392,6 @@ class ModelDB: public DB {
    return s;
  }

-  using DB::AddFile;
-  virtual Status AddFile(ColumnFamilyHandle* column_family,
-                         const ExternalSstFileInfo* file_path,
-                         bool move_file) override {
-    return Status::NotSupported("Not implemented.");
-  }
-  virtual Status AddFile(ColumnFamilyHandle* column_family,
-                         const std::string& file_path,
-                         bool move_file) override {
-    return Status::NotSupported("Not implemented.");
-  }
-
  using DB::GetPropertiesOfAllTables;
  virtual Status GetPropertiesOfAllTables(
      ColumnFamilyHandle* column_family,
@@ -5703,14 +5493,6 @@ class ModelDB: public DB {
    return Status::NotSupported("Not supported operation.");
  }

-  Status PauseBackgroundWork() override {
-    return Status::NotSupported("Not supported operation.");
-  }
-
-  Status ContinueBackgroundWork() override {
-    return Status::NotSupported("Not supported operation.");
-  }
-
  using DB::NumberLevels;
  virtual int NumberLevels(ColumnFamilyHandle* column_family) override {
    return 1;
@@ -6110,8 +5892,6 @@ TEST_F(DBTest, PrefixScan) {
    snprintf(buf, sizeof(buf), "03______:");
    prefix = Slice(buf, 8);
    key = Slice(buf, 9);
-    ASSERT_EQ(key.difference_offset(prefix), 8);
-    ASSERT_EQ(prefix.difference_offset(key), 8);
    // db configs
    env_->count_random_reads_ = true;
    Options options = CurrentOptions();
@@ -6337,7 +6117,7 @@ TEST_F(DBTest, TableOptionsSanitizeTest) {
  options.table_factory.reset(new PlainTableFactory());
  options.prefix_extractor.reset(NewNoopTransform());
  Destroy(options);
-  ASSERT_TRUE(!TryReopen(options).IsNotSupported());
+  ASSERT_TRUE(TryReopen(options).IsNotSupported());

  // Test for check of prefix_extractor when hash index is used for
  // block-based table
@@ -8692,53 +8472,6 @@ TEST_F(DBTest, DelayedWriteRate) {
  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }

-TEST_F(DBTest, HardLimit) {
-  Options options;
-  options.env = env_;
-  env_->SetBackgroundThreads(1, Env::LOW);
-  options = CurrentOptions(options);
-  options.max_write_buffer_number = 256;
-  options.write_buffer_size = 110 << 10;  // 110KB
-  options.arena_block_size = 4 * 1024;
-  options.level0_file_num_compaction_trigger = 4;
-  options.level0_slowdown_writes_trigger = 999999;
-  options.level0_stop_writes_trigger = 999999;
-  options.hard_pending_compaction_bytes_limit = 800 << 10;
-  options.max_bytes_for_level_base = 10000000000u;
-  options.max_background_compactions = 1;
-
-  env_->SetBackgroundThreads(1, Env::LOW);
-  test::SleepingBackgroundTask sleeping_task_low;
-  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
-                 Env::Priority::LOW);
-
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  std::atomic<int> callback_count(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack("DBImpl::DelayWrite:Wait",
-                                                 [&](void* arg) {
-                                                   callback_count.fetch_add(1);
-                                                   sleeping_task_low.WakeUp();
-                                                 });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-  int key_idx = 0;
-  for (int num = 0; num < 5; num++) {
-    GenerateNewFile(&rnd, &key_idx, true);
-  }
-
-  ASSERT_EQ(0, callback_count.load());
-
-  for (int num = 0; num < 5; num++) {
-    GenerateNewFile(&rnd, &key_idx, true);
-    dbfull()->TEST_WaitForFlushMemTable();
-  }
-  ASSERT_GE(callback_count.load(), 1);
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-}
-
 TEST_F(DBTest, SoftLimit) {
  Options options;
  options.env = env_;
@@ -9366,502 +9099,9 @@ TEST_F(DBTest, GetTotalSstFilesSizeVersionsFilesShared) {
  ASSERT_EQ(total_sst_files_size, 0);
 }

-TEST_F(DBTest, AddExternalSstFile) {
-  do {
-    std::string sst_files_folder = test::TmpDir(env_) + "/sst_files/";
-    env_->CreateDir(sst_files_folder);
-    Options options = CurrentOptions();
-    options.env = env_;
-    const ImmutableCFOptions ioptions(options);
-
-    SstFileWriter sst_file_writer(EnvOptions(), ioptions, options.comparator);
-
-    // file1.sst (0 => 99)
-    std::string file1 = sst_files_folder + "file1.sst";
-    ASSERT_OK(sst_file_writer.Open(file1));
-    for (int k = 0; k < 100; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
-    }
-    ExternalSstFileInfo file1_info;
-    Status s = sst_file_writer.Finish(&file1_info);
-    ASSERT_TRUE(s.ok()) << s.ToString();
-    ASSERT_EQ(file1_info.file_path, file1);
-    ASSERT_EQ(file1_info.num_entries, 100);
-    ASSERT_EQ(file1_info.smallest_key, Key(0));
-    ASSERT_EQ(file1_info.largest_key, Key(99));
-    // sst_file_writer already finished, cannot add this value
-    s = sst_file_writer.Add(Key(100), "bad_val");
-    ASSERT_FALSE(s.ok()) << s.ToString();
-
-    // file2.sst (100 => 199)
-    std::string file2 = sst_files_folder + "file2.sst";
-    ASSERT_OK(sst_file_writer.Open(file2));
-    for (int k = 100; k < 200; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
-    }
-    // Cannot add this key because it's not after last added key
-    s = sst_file_writer.Add(Key(99), "bad_val");
-    ASSERT_FALSE(s.ok()) << s.ToString();
-    ExternalSstFileInfo file2_info;
-    s = sst_file_writer.Finish(&file2_info);
-    ASSERT_TRUE(s.ok()) << s.ToString();
-    ASSERT_EQ(file2_info.file_path, file2);
-    ASSERT_EQ(file2_info.num_entries, 100);
-    ASSERT_EQ(file2_info.smallest_key, Key(100));
-    ASSERT_EQ(file2_info.largest_key, Key(199));
-
-    // file3.sst (195 => 299)
-    // This file values overlap with file2 values
-    std::string file3 = sst_files_folder + "file3.sst";
-    ASSERT_OK(sst_file_writer.Open(file3));
-    for (int k = 195; k < 300; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
-    }
-    ExternalSstFileInfo file3_info;
-    s = sst_file_writer.Finish(&file3_info);
-    ASSERT_TRUE(s.ok()) << s.ToString();
-    ASSERT_EQ(file3_info.file_path, file3);
-    ASSERT_EQ(file3_info.num_entries, 105);
-    ASSERT_EQ(file3_info.smallest_key, Key(195));
-    ASSERT_EQ(file3_info.largest_key, Key(299));
-
-    // file4.sst (30 => 39)
-    // This file values overlap with file1 values
-    std::string file4 = sst_files_folder + "file4.sst";
-    ASSERT_OK(sst_file_writer.Open(file4));
-    for (int k = 30; k < 40; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
-    }
-    ExternalSstFileInfo file4_info;
-    s = sst_file_writer.Finish(&file4_info);
-    ASSERT_TRUE(s.ok()) << s.ToString();
-    ASSERT_EQ(file4_info.file_path, file4);
-    ASSERT_EQ(file4_info.num_entries, 10);
-    ASSERT_EQ(file4_info.smallest_key, Key(30));
-    ASSERT_EQ(file4_info.largest_key, Key(39));
-
-    // file5.sst (400 => 499)
-    std::string file5 = sst_files_folder + "file5.sst";
-    ASSERT_OK(sst_file_writer.Open(file5));
-    for (int k = 400; k < 500; k++) {
-      ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
-    }
-    ExternalSstFileInfo file5_info;
-    s = sst_file_writer.Finish(&file5_info);
-    ASSERT_TRUE(s.ok()) << s.ToString();
-    ASSERT_EQ(file5_info.file_path, file5);
-    ASSERT_EQ(file5_info.num_entries, 100);
-    ASSERT_EQ(file5_info.smallest_key, Key(400));
-    ASSERT_EQ(file5_info.largest_key, Key(499));
-
-    DestroyAndReopen(options);
-    // Add file using file path
-    s = db_->AddFile(file1);
-    ASSERT_TRUE(s.ok()) << s.ToString();
-    ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
-    for (int k = 0; k < 100; k++) {
-      ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
-    }
-
-    // Add file using file info
-    s = db_->AddFile(&file2_info);
-    ASSERT_TRUE(s.ok()) << s.ToString();
-    ASSERT_EQ(db_->GetLatestSequenceNumber(), 0U);
-    for (int k = 0; k < 200; k++) {
-      ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
-    }
-
-    // This file have overlapping values with the exisitng data
-    s = db_->AddFile(file3);
-    ASSERT_FALSE(s.ok()) << s.ToString();
-
-    // This file have overlapping values with the exisitng data
-    s = db_->AddFile(&file4_info);
-    ASSERT_FALSE(s.ok()) << s.ToString();
-
-    // Overwrite values of keys divisible by 5
-    for (int k = 0; k < 200; k += 5) {
-      ASSERT_OK(Put(Key(k), Key(k) + "_val_new"));
-    }
-    ASSERT_NE(db_->GetLatestSequenceNumber(), 0U);
-
-    // DB have values in memtable now, we cannot add files anymore
-    s = db_->AddFile(file5);
-    ASSERT_FALSE(s.ok()) << s.ToString();
-
-    // Make sure values are correct before and after flush/compaction
-    for (int i = 0; i < 2; i++) {
-      for (int k = 0; k < 200; k++) {
-        std::string value = Key(k) + "_val";
-        if (k % 5 == 0) {
-          value += "_new";
-        }
-        ASSERT_EQ(Get(Key(k)), value);
-      }
-      ASSERT_OK(Flush());
-      ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-    }
-
-    // DB sequence number is not zero, cannot add files anymore
-    s = db_->AddFile(file5);
-    ASSERT_FALSE(s.ok()) << s.ToString();
-  } while (ChangeOptions(kSkipPlainTable | kSkipUniversalCompaction |
-                         kSkipFIFOCompaction));
-}
-
-TEST_F(DBTest, AddExternalSstFileNoCopy) {
-  std::string sst_files_folder = test::TmpDir(env_) + "/sst_files/";
-  env_->CreateDir(sst_files_folder);
-  Options options = CurrentOptions();
-  options.env = env_;
-  const ImmutableCFOptions ioptions(options);
-
-  SstFileWriter sst_file_writer(EnvOptions(), ioptions, options.comparator);
-
-  // file1.sst (0 => 99)
-  std::string file1 = sst_files_folder + "file1.sst";
-  ASSERT_OK(sst_file_writer.Open(file1));
-  for (int k = 0; k < 100; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
-  }
-  ExternalSstFileInfo file1_info;
-  Status s = sst_file_writer.Finish(&file1_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file1_info.file_path, file1);
-  ASSERT_EQ(file1_info.num_entries, 100);
-  ASSERT_EQ(file1_info.smallest_key, Key(0));
-  ASSERT_EQ(file1_info.largest_key, Key(99));
-
-  // file2.sst (100 => 299)
-  std::string file2 = sst_files_folder + "file2.sst";
-  ASSERT_OK(sst_file_writer.Open(file2));
-  for (int k = 100; k < 300; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val"));
-  }
-  ExternalSstFileInfo file2_info;
-  s = sst_file_writer.Finish(&file2_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file2_info.file_path, file2);
-  ASSERT_EQ(file2_info.num_entries, 200);
-  ASSERT_EQ(file2_info.smallest_key, Key(100));
-  ASSERT_EQ(file2_info.largest_key, Key(299));
-
-  // file3.sst (110 => 124) .. overlap with file2.sst
-  std::string file3 = sst_files_folder + "file3.sst";
-  ASSERT_OK(sst_file_writer.Open(file3));
-  for (int k = 110; k < 125; k++) {
-    ASSERT_OK(sst_file_writer.Add(Key(k), Key(k) + "_val_overlap"));
-  }
-  ExternalSstFileInfo file3_info;
-  s = sst_file_writer.Finish(&file3_info);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(file3_info.file_path, file3);
-  ASSERT_EQ(file3_info.num_entries, 15);
-  ASSERT_EQ(file3_info.smallest_key, Key(110));
-  ASSERT_EQ(file3_info.largest_key, Key(124));
-
-  s = db_->AddFile(&file1_info, true /* move file */);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_EQ(Status::NotFound(), env_->FileExists(file1));
-
-  s = db_->AddFile(&file2_info, false /* copy file */);
-  ASSERT_TRUE(s.ok()) << s.ToString();
-  ASSERT_OK(env_->FileExists(file2));
-
-  // This file have overlapping values with the exisitng data
-  s = db_->AddFile(&file3_info, true /* move file */);
-  ASSERT_FALSE(s.ok()) << s.ToString();
-  ASSERT_OK(env_->FileExists(file3));
-
-  for (int k = 0; k < 300; k++) {
-    ASSERT_EQ(Get(Key(k)), Key(k) + "_val");
-  }
-}
-
-TEST_F(DBTest, AddExternalSstFileMultiThreaded) {
-  std::string sst_files_folder = test::TmpDir(env_) + "/sst_files/";
-  // Bulk load 10 files every file contain 1000 keys
-  int num_files = 10;
-  int keys_per_file = 1000;
-
-  // Generate file names
-  std::vector<std::string> file_names;
-  for (int i = 0; i < num_files; i++) {
-    std::string file_name = "file_" + ToString(i) + ".sst";
-    file_names.push_back(sst_files_folder + file_name);
-  }
-
-  do {
-    env_->CreateDir(sst_files_folder);
-    Options options = CurrentOptions();
-    const ImmutableCFOptions ioptions(options);
-
-    std::atomic<int> thread_num(0);
-    std::function<void()> write_file_func = [&]() {
-      int file_idx = thread_num.fetch_add(1);
-      int range_start = file_idx * keys_per_file;
-      int range_end = range_start + keys_per_file;
-
-      SstFileWriter sst_file_writer(EnvOptions(), ioptions, options.comparator);
-
-      ASSERT_OK(sst_file_writer.Open(file_names[file_idx]));
-
-      for (int k = range_start; k < range_end; k++) {
-        ASSERT_OK(sst_file_writer.Add(Key(k), Key(k)));
-      }
-
-      Status s = sst_file_writer.Finish();
-      ASSERT_TRUE(s.ok()) << s.ToString();
-    };
-    // Write num_files files in parallel
-    std::vector<std::thread> sst_writer_threads;
-    for (int i = 0; i < num_files; ++i) {
-      sst_writer_threads.emplace_back(write_file_func);
-    }
-
-    for (auto& t : sst_writer_threads) {
-      t.join();
-    }
-
-    fprintf(stderr, "Wrote %d files (%d keys)\n", num_files,
-            num_files * keys_per_file);
-
-    thread_num.store(0);
-    std::atomic<int> files_added(0);
-    std::function<void()> load_file_func = [&]() {
-      // We intentionally add every file twice, and assert that it was added
-      // only once and the other add failed
-      int thread_id = thread_num.fetch_add(1);
-      int file_idx = thread_id / 2;
-      // sometimes we use copy, sometimes link .. the result should be the same
-      bool move_file = (thread_id % 3 == 0);
-
-      Status s = db_->AddFile(file_names[file_idx], move_file);
-      if (s.ok()) {
-        files_added++;
-      }
-    };
-    // Bulk load num_files files in parallel
-    std::vector<std::thread> add_file_threads;
-    DestroyAndReopen(options);
-    for (int i = 0; i < num_files * 2; ++i) {
-      add_file_threads.emplace_back(load_file_func);
-    }
-
-    for (auto& t : add_file_threads) {
-      t.join();
-    }
-    ASSERT_EQ(files_added.load(), num_files);
-    fprintf(stderr, "Loaded %d files (%d keys)\n", num_files,
-            num_files * keys_per_file);
-
-    // Overwrite values of keys divisible by 100
-    for (int k = 0; k < num_files * keys_per_file; k += 100) {
-      std::string key = Key(k);
-      Status s = Put(key, key + "_new");
-      ASSERT_TRUE(s.ok());
-    }
-
-    for (int i = 0; i < 2; i++) {
-      // Make sure the values are correct before and after flush/compaction
-      for (int k = 0; k < num_files * keys_per_file; ++k) {
-        std::string key = Key(k);
-        std::string value = (k % 100 == 0) ? (key + "_new") : key;
-        ASSERT_EQ(Get(key), value);
-      }
-      ASSERT_OK(Flush());
-      ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-    }
-
-    fprintf(stderr, "Verified %d values\n", num_files * keys_per_file);
-  } while (ChangeOptions(kSkipPlainTable | kSkipUniversalCompaction |
-                         kSkipFIFOCompaction));
-}
-
-// 1 Create some SST files by inserting K-V pairs into DB
-// 2 Close DB and change suffix from ".sst" to ".ldb" for every other SST file
-// 3 Open DB and check if all key can be read
-TEST_F(DBTest, SSTsWithLdbSuffixHandling) {
-  Options options = CurrentOptions();
-  options.write_buffer_size = 110 << 10;  // 110KB
-  options.num_levels = 4;
-  DestroyAndReopen(options);
-
-  Random rnd(301);
-  int key_id = 0;
-  for (int i = 0; i < 10; ++i) {
-    GenerateNewFile(&rnd, &key_id, false);
-  }
-  Flush();
-  Close();
-  int const num_files = GetSstFileCount(dbname_);
-  ASSERT_GT(num_files, 0);
-
-  std::vector<std::string> filenames;
-  GetSstFiles(dbname_, &filenames);
-  int num_ldb_files = 0;
-  for (unsigned int i = 0; i < filenames.size(); ++i) {
-    if (i & 1) {
-      continue;
-    }
-    std::string const rdb_name = dbname_ + "/" + filenames[i];
-    std::string const ldb_name = Rocks2LevelTableFileName(rdb_name);
-    ASSERT_TRUE(env_->RenameFile(rdb_name, ldb_name).ok());
-    ++num_ldb_files;
-  }
-  ASSERT_GT(num_ldb_files, 0);
-  ASSERT_EQ(num_files, GetSstFileCount(dbname_));
-
-  Reopen(options);
-  for (int k = 0; k < key_id; ++k) {
-    ASSERT_NE("NOT_FOUND", Get(Key(k)));
-  }
-  Destroy(options);
-}
-
 INSTANTIATE_TEST_CASE_P(DBTestWithParam, DBTestWithParam,
                        ::testing::Values(1, 4));

-TEST_F(DBTest, PauseBackgroundWorkTest) {
-  Options options;
-  options.write_buffer_size = 100000;  // Small write buffer
-  options = CurrentOptions(options);
-  Reopen(options);
-
-  std::vector<std::thread> threads;
-  std::atomic<bool> done(false);
-  db_->PauseBackgroundWork();
-  threads.emplace_back([&]() {
-    Random rnd(301);
-    for (int i = 0; i < 10000; ++i) {
-      Put(RandomString(&rnd, 10), RandomString(&rnd, 10));
-    }
-    done.store(true);
-  });
-  env_->SleepForMicroseconds(200000);
-  // make sure the thread is not done
-  ASSERT_EQ(false, done.load());
-  db_->ContinueBackgroundWork();
-  for (auto& t : threads) {
-    t.join();
-  }
-  // now it's done
-  ASSERT_EQ(true, done.load());
-}
-
-// 1 Insert 2 K-V pairs into DB
-// 2 Call Get() for both keys - expext memtable bloom hit stat to be 2
-// 3 Call Get() for nonexisting key - expect memtable bloom miss stat to be 1
-// 4 Call Flush() to create SST
-// 5 Call Get() for both keys - expext SST bloom hit stat to be 2
-// 6 Call Get() for nonexisting key - expect SST bloom miss stat to be 1
-// Test both: block and plain SST
-TEST_P(BloomStatsTestWithParam, BloomStatsTest) {
-  std::string key1("AAAA");
-  std::string key2("RXDB");  // not in DB
-  std::string key3("ZBRA");
-  std::string value1("Value1");
-  std::string value3("Value3");
-
-  ASSERT_OK(Put(key1, value1, WriteOptions()));
-  ASSERT_OK(Put(key3, value3, WriteOptions()));
-
-  // check memtable bloom stats
-  ASSERT_EQ(value1, Get(key1));
-  ASSERT_EQ(1, perf_context.bloom_memtable_hit_count);
-  ASSERT_EQ(value3, Get(key3));
-  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
-  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);
-
-  ASSERT_EQ("NOT_FOUND", Get(key2));
-  ASSERT_EQ(1, perf_context.bloom_memtable_miss_count);
-  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
-
-  // sanity checks
-  ASSERT_EQ(0, perf_context.bloom_sst_hit_count);
-  ASSERT_EQ(0, perf_context.bloom_sst_miss_count);
-
-  Flush();
-
-  // sanity checks
-  ASSERT_EQ(0, perf_context.bloom_sst_hit_count);
-  ASSERT_EQ(0, perf_context.bloom_sst_miss_count);
-
-  // check SST bloom stats
-  // NOTE: hits per get differs because of code paths differences
-  // in BlockBasedTable::Get()
-  int hits_per_get = use_block_table_ && !use_block_based_builder_ ? 2 : 1;
-  ASSERT_EQ(value1, Get(key1));
-  ASSERT_EQ(hits_per_get, perf_context.bloom_sst_hit_count);
-  ASSERT_EQ(value3, Get(key3));
-  ASSERT_EQ(2 * hits_per_get, perf_context.bloom_sst_hit_count);
-
-  ASSERT_EQ("NOT_FOUND", Get(key2));
-  ASSERT_EQ(1, perf_context.bloom_sst_miss_count);
-}
-
-// Same scenario as in BloomStatsTest but using an iterator
-TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
-  std::string key1("AAAA");
-  std::string key2("RXDB");  // not in DB
-  std::string key3("ZBRA");
-  std::string value1("Value1");
-  std::string value3("Value3");
-
-  ASSERT_OK(Put(key1, value1, WriteOptions()));
-  ASSERT_OK(Put(key3, value3, WriteOptions()));
-
-  unique_ptr<Iterator> iter(dbfull()->NewIterator(ReadOptions()));
-
-  // check memtable bloom stats
-  iter->Seek(key1);
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(value1, iter->value().ToString());
-  ASSERT_EQ(1, perf_context.bloom_memtable_hit_count);
-  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);
-
-  iter->Seek(key3);
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(value3, iter->value().ToString());
-  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
-  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);
-
-  iter->Seek(key2);
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(!iter->Valid());
-  ASSERT_EQ(1, perf_context.bloom_memtable_miss_count);
-  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
-
-  Flush();
-
-  iter.reset(dbfull()->NewIterator(ReadOptions()));
-
-  // check SST bloom stats
-  iter->Seek(key1);
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(value1, iter->value().ToString());
-  ASSERT_EQ(1, perf_context.bloom_sst_hit_count);
-
-  iter->Seek(key3);
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(value3, iter->value().ToString());
-  ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
-
-  iter->Seek(key2);
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(!iter->Valid());
-  ASSERT_EQ(1, perf_context.bloom_sst_miss_count);
-  ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
-}
-
-INSTANTIATE_TEST_CASE_P(BloomStatsTestWithParam, BloomStatsTestWithParam,
-                        ::testing::Values(std::make_tuple(true, true),
-                                          std::make_tuple(true, false),
-                                          std::make_tuple(false, false)));
 }  // namespace rocksdb

 #endif
@@ -22,7 +22,7 @@ namespace rocksdb {

 uint64_t PackSequenceAndType(uint64_t seq, ValueType t) {
  assert(seq <= kMaxSequenceNumber);
-  assert(IsValueType(t));
+  assert(t <= kValueTypeForSeek);
  return (seq << 8) | t;
 }

@@ -31,7 +31,7 @@ void UnPackSequenceAndType(uint64_t packed, uint64_t* seq, ValueType* t) {
  *t = static_cast<ValueType>(packed & 0xff);

  assert(*seq <= kMaxSequenceNumber);
-  assert(IsValueType(*t));
+  assert(*t <= kValueTypeForSeek);
 }

 void AppendInternalKey(std::string* result, const ParsedInternalKey& key) {
@@ -33,13 +33,13 @@ enum ValueType : unsigned char {
  kTypeDeletion = 0x0,
  kTypeValue = 0x1,
  kTypeMerge = 0x2,
-  kTypeLogData = 0x3,               // WAL only.
-  kTypeColumnFamilyDeletion = 0x4,  // WAL only.
-  kTypeColumnFamilyValue = 0x5,     // WAL only.
-  kTypeColumnFamilyMerge = 0x6,     // WAL only.
-  kTypeSingleDeletion = 0x7,
-  kTypeColumnFamilySingleDeletion = 0x8,  // WAL only.
-  kMaxValue = 0x7F                        // Not used for storing records.
+  // Following types are used only in write ahead logs. They are not used in
+  // memtables or sst files:
+  kTypeLogData = 0x3,
+  kTypeColumnFamilyDeletion = 0x4,
+  kTypeColumnFamilyValue = 0x5,
+  kTypeColumnFamilyMerge = 0x6,
+  kMaxValue = 0x7F
 };

 // kValueTypeForSeek defines the ValueType that should be passed when
@@ -48,13 +48,7 @@ enum ValueType : unsigned char {
 // and the value type is embedded as the low 8 bits in the sequence
 // number in internal keys, we need to use the highest-numbered
 // ValueType, not the lowest).
-static const ValueType kValueTypeForSeek = kTypeSingleDeletion;
-
-// Checks whether a type is a value type (i.e. a type used in memtables and sst
-// files).
-inline bool IsValueType(ValueType t) {
-  return t <= kTypeMerge || t == kTypeSingleDeletion;
-}
+static const ValueType kValueTypeForSeek = kTypeMerge;

 // We leave eight bits empty at the bottom so a type and sequence#
 // can be packed together into 64-bits.
@@ -199,12 +193,13 @@ inline bool ParseInternalKey(const Slice& internal_key,
  result->type = static_cast<ValueType>(c);
  assert(result->type <= ValueType::kMaxValue);
  result->user_key = Slice(internal_key.data(), n - 8);
-  return IsValueType(result->type);
+  return (c <= static_cast<unsigned char>(kValueTypeForSeek));
 }

 // Update the sequence number in the internal key.
 // Guarantees not to invalidate ikey.data().
-inline void UpdateInternalKey(std::string* ikey, uint64_t seq, ValueType t) {
+inline void UpdateInternalKey(std::string* ikey,
+                              uint64_t seq, ValueType t) {
  size_t ikey_sz = ikey->size();
  assert(ikey_sz >= 8);
  uint64_t newval = (seq << 8) | t;
@@ -277,12 +272,7 @@ class IterKey {

  Slice GetKey() const { return Slice(key_, key_size_); }

-  Slice GetUserKey() const {
-    assert(key_size_ >= 8);
-    return Slice(key_, key_size_ - 8);
-  }
-
-  size_t Size() const { return key_size_; }
+  size_t Size() { return key_size_; }

  void Clear() { key_size_ = 0; }

@@ -302,7 +292,7 @@ class IterKey {
      char* p = new char[total_size];
      memcpy(p, key_, shared_len);

-      if (key_ != space_) {
+      if (key_ != nullptr && key_ != space_) {
        delete[] key_;
      }

@@ -314,30 +304,11 @@ class IterKey {
    memcpy(key_ + shared_len, non_shared_data, non_shared_len);
  }

-  Slice SetKey(const Slice& key) {
+  void SetKey(const Slice& key) {
    size_t size = key.size();
    EnlargeBufferIfNeeded(size);
    memcpy(key_, key.data(), size);
    key_size_ = size;
-    return Slice(key_, key_size_);
-  }
-
-  // Copies the content of key, updates the reference to the user key in ikey
-  // and returns a Slice referencing the new copy.
-  Slice SetKey(const Slice& key, ParsedInternalKey* ikey) {
-    size_t key_n = key.size();
-    assert(key_n >= 8);
-    SetKey(key);
-    ikey->user_key = Slice(key_, key_n - 8);
-    return Slice(key_, key_n);
-  }
-
-  // Update the sequence number in the internal key.  Guarantees not to
-  // invalidate slices to the key (and the user key).
-  void UpdateInternalKey(uint64_t seq, ValueType t) {
-    assert(key_size_ >= 8);
-    uint64_t newval = (seq << 8) | t;
-    EncodeFixed64(&key_[key_size_ - 8], newval);
  }

  void SetInternalKey(const Slice& key_prefix, const Slice& user_key,
@@ -388,10 +359,10 @@ class IterKey {
  char space_[32];  // Avoid allocation for short keys

  void ResetBuffer() {
-    if (key_ != space_) {
+    if (key_ != nullptr && key_ != space_) {
      delete[] key_;
-      key_ = space_;
    }
+    key_ = space_;
    buf_size_ = sizeof(space_);
    key_size_ = 0;
  }
@@ -141,7 +141,6 @@ class TestWritableFile : public WritableFile {
                            FaultInjectionTestEnv* env);
  virtual ~TestWritableFile();
  virtual Status Append(const Slice& data) override;
-  virtual Status Truncate(uint64_t size) override { return target_->Truncate(size); }
  virtual Status Close() override;
  virtual Status Flush() override;
  virtual Status Sync() override;
@@ -779,12 +778,44 @@ TEST_P(FaultInjectionTest, FaultTest) {
  } while (ChangeOptions());
 }

+class SleepingBackgroundTask {
+ public:
+  SleepingBackgroundTask()
+      : bg_cv_(&mutex_), should_sleep_(true), done_with_sleep_(false) {}
+  void DoSleep() {
+    MutexLock l(&mutex_);
+    while (should_sleep_) {
+      bg_cv_.Wait();
+    }
+    done_with_sleep_ = true;
+    bg_cv_.SignalAll();
+  }
+  void WakeUp() {
+    MutexLock l(&mutex_);
+    should_sleep_ = false;
+    bg_cv_.SignalAll();
+    while (!done_with_sleep_) {
+      bg_cv_.Wait();
+    }
+  }
+
+  static void DoSleepTask(void* arg) {
+    reinterpret_cast<SleepingBackgroundTask*>(arg)->DoSleep();
+  }
+
+ private:
+  port::Mutex mutex_;
+  port::CondVar bg_cv_;  // Signalled when background work finishes
+  bool should_sleep_;
+  bool done_with_sleep_;
+};
+
 // Previous log file is not fsynced if sync is forced after log rolling.
 TEST_P(FaultInjectionTest, WriteOptionSyncTest) {
-  test::SleepingBackgroundTask sleeping_task_low;
+  SleepingBackgroundTask sleeping_task_low;
  env_->SetBackgroundThreads(1, Env::HIGH);
  // Block the job queue to prevent flush job from running.
-  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
+  env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
                 Env::Priority::HIGH);

  WriteOptions write_options;
@@ -864,10 +895,10 @@ TEST_P(FaultInjectionTest, UninstalledCompaction) {
 }

 TEST_P(FaultInjectionTest, ManualLogSyncTest) {
-  test::SleepingBackgroundTask sleeping_task_low;
+  SleepingBackgroundTask sleeping_task_low;
  env_->SetBackgroundThreads(1, Env::HIGH);
  // Block the job queue to prevent flush job from running.
-  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
+  env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
                 Env::Priority::HIGH);

  WriteOptions write_options;
@@ -24,9 +24,6 @@

 namespace rocksdb {

-static const std::string kRocksDbTFileExt = "sst";
-static const std::string kLevelDbTFileExt = "ldb";
-
 // Given a path, flatten the path name by replacing all chars not in
 // {[0-9,a-z,A-Z,-,_,.]} with _. And append '_LOG\0' at the end.
 // Return the number of chars stored in dest not including the trailing '\0'.
@@ -81,16 +78,7 @@ std::string ArchivedLogFileName(const std::string& name, uint64_t number) {
 }

 std::string MakeTableFileName(const std::string& path, uint64_t number) {
-  return MakeFileName(path, number, kRocksDbTFileExt.c_str());
-}
-
-std::string Rocks2LevelTableFileName(const std::string& fullname) {
-  assert(fullname.size() > kRocksDbTFileExt.size() + 1);
-  if (fullname.size() <= kRocksDbTFileExt.size() + 1) {
-    return "";
-  }
-  return fullname.substr(0, fullname.size() - kRocksDbTFileExt.size()) +
-         kLevelDbTFileExt;
+  return MakeFileName(path, number, "sst");
 }

 uint64_t TableFileNameToNumber(const std::string& name) {
@@ -163,9 +151,8 @@ InfoLogPrefix::InfoLogPrefix(bool has_log_dir,

 std::string InfoLogFileName(const std::string& dbname,
    const std::string& db_path, const std::string& log_dir) {
-  if (log_dir.empty()) {
+  if (log_dir.empty())
    return dbname + "/LOG";
-  }

  InfoLogPrefix info_log_prefix(true, db_path);
  return log_dir + "/" + info_log_prefix.buf;
@@ -177,9 +164,8 @@ std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
  char buf[50];
  snprintf(buf, sizeof(buf), "%llu", static_cast<unsigned long long>(ts));

-  if (log_dir.empty()) {
+  if (log_dir.empty())
    return dbname + "/LOG.old." + buf;
-  }

  InfoLogPrefix info_log_prefix(true, db_path);
  return log_dir + "/" + info_log_prefix.buf + ".old." + buf;
@@ -285,23 +271,17 @@ bool ParseFileName(const std::string& fname, uint64_t* number,
    if (!ConsumeDecimalNumber(&rest, &num)) {
      return false;
    }
-    if (rest.size() <= 1 || rest[0] != '.') {
-      return false;
-    }
-    rest.remove_prefix(1);
-
    Slice suffix = rest;
-    if (suffix == Slice("log")) {
+    if (suffix == Slice(".log")) {
      *type = kLogFile;
      if (log_type && !archive_dir_found) {
        *log_type = kAliveLogFile;
      }
    } else if (archive_dir_found) {
      return false; // Archive dir can contain only log files
-    } else if (suffix == Slice(kRocksDbTFileExt) ||
-               suffix == Slice(kLevelDbTFileExt)) {
+    } else if (suffix == Slice(".sst")) {
      *type = kTableFile;
-    } else if (suffix == Slice("dbtmp")) {
+    } else if (suffix == Slice(".dbtmp")) {
      *type = kTempFile;
    } else {
      return false;
@@ -55,10 +55,6 @@ extern std::string ArchivedLogFileName(const std::string& dbname,

 extern std::string MakeTableFileName(const std::string& name, uint64_t number);

-// Return the name of sstable with LevelDB suffix
-// created from RocksDB sstable suffixed name
-extern std::string Rocks2LevelTableFileName(const std::string& fullname);
-
 // the reverse function of MakeTableFileName
 // TODO(yhchiang): could merge this function with ParseFileName()
 extern uint64_t TableFileNameToNumber(const std::string& name);
@@ -106,13 +102,11 @@ struct InfoLogPrefix {

 // Return the name of the info log file for "dbname".
 extern std::string InfoLogFileName(const std::string& dbname,
-                                   const std::string& db_path = "",
-                                   const std::string& log_dir = "");
+    const std::string& db_path="", const std::string& log_dir="");

 // Return the name of the old info log file for "dbname".
 extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
-                                      const std::string& db_path = "",
-                                      const std::string& log_dir = "");
+    const std::string& db_path="", const std::string& log_dir="");

 // Return the name to use for a metadatabase. The result will be prefixed with
 // "dbname".
@@ -57,7 +57,7 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
      NumberToHumanString(stats.num_dropped_records);

  snprintf(buf, len,
-           "%4s %6d/%-3d %8.2f %5.1f " /* Level, Files, Size(MB), Score */
+           "%4s %6d/%-3d %8.0f %5.1f " /* Level, Files, Size(MB), Score */
           "%8.1f "                    /* Read(GB) */
           "%7.1f "                    /* Rn(GB) */
           "%8.1f "                    /* Rnp1(GB) */
@@ -669,13 +669,12 @@ void InternalStats::DumpCFStats(std::string* value) {
    total_files += files;
    total_files_being_compacted += files_being_compacted[level];
    if (comp_stats_[level].micros > 0 || files > 0) {
-      uint64_t stalls =
-          level == 0 ? (cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL] +
-                        cf_stats_count_[LEVEL0_NUM_FILES_TOTAL] +
-                        cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT] +
-                        cf_stats_count_[MEMTABLE_COMPACTION])
-                     : (stall_leveln_slowdown_count_soft_[level] +
-                        stall_leveln_slowdown_count_hard_[level]);
+      uint64_t stalls = level == 0 ?
+        (cf_stats_count_[LEVEL0_SLOWDOWN] +
+         cf_stats_count_[LEVEL0_NUM_FILES] +
+         cf_stats_count_[MEMTABLE_COMPACTION])
+        : (stall_leveln_slowdown_count_soft_[level] +
+           stall_leveln_slowdown_count_hard_[level]);

      stats_sum.Add(comp_stats_[level]);
      total_file_size += vstorage->NumLevelBytes(level);
@@ -717,28 +716,15 @@ void InternalStats::DumpCFStats(std::string* value) {
           curr_ingest / kGB, interval_ingest / kGB);
  value->append(buf);

-  snprintf(buf, sizeof(buf), "Stalls(count): %" PRIu64
-                             " level0_slowdown, "
-                             "%" PRIu64
-                             " level0_slowdown_with_compaction, "
-                             "%" PRIu64
-                             " level0_numfiles, "
-                             "%" PRIu64
-                             " level0_numfiles_with_compaction, "
-                             "%" PRIu64
-                             " pending_compaction_bytes, "
-                             "%" PRIu64
-                             " memtable_compaction, "
-                             "%" PRIu64
-                             " leveln_slowdown_soft, "
-                             "%" PRIu64 " leveln_slowdown_hard\n",
-           cf_stats_count_[LEVEL0_SLOWDOWN_TOTAL],
-           cf_stats_count_[LEVEL0_SLOWDOWN_WITH_COMPACTION],
-           cf_stats_count_[LEVEL0_NUM_FILES_TOTAL],
-           cf_stats_count_[LEVEL0_NUM_FILES_WITH_COMPACTION],
-           cf_stats_count_[HARD_PENDING_COMPACTION_BYTES_LIMIT],
-           cf_stats_count_[MEMTABLE_COMPACTION], total_slowdown_count_soft,
-           total_slowdown_count_hard);
+  snprintf(buf, sizeof(buf),
+           "Stalls(count): %" PRIu64 " level0_slowdown, "
+           "%" PRIu64 " level0_numfiles, %" PRIu64 " memtable_compaction, "
+           "%" PRIu64 " leveln_slowdown_soft, "
+           "%" PRIu64 " leveln_slowdown_hard\n",
+           cf_stats_count_[LEVEL0_SLOWDOWN],
+           cf_stats_count_[LEVEL0_NUM_FILES],
+           cf_stats_count_[MEMTABLE_COMPACTION],
+           total_slowdown_count_soft, total_slowdown_count_hard);
  value->append(buf);

  cf_stats_snapshot_.ingest_bytes = curr_ingest;
@@ -80,12 +80,9 @@ extern DBPropertyType GetPropertyType(const Slice& property,
 class InternalStats {
 public:
  enum InternalCFStatsType {
-    LEVEL0_SLOWDOWN_TOTAL,
-    LEVEL0_SLOWDOWN_WITH_COMPACTION,
+    LEVEL0_SLOWDOWN,
    MEMTABLE_COMPACTION,
-    LEVEL0_NUM_FILES_TOTAL,
-    LEVEL0_NUM_FILES_WITH_COMPACTION,
-    HARD_PENDING_COMPACTION_BYTES_LIMIT,
+    LEVEL0_NUM_FILES,
    WRITE_STALLS_ENUM_MAX,
    BYTES_FLUSHED,
    INTERNAL_CF_STATS_ENUM_MAX,
@@ -353,12 +350,9 @@ class InternalStats {
 class InternalStats {
 public:
  enum InternalCFStatsType {
-    LEVEL0_SLOWDOWN_TOTAL,
-    LEVEL0_SLOWDOWN_WITH_COMPACTION,
+    LEVEL0_SLOWDOWN,
    MEMTABLE_COMPACTION,
-    LEVEL0_NUM_FILES_TOTAL,
-    LEVEL0_NUM_FILES_WITH_COMPACTION,
-    HARD_PENDING_COMPACTION_BYTES_LIMIT,
+    LEVEL0_NUM_FILES,
    WRITE_STALLS_ENUM_MAX,
    BYTES_FLUSHED,
    INTERNAL_CF_STATS_ENUM_MAX,
@@ -230,15 +230,10 @@ class MemTableIterator: public Iterator {
  virtual void Seek(const Slice& k) override {
    PERF_TIMER_GUARD(seek_on_memtable_time);
    PERF_COUNTER_ADD(seek_on_memtable_count, 1);
-    if (bloom_ != nullptr) {
-      if (!bloom_->MayContain(
-              prefix_extractor_->Transform(ExtractUserKey(k)))) {
-        PERF_COUNTER_ADD(bloom_memtable_miss_count, 1);
-        valid_ = false;
-        return;
-      } else {
-        PERF_COUNTER_ADD(bloom_memtable_hit_count, 1);
-      }
+    if (bloom_ != nullptr &&
+        !bloom_->MayContain(prefix_extractor_->Transform(ExtractUserKey(k)))) {
+      valid_ = false;
+      return;
    }
    iter_->Seek(k, nullptr);
    valid_ = iter_->Valid();
@@ -449,10 +444,9 @@ static bool SaveValue(void* arg, const char* entry) {
        *(s->found_final_value) = true;
        return false;
      }
-      case kTypeDeletion:
-      case kTypeSingleDeletion: {
+      case kTypeDeletion: {
        if (*(s->merge_in_progress)) {
-          assert(merge_operator != nullptr);
+          assert(merge_operator);
          *(s->status) = Status::OK();
          bool merge_success = false;
          {
@@ -513,18 +507,12 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
  Slice user_key = key.user_key();
  bool found_final_value = false;
  bool merge_in_progress = s->IsMergeInProgress();
-  bool const may_contain =
-      nullptr == prefix_bloom_
-          ? false
-          : prefix_bloom_->MayContain(prefix_extractor_->Transform(user_key));
-  if (prefix_bloom_ && !may_contain) {
+
+  if (prefix_bloom_ &&
+      !prefix_bloom_->MayContain(prefix_extractor_->Transform(user_key))) {
    // iter is null if prefix bloom says the key does not exist
-    PERF_COUNTER_ADD(bloom_memtable_miss_count, 1);
    *seq = kMaxSequenceNumber;
  } else {
-    if (prefix_bloom_) {
-      PERF_COUNTER_ADD(bloom_memtable_hit_count, 1);
-    }
    Saver saver;
    saver.status = s;
    saver.found_final_value = &found_final_value;
@@ -14,6 +14,7 @@
 #include "rocksdb/merge_operator.h"
 #include "util/perf_context_imp.h"
 #include "util/statistics.h"
+#include "util/stop_watch.h"

 namespace rocksdb {

@@ -40,7 +41,8 @@ Status MergeHelper::TimedFullMerge(const Slice& key, const Slice* value,
  bool success =
      merge_operator->FullMerge(key, value, operands, result, logger);

-  RecordTick(statistics, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanosSafe());
+  RecordTick(statistics, MERGE_OPERATION_TOTAL_TIME,
+             env != nullptr ? timer.ElapsedNanos() : 0);

  if (!success) {
    RecordTick(statistics, NUMBER_MERGE_FAILURES);
@@ -57,45 +59,38 @@ Status MergeHelper::TimedFullMerge(const Slice& key, const Slice* value,
 //       operands_ stores the list of merge operands encountered while merging.
 //       keys_[i] corresponds to operands_[i] for each i.
 Status MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
-                               const bool at_bottom) {
+                               const bool at_bottom, Statistics* stats,
+                               Env* env_) {
  // Get a copy of the internal key, before it's invalidated by iter->Next()
  // Also maintain the list of merge operands seen.
  assert(HasOperator());
  keys_.clear();
  operands_.clear();
+  keys_.push_front(iter->key().ToString());
+  operands_.push_front(iter->value().ToString());
  assert(user_merge_operator_);
-  bool first_key = true;

  // We need to parse the internal key again as the parsed key is
  // backed by the internal key!
  // Assume no internal key corruption as it has been successfully parsed
  // by the caller.
-  // original_key_is_iter variable is just caching the information:
-  // original_key_is_iter == (iter->key().ToString() == original_key)
-  bool original_key_is_iter = true;
-  std::string original_key = iter->key().ToString();
-  // Important:
-  // orig_ikey is backed by original_key if keys_.empty()
-  // orig_ikey is backed by keys_.back() if !keys_.empty()
+  // Invariant: keys_.back() will not change. Hence, orig_ikey is always valid.
  ParsedInternalKey orig_ikey;
-  ParseInternalKey(original_key, &orig_ikey);
+  ParseInternalKey(keys_.back(), &orig_ikey);

  Status s;
  bool hit_the_next_user_key = false;
-  for (; iter->Valid(); iter->Next(), original_key_is_iter = false) {
+  for (iter->Next(); iter->Valid(); iter->Next()) {
    ParsedInternalKey ikey;
+    assert(operands_.size() >= 1);        // Should be invariants!
    assert(keys_.size() == operands_.size());

    if (!ParseInternalKey(iter->key(), &ikey)) {
      // stop at corrupted key
      if (assert_valid_internal_key_) {
-        assert(!"Corrupted internal key not expected.");
-        return Status::Corruption("Corrupted internal key not expected.");
+        assert(!"corrupted internal key is not expected");
      }
      break;
-    } else if (first_key) {
-      assert(user_comparator_->Equal(ikey.user_key, orig_ikey.user_key));
-      first_key = false;
    } else if (!user_comparator_->Equal(ikey.user_key, orig_ikey.user_key)) {
      // hit a different user key, stop right here
      hit_the_next_user_key = true;
@@ -107,31 +102,14 @@ Status MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,

    // At this point we are guaranteed that we need to process this key.

-    assert(IsValueType(ikey.type));
+    assert(ikey.type <= kValueTypeForSeek);
    if (ikey.type != kTypeMerge) {
-      if (ikey.type != kTypeValue && ikey.type != kTypeDeletion) {
-        // Merges operands can only be used with puts and deletions, single
-        // deletions are not supported.
-        assert(false);
-        // release build doesn't have asserts, so we return error status
-        return Status::InvalidArgument(
-            " Merges operands can only be used with puts and deletions, single "
-            "deletions are not supported.");
-      }
-
      // hit a put/delete
      //   => merge the put value or a nullptr with operands_
      //   => store result in operands_.back() (and update keys_.back())
      //   => change the entry type to kTypeValue for keys_.back()
      // We are done! Success!
-
-      // If there are no operands, just return the Status::OK(). That will cause
-      // the compaction iterator to write out the key we're currently at, which
-      // is the put/delete we just encountered.
-      if (keys_.empty()) {
-        return Status::OK();
-      }
-
+      //
      // TODO(noetzli) If the merge operator returns false, we are currently
      // (almost) silently dropping the put/delete. That's probably not what we
      // want.
@@ -139,14 +117,14 @@ Status MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
      const Slice* val_ptr = (kTypeValue == ikey.type) ? &val : nullptr;
      std::string merge_result;
      s = TimedFullMerge(ikey.user_key, val_ptr, operands_,
-                         user_merge_operator_, stats_, env_, logger_,
+                         user_merge_operator_, stats, env_, logger_,
                         &merge_result);

      // We store the result in keys_.back() and operands_.back()
      // if nothing went wrong (i.e.: no operand corruption on disk)
      if (s.ok()) {
        // The original key encountered
-        original_key = std::move(keys_.back());
+        std::string original_key = std::move(keys_.back());
        orig_ikey.type = kTypeValue;
        UpdateInternalKey(&original_key, orig_ikey.sequence, orig_ikey.type);
        keys_.clear();
@@ -160,42 +138,18 @@ Status MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
      return s;
    } else {
      // hit a merge
-      //   => if there is a compaction filter, apply it.
      //   => merge the operand into the front of the operands_ list
-      //      if not filtered
+      //   => use the user's associative merge function to determine how.
      //   => then continue because we haven't yet seen a Put/Delete.
-      //
-      // Keep queuing keys and operands until we either meet a put / delete
+      assert(!operands_.empty()); // Should have at least one element in it
+
+      // keep queuing keys and operands until we either meet a put / delete
      // request or later did a partial merge.
-
-      Slice value_slice = iter->value();
-      // add an operand to the list if:
-      // 1) it's included in one of the snapshots. in that case we *must* write
-      // it out, no matter what compaction filter says
-      // 2) it's not filtered by a compaction filter
-      if (ikey.sequence <= latest_snapshot_ ||
-          !FilterMerge(orig_ikey.user_key, value_slice)) {
-        if (original_key_is_iter) {
-          // this is just an optimization that saves us one memcpy
-          keys_.push_front(std::move(original_key));
-        } else {
-          keys_.push_front(iter->key().ToString());
-        }
-        if (keys_.size() == 1) {
-          // we need to re-anchor the orig_ikey because it was anchored by
-          // original_key before
-          ParseInternalKey(keys_.back(), &orig_ikey);
-        }
-        operands_.push_front(value_slice.ToString());
-      }
+      keys_.push_front(iter->key().ToString());
+      operands_.push_front(iter->value().ToString());
    }
  }

-  if (operands_.size() == 0) {
-    // we filtered out all the merge operands
-    return Status::OK();
-  }
-
  // We are sure we have seen this key's entire history if we are at the
  // last level and exhausted all internal keys of this user key.
  // NOTE: !iter->Valid() does not necessarily mean we hit the
@@ -220,13 +174,11 @@ Status MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
    assert(operands_.size() == keys_.size());
    std::string merge_result;
    s = TimedFullMerge(orig_ikey.user_key, nullptr, operands_,
-                       user_merge_operator_, stats_, env_, logger_,
+                       user_merge_operator_, stats, env_, logger_,
                       &merge_result);
    if (s.ok()) {
      // The original key encountered
-      // We are certain that keys_ is not empty here (see assertions couple of
-      // lines before).
-      original_key = std::move(keys_.back());
+      std::string original_key = std::move(keys_.back());
      orig_ikey.type = kTypeValue;
      UpdateInternalKey(&original_key, orig_ikey.sequence, orig_ikey.type);
      keys_.clear();
@@ -248,14 +200,14 @@ Status MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
      bool merge_success = false;
      std::string merge_result;
      {
-        StopWatchNano timer(env_, stats_ != nullptr);
+        StopWatchNano timer(env_, stats != nullptr);
        PERF_TIMER_GUARD(merge_operator_time_nanos);
        merge_success = user_merge_operator_->PartialMergeMulti(
            orig_ikey.user_key,
            std::deque<Slice>(operands_.begin(), operands_.end()),
            &merge_result, logger_);
-        RecordTick(stats_, MERGE_OPERATION_TOTAL_TIME,
-                   timer.ElapsedNanosSafe());
+        RecordTick(stats, MERGE_OPERATION_TOTAL_TIME,
+                   env_ != nullptr ? timer.ElapsedNanos() : 0);
      }
      if (merge_success) {
        // Merging of operands (associative merge) was successful.
@@ -279,6 +231,7 @@ MergeOutputIterator::MergeOutputIterator(const MergeHelper* merge_helper)
 void MergeOutputIterator::SeekToFirst() {
  const auto& keys = merge_helper_->keys();
  const auto& values = merge_helper_->values();
+  assert(keys.size() > 0);
  assert(keys.size() == values.size());
  it_keys_ = keys.rbegin();
  it_values_ = values.rbegin();
@@ -289,17 +242,4 @@ void MergeOutputIterator::Next() {
  ++it_values_;
 }

-bool MergeHelper::FilterMerge(const Slice& user_key, const Slice& value_slice) {
-  if (compaction_filter_ == nullptr) {
-    return false;
-  }
-  if (stats_ != nullptr) {
-    filter_timer_.Start();
-  }
-  bool to_delete =
-      compaction_filter_->FilterMergeOperand(level_, user_key, value_slice);
-  total_filter_time_ += filter_timer_.ElapsedNanosSafe();
-  return to_delete;
-}
-
 } // namespace rocksdb
@@ -10,10 +10,8 @@
 #include <string>

 #include "db/dbformat.h"
-#include "rocksdb/compaction_filter.h"
 #include "rocksdb/env.h"
 #include "rocksdb/slice.h"
-#include "util/stop_watch.h"

 namespace rocksdb {

@@ -25,26 +23,17 @@ class Statistics;

 class MergeHelper {
 public:
-  MergeHelper(Env* env, const Comparator* user_comparator,
-              const MergeOperator* user_merge_operator,
-              const CompactionFilter* compaction_filter, Logger* logger,
+  MergeHelper(const Comparator* user_comparator,
+              const MergeOperator* user_merge_operator, Logger* logger,
              unsigned min_partial_merge_operands,
-              bool assert_valid_internal_key, SequenceNumber latest_snapshot,
-              int level = 0, Statistics* stats = nullptr)
-      : env_(env),
-        user_comparator_(user_comparator),
+              bool assert_valid_internal_key)
+      : user_comparator_(user_comparator),
        user_merge_operator_(user_merge_operator),
-        compaction_filter_(compaction_filter),
        logger_(logger),
        min_partial_merge_operands_(min_partial_merge_operands),
        assert_valid_internal_key_(assert_valid_internal_key),
-        latest_snapshot_(latest_snapshot),
-        level_(level),
        keys_(),
-        operands_(),
-        filter_timer_(env_),
-        total_filter_time_(0U),
-        stats_(stats) {
+        operands_() {
    assert(user_comparator_ != nullptr);
  }

@@ -73,21 +62,15 @@ class MergeHelper {
  //                   0 means no restriction
  // at_bottom:   (IN) true if the iterator covers the bottem level, which means
  //                   we could reach the start of the history of this user key.
-  //
  // Returns one of the following statuses:
  // - OK: Entries were successfully merged.
  // - MergeInProgress: Put/Delete not encountered and unable to merge operands.
-  // - Corruption: Merge operator reported unsuccessful merge or a corrupted
-  //   key has been encountered and not expected (applies only when compiling
-  //   with asserts removed).
+  // - Corruption: Merge operator reported unsuccessful merge.
  //
  // REQUIRED: The first key in the input is not corrupted.
  Status MergeUntil(Iterator* iter, const SequenceNumber stop_before = 0,
-                    const bool at_bottom = false);
-
-  // Filters a merge operand using the compaction filter specified
-  // in the constructor. Returns true if the operand should be filtered out.
-  bool FilterMerge(const Slice& user_key, const Slice& value_slice);
+                    const bool at_bottom = false, Statistics* stats = nullptr,
+                    Env* env_ = nullptr);

  // Query the merge result
  // These are valid until the next MergeUntil call
@@ -116,28 +99,19 @@ class MergeHelper {
  // TODO: Re-style this comment to be like the first one
  const std::deque<std::string>& keys() const { return keys_; }
  const std::deque<std::string>& values() const { return operands_; }
-  uint64_t TotalFilterTime() const { return total_filter_time_; }
  bool HasOperator() const { return user_merge_operator_ != nullptr; }

 private:
-  Env* env_;
  const Comparator* user_comparator_;
  const MergeOperator* user_merge_operator_;
-  const CompactionFilter* compaction_filter_;
  Logger* logger_;
  unsigned min_partial_merge_operands_;
  bool assert_valid_internal_key_; // enforce no internal key corruption?
-  SequenceNumber latest_snapshot_;
-  int level_;

  // the scratch area that holds the result of MergeUntil
  // valid up to the next MergeUntil call
  std::deque<std::string> keys_;    // Keeps track of the sequence of keys seen
  std::deque<std::string> operands_;  // Parallel with keys_; stores the values
-
-  StopWatchNano filter_timer_;
-  uint64_t total_filter_time_;
-  Statistics* stats_;
 };

 // MergeOutputIterator can be used to iterate over the result of a merge.
@@ -18,24 +18,37 @@ namespace rocksdb {

 class MergeHelperTest : public testing::Test {
 public:
-  MergeHelperTest() { env_ = Env::Default(); }
-
+  MergeHelperTest() = default;
  ~MergeHelperTest() = default;

-  Status Run(SequenceNumber stop_before, bool at_bottom,
-             SequenceNumber latest_snapshot = 0) {
-    iter_.reset(new test::VectorIterator(ks_, vs_));
-    iter_->SeekToFirst();
-    merge_helper_.reset(new MergeHelper(env_, BytewiseComparator(),
-                                        merge_op_.get(), filter_.get(), nullptr,
-                                        2U, false, latest_snapshot));
-    return merge_helper_->MergeUntil(iter_.get(), stop_before, at_bottom);
+  Status RunUInt64MergeHelper(SequenceNumber stop_before, bool at_bottom) {
+    InitIterator();
+    merge_op_ = MergeOperators::CreateUInt64AddOperator();
+    merge_helper_.reset(new MergeHelper(BytewiseComparator(), merge_op_.get(),
+                                        nullptr, 2U, false));
+    return merge_helper_->MergeUntil(iter_.get(), stop_before, at_bottom,
+                                     nullptr, Env::Default());
+  }
+
+  Status RunStringAppendMergeHelper(SequenceNumber stop_before,
+                                    bool at_bottom) {
+    InitIterator();
+    merge_op_ = MergeOperators::CreateStringAppendTESTOperator();
+    merge_helper_.reset(new MergeHelper(BytewiseComparator(), merge_op_.get(),
+                                        nullptr, 2U, false));
+    return merge_helper_->MergeUntil(iter_.get(), stop_before, at_bottom,
+                                     nullptr, Env::Default());
+  }
+
+  std::string Key(const std::string& user_key, const SequenceNumber& seq,
+      const ValueType& t) {
+    return InternalKey(user_key, seq, t).Encode().ToString();
  }

  void AddKeyVal(const std::string& user_key, const SequenceNumber& seq,
                 const ValueType& t, const std::string& val,
                 bool corrupt = false) {
-    InternalKey ikey(user_key, seq, t);
+    InternalKey ikey = InternalKey(user_key, seq, t);
    if (corrupt) {
      test::CorruptKeyType(&ikey);
    }
@@ -43,63 +56,66 @@ class MergeHelperTest : public testing::Test {
    vs_.push_back(val);
  }

-  Env* env_;
+  void InitIterator() {
+    iter_.reset(new test::VectorIterator(ks_, vs_));
+    iter_->SeekToFirst();
+  }
+
+  std::string EncodeInt(uint64_t x) {
+    std::string result;
+    PutFixed64(&result, x);
+    return result;
+  }
+
  std::unique_ptr<test::VectorIterator> iter_;
  std::shared_ptr<MergeOperator> merge_op_;
  std::unique_ptr<MergeHelper> merge_helper_;
  std::vector<std::string> ks_;
  std::vector<std::string> vs_;
-  std::unique_ptr<test::FilterNumber> filter_;
 };

 // If MergeHelper encounters a new key on the last level, we know that
 // the key has no more history and it can merge keys.
 TEST_F(MergeHelperTest, MergeAtBottomSuccess) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
+  AddKeyVal("a", 20, kTypeMerge, EncodeInt(1U));
+  AddKeyVal("a", 10, kTypeMerge, EncodeInt(3U));
+  AddKeyVal("b", 10, kTypeMerge, EncodeInt(4U));  // <- Iterator after merge

-  AddKeyVal("a", 20, kTypeMerge, test::EncodeInt(1U));
-  AddKeyVal("a", 10, kTypeMerge, test::EncodeInt(3U));
-  AddKeyVal("b", 10, kTypeMerge, test::EncodeInt(4U));  // <- iter_ after merge
-
-  ASSERT_TRUE(Run(0, true).ok());
+  ASSERT_TRUE(RunUInt64MergeHelper(0, true).ok());
  ASSERT_EQ(ks_[2], iter_->key());
-  ASSERT_EQ(test::KeyStr("a", 20, kTypeValue), merge_helper_->keys()[0]);
-  ASSERT_EQ(test::EncodeInt(4U), merge_helper_->values()[0]);
+  ASSERT_EQ(Key("a", 20, kTypeValue), merge_helper_->keys()[0]);
+  ASSERT_EQ(EncodeInt(4U), merge_helper_->values()[0]);
  ASSERT_EQ(1U, merge_helper_->keys().size());
  ASSERT_EQ(1U, merge_helper_->values().size());
 }

 // Merging with a value results in a successful merge.
 TEST_F(MergeHelperTest, MergeValue) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
+  AddKeyVal("a", 40, kTypeMerge, EncodeInt(1U));
+  AddKeyVal("a", 30, kTypeMerge, EncodeInt(3U));
+  AddKeyVal("a", 20, kTypeValue, EncodeInt(4U));  // <- Iterator after merge
+  AddKeyVal("a", 10, kTypeMerge, EncodeInt(1U));

-  AddKeyVal("a", 40, kTypeMerge, test::EncodeInt(1U));
-  AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(3U));
-  AddKeyVal("a", 20, kTypeValue, test::EncodeInt(4U));  // <- iter_ after merge
-  AddKeyVal("a", 10, kTypeMerge, test::EncodeInt(1U));
-
-  ASSERT_TRUE(Run(0, false).ok());
+  ASSERT_TRUE(RunUInt64MergeHelper(0, false).ok());
  ASSERT_EQ(ks_[3], iter_->key());
-  ASSERT_EQ(test::KeyStr("a", 40, kTypeValue), merge_helper_->keys()[0]);
-  ASSERT_EQ(test::EncodeInt(8U), merge_helper_->values()[0]);
+  ASSERT_EQ(Key("a", 40, kTypeValue), merge_helper_->keys()[0]);
+  ASSERT_EQ(EncodeInt(8U), merge_helper_->values()[0]);
  ASSERT_EQ(1U, merge_helper_->keys().size());
  ASSERT_EQ(1U, merge_helper_->values().size());
 }

 // Merging stops before a snapshot.
 TEST_F(MergeHelperTest, SnapshotBeforeValue) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
+  AddKeyVal("a", 50, kTypeMerge, EncodeInt(1U));
+  AddKeyVal("a", 40, kTypeMerge, EncodeInt(3U));  // <- Iterator after merge
+  AddKeyVal("a", 30, kTypeMerge, EncodeInt(1U));
+  AddKeyVal("a", 20, kTypeValue, EncodeInt(4U));
+  AddKeyVal("a", 10, kTypeMerge, EncodeInt(1U));

-  AddKeyVal("a", 50, kTypeMerge, test::EncodeInt(1U));
-  AddKeyVal("a", 40, kTypeMerge, test::EncodeInt(3U));  // <- iter_ after merge
-  AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(1U));
-  AddKeyVal("a", 20, kTypeValue, test::EncodeInt(4U));
-  AddKeyVal("a", 10, kTypeMerge, test::EncodeInt(1U));
-
-  ASSERT_TRUE(Run(31, true).IsMergeInProgress());
+  ASSERT_TRUE(RunUInt64MergeHelper(31, true).IsMergeInProgress());
  ASSERT_EQ(ks_[2], iter_->key());
-  ASSERT_EQ(test::KeyStr("a", 50, kTypeMerge), merge_helper_->keys()[0]);
-  ASSERT_EQ(test::EncodeInt(4U), merge_helper_->values()[0]);
+  ASSERT_EQ(Key("a", 50, kTypeMerge), merge_helper_->keys()[0]);
+  ASSERT_EQ(EncodeInt(4U), merge_helper_->values()[0]);
  ASSERT_EQ(1U, merge_helper_->keys().size());
  ASSERT_EQ(1U, merge_helper_->values().size());
 }
@@ -107,17 +123,15 @@ TEST_F(MergeHelperTest, SnapshotBeforeValue) {
 // MergeHelper preserves the operand stack for merge operators that
 // cannot do a partial merge.
 TEST_F(MergeHelperTest, NoPartialMerge) {
-  merge_op_ = MergeOperators::CreateStringAppendTESTOperator();
-
  AddKeyVal("a", 50, kTypeMerge, "v2");
-  AddKeyVal("a", 40, kTypeMerge, "v");  // <- iter_ after merge
+  AddKeyVal("a", 40, kTypeMerge, "v");  // <- Iterator after merge
  AddKeyVal("a", 30, kTypeMerge, "v");

-  ASSERT_TRUE(Run(31, true).IsMergeInProgress());
+  ASSERT_TRUE(RunStringAppendMergeHelper(31, true).IsMergeInProgress());
  ASSERT_EQ(ks_[2], iter_->key());
-  ASSERT_EQ(test::KeyStr("a", 40, kTypeMerge), merge_helper_->keys()[0]);
+  ASSERT_EQ(Key("a", 40, kTypeMerge), merge_helper_->keys()[0]);
  ASSERT_EQ("v", merge_helper_->values()[0]);
-  ASSERT_EQ(test::KeyStr("a", 50, kTypeMerge), merge_helper_->keys()[1]);
+  ASSERT_EQ(Key("a", 50, kTypeMerge), merge_helper_->keys()[1]);
  ASSERT_EQ("v2", merge_helper_->values()[1]);
  ASSERT_EQ(2U, merge_helper_->keys().size());
  ASSERT_EQ(2U, merge_helper_->values().size());
@@ -125,162 +139,44 @@ TEST_F(MergeHelperTest, NoPartialMerge) {

 // A single operand can not be merged.
 TEST_F(MergeHelperTest, SingleOperand) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
+  AddKeyVal("a", 50, kTypeMerge, EncodeInt(1U));

-  AddKeyVal("a", 50, kTypeMerge, test::EncodeInt(1U));
-
-  ASSERT_TRUE(Run(31, true).IsMergeInProgress());
+  ASSERT_TRUE(RunUInt64MergeHelper(31, true).IsMergeInProgress());
  ASSERT_FALSE(iter_->Valid());
-  ASSERT_EQ(test::KeyStr("a", 50, kTypeMerge), merge_helper_->keys()[0]);
-  ASSERT_EQ(test::EncodeInt(1U), merge_helper_->values()[0]);
+  ASSERT_EQ(Key("a", 50, kTypeMerge), merge_helper_->keys()[0]);
+  ASSERT_EQ(EncodeInt(1U), merge_helper_->values()[0]);
  ASSERT_EQ(1U, merge_helper_->keys().size());
  ASSERT_EQ(1U, merge_helper_->values().size());
 }

 // Merging with a deletion turns the deletion into a value
 TEST_F(MergeHelperTest, MergeDeletion) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
-
-  AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(3U));
+  AddKeyVal("a", 30, kTypeMerge, EncodeInt(3U));
  AddKeyVal("a", 20, kTypeDeletion, "");

-  ASSERT_TRUE(Run(15, false).ok());
+  ASSERT_TRUE(RunUInt64MergeHelper(15, false).ok());
  ASSERT_FALSE(iter_->Valid());
-  ASSERT_EQ(test::KeyStr("a", 30, kTypeValue), merge_helper_->keys()[0]);
-  ASSERT_EQ(test::EncodeInt(3U), merge_helper_->values()[0]);
+  ASSERT_EQ(Key("a", 30, kTypeValue), merge_helper_->keys()[0]);
+  ASSERT_EQ(EncodeInt(3U), merge_helper_->values()[0]);
  ASSERT_EQ(1U, merge_helper_->keys().size());
  ASSERT_EQ(1U, merge_helper_->values().size());
 }

 // The merge helper stops upon encountering a corrupt key
 TEST_F(MergeHelperTest, CorruptKey) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
-
-  AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(3U));
-  AddKeyVal("a", 25, kTypeMerge, test::EncodeInt(1U));
+  AddKeyVal("a", 30, kTypeMerge, EncodeInt(3U));
+  AddKeyVal("a", 25, kTypeMerge, EncodeInt(1U));
  // Corrupt key
-  AddKeyVal("a", 20, kTypeDeletion, "", true);  // <- iter_ after merge
+  AddKeyVal("a", 20, kTypeDeletion, "", true);  // <- Iterator after merge

-  ASSERT_TRUE(Run(15, false).IsMergeInProgress());
+  ASSERT_TRUE(RunUInt64MergeHelper(15, false).IsMergeInProgress());
  ASSERT_EQ(ks_[2], iter_->key());
-  ASSERT_EQ(test::KeyStr("a", 30, kTypeMerge), merge_helper_->keys()[0]);
-  ASSERT_EQ(test::EncodeInt(4U), merge_helper_->values()[0]);
+  ASSERT_EQ(Key("a", 30, kTypeMerge), merge_helper_->keys()[0]);
+  ASSERT_EQ(EncodeInt(4U), merge_helper_->values()[0]);
  ASSERT_EQ(1U, merge_helper_->keys().size());
  ASSERT_EQ(1U, merge_helper_->values().size());
 }

-// The compaction filter is called on every merge operand
-TEST_F(MergeHelperTest, FilterMergeOperands) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
-  filter_.reset(new test::FilterNumber(5U));
-
-  AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(3U));
-  AddKeyVal("a", 29, kTypeMerge, test::EncodeInt(5U));  // Filtered
-  AddKeyVal("a", 28, kTypeMerge, test::EncodeInt(3U));
-  AddKeyVal("a", 27, kTypeMerge, test::EncodeInt(1U));
-  AddKeyVal("a", 26, kTypeMerge, test::EncodeInt(5U));  // Filtered
-  AddKeyVal("a", 25, kTypeValue, test::EncodeInt(1U));
-
-  ASSERT_TRUE(Run(15, false).ok());
-  ASSERT_FALSE(iter_->Valid());
-  MergeOutputIterator merge_output_iter(merge_helper_.get());
-  merge_output_iter.SeekToFirst();
-  ASSERT_EQ(test::KeyStr("a", 30, kTypeValue),
-            merge_output_iter.key().ToString());
-  ASSERT_EQ(test::EncodeInt(8U), merge_output_iter.value().ToString());
-  merge_output_iter.Next();
-  ASSERT_FALSE(merge_output_iter.Valid());
-}
-
-TEST_F(MergeHelperTest, FilterAllMergeOperands) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
-  filter_.reset(new test::FilterNumber(5U));
-
-  AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(5U));
-  AddKeyVal("a", 29, kTypeMerge, test::EncodeInt(5U));
-  AddKeyVal("a", 28, kTypeMerge, test::EncodeInt(5U));
-  AddKeyVal("a", 27, kTypeMerge, test::EncodeInt(5U));
-  AddKeyVal("a", 26, kTypeMerge, test::EncodeInt(5U));
-  AddKeyVal("a", 25, kTypeMerge, test::EncodeInt(5U));
-
-  // filtered out all
-  ASSERT_TRUE(Run(15, false).ok());
-  ASSERT_FALSE(iter_->Valid());
-  MergeOutputIterator merge_output_iter(merge_helper_.get());
-  merge_output_iter.SeekToFirst();
-  ASSERT_FALSE(merge_output_iter.Valid());
-
-  // we have one operand that will survive because it's a delete
-  AddKeyVal("a", 24, kTypeDeletion, test::EncodeInt(5U));
-  AddKeyVal("b", 23, kTypeValue, test::EncodeInt(5U));
-  ASSERT_TRUE(Run(15, true).ok());
-  merge_output_iter = MergeOutputIterator(merge_helper_.get());
-  ASSERT_TRUE(iter_->Valid());
-  merge_output_iter.SeekToFirst();
-  ASSERT_FALSE(merge_output_iter.Valid());
-
-  // when all merge operands are filtered out, we leave the iterator pointing to
-  // the Put/Delete that survived
-  ASSERT_EQ(test::KeyStr("a", 24, kTypeDeletion), iter_->key().ToString());
-  ASSERT_EQ(test::EncodeInt(5U), iter_->value().ToString());
-}
-
-// Make sure that merge operands are filtered at the beginning
-TEST_F(MergeHelperTest, FilterFirstMergeOperand) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
-  filter_.reset(new test::FilterNumber(5U));
-
-  AddKeyVal("a", 31, kTypeMerge, test::EncodeInt(5U));  // Filtered
-  AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(5U));  // Filtered
-  AddKeyVal("a", 29, kTypeMerge, test::EncodeInt(2U));
-  AddKeyVal("a", 28, kTypeMerge, test::EncodeInt(1U));
-  AddKeyVal("a", 27, kTypeMerge, test::EncodeInt(3U));
-  AddKeyVal("a", 26, kTypeMerge, test::EncodeInt(5U));  // Filtered
-  AddKeyVal("a", 25, kTypeMerge, test::EncodeInt(5U));  // Filtered
-  AddKeyVal("b", 24, kTypeValue, test::EncodeInt(5U));  // next user key
-
-  ASSERT_OK(Run(15, true));
-  ASSERT_TRUE(iter_->Valid());
-  MergeOutputIterator merge_output_iter(merge_helper_.get());
-  merge_output_iter.SeekToFirst();
-  // sequence number is 29 here, because the first merge operand got filtered
-  // out
-  ASSERT_EQ(test::KeyStr("a", 29, kTypeValue),
-            merge_output_iter.key().ToString());
-  ASSERT_EQ(test::EncodeInt(6U), merge_output_iter.value().ToString());
-  merge_output_iter.Next();
-  ASSERT_FALSE(merge_output_iter.Valid());
-
-  // make sure that we're passing user keys into the filter
-  ASSERT_EQ("a", filter_->last_merge_operand_key());
-}
-
-// Make sure that merge operands are not filtered out if there's a snapshot
-// pointing at them
-TEST_F(MergeHelperTest, DontFilterMergeOperandsBeforeSnapshotTest) {
-  merge_op_ = MergeOperators::CreateUInt64AddOperator();
-  filter_.reset(new test::FilterNumber(5U));
-
-  AddKeyVal("a", 31, kTypeMerge, test::EncodeInt(5U));
-  AddKeyVal("a", 30, kTypeMerge, test::EncodeInt(5U));
-  AddKeyVal("a", 29, kTypeMerge, test::EncodeInt(2U));
-  AddKeyVal("a", 28, kTypeMerge, test::EncodeInt(1U));
-  AddKeyVal("a", 27, kTypeMerge, test::EncodeInt(3U));
-  AddKeyVal("a", 26, kTypeMerge, test::EncodeInt(5U));
-  AddKeyVal("a", 25, kTypeMerge, test::EncodeInt(5U));
-  AddKeyVal("b", 24, kTypeValue, test::EncodeInt(5U));
-
-  ASSERT_OK(Run(15, true, 32));
-  ASSERT_TRUE(iter_->Valid());
-  MergeOutputIterator merge_output_iter(merge_helper_.get());
-  merge_output_iter.SeekToFirst();
-  ASSERT_EQ(test::KeyStr("a", 31, kTypeValue),
-            merge_output_iter.key().ToString());
-  ASSERT_EQ(test::EncodeInt(26U), merge_output_iter.value().ToString());
-  merge_output_iter.Next();
-  ASSERT_FALSE(merge_output_iter.Valid());
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -27,7 +27,6 @@
 #include "rocksdb/table.h"
 #include "table/meta_blocks.h"
 #include "table/bloom_block.h"
-#include "table/table_builder.h"
 #include "table/plain_table_factory.h"
 #include "table/plain_table_reader.h"
 #include "util/hash.h"
@@ -42,33 +41,28 @@ using std::unique_ptr;

 namespace rocksdb {

-class PlainTableDBTest : public testing::Test,
-                         public testing::WithParamInterface<bool> {
+class PlainTableDBTest : public testing::Test {
 protected:
 private:
  std::string dbname_;
  Env* env_;
  DB* db_;

-  bool mmap_mode_;
  Options last_options_;

 public:
-  PlainTableDBTest() : env_(Env::Default()) {}
-
-  ~PlainTableDBTest() {
-    delete db_;
-    EXPECT_OK(DestroyDB(dbname_, Options()));
-  }
-
-  void SetUp() override {
-    mmap_mode_ = GetParam();
+  PlainTableDBTest() : env_(Env::Default()) {
    dbname_ = test::TmpDir() + "/plain_table_db_test";
    EXPECT_OK(DestroyDB(dbname_, Options()));
    db_ = nullptr;
    Reopen();
  }

+  ~PlainTableDBTest() {
+    delete db_;
+    EXPECT_OK(DestroyDB(dbname_, Options()));
+  }
+
  // Return the current option configuration.
  Options CurrentOptions() {
    Options options;
@@ -87,7 +81,7 @@ class PlainTableDBTest : public testing::Test,
    options.memtable_factory.reset(NewHashLinkListRepFactory(4, 0, 3, true));

    options.prefix_extractor.reset(NewFixedPrefixTransform(8));
-    options.allow_mmap_reads = mmap_mode_;
+    options.allow_mmap_reads = true;
    return options;
  }

@@ -192,7 +186,7 @@ class PlainTableDBTest : public testing::Test,
  }
 };

-TEST_P(PlainTableDBTest, Empty) {
+TEST_F(PlainTableDBTest, Empty) {
  ASSERT_TRUE(dbfull() != nullptr);
  ASSERT_EQ("NOT_FOUND", Get("0000000000000foo"));
 }
@@ -213,7 +207,7 @@ class TestPlainTableReader : public PlainTableReader {
      : PlainTableReader(ioptions, std::move(file), env_options, icomparator,
                         encoding_type, file_size, table_properties),
        expect_bloom_not_match_(expect_bloom_not_match) {
-    Status s = MmapDataIfNeeded();
+    Status s = MmapDataFile();
    EXPECT_TRUE(s.ok());

    s = PopulateIndex(const_cast<TableProperties*>(table_properties),
@@ -262,29 +256,28 @@ class TestPlainTableFactory : public PlainTableFactory {
        store_index_in_file_(options.store_index_in_file),
        expect_bloom_not_match_(expect_bloom_not_match) {}

-  Status NewTableReader(const TableReaderOptions& table_reader_options,
+  Status NewTableReader(const ImmutableCFOptions& ioptions,
+                        const EnvOptions& env_options,
+                        const InternalKeyComparator& internal_comparator,
                        unique_ptr<RandomAccessFileReader>&& file,
                        uint64_t file_size,
                        unique_ptr<TableReader>* table) const override {
    TableProperties* props = nullptr;
-    auto s =
-        ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber,
-                            table_reader_options.ioptions.env,
-                            table_reader_options.ioptions.info_log, &props);
+    auto s = ReadTableProperties(file.get(), file_size, kPlainTableMagicNumber,
+                                 ioptions.env, ioptions.info_log, &props);
    EXPECT_TRUE(s.ok());

    if (store_index_in_file_) {
      BlockHandle bloom_block_handle;
      s = FindMetaBlock(file.get(), file_size, kPlainTableMagicNumber,
-                        table_reader_options.ioptions.env,
-                        BloomBlockBuilder::kBloomBlock, &bloom_block_handle);
+                        ioptions.env, BloomBlockBuilder::kBloomBlock,
+                        &bloom_block_handle);
      EXPECT_TRUE(s.ok());

      BlockHandle index_block_handle;
-      s = FindMetaBlock(file.get(), file_size, kPlainTableMagicNumber,
-                        table_reader_options.ioptions.env,
-                        PlainTableIndexBuilder::kPlainTableIndexBlock,
-                        &index_block_handle);
+      s = FindMetaBlock(
+          file.get(), file_size, kPlainTableMagicNumber, ioptions.env,
+          PlainTableIndexBuilder::kPlainTableIndexBlock, &index_block_handle);
      EXPECT_TRUE(s.ok());
    }

@@ -296,10 +289,9 @@ class TestPlainTableFactory : public PlainTableFactory {
        DecodeFixed32(encoding_type_prop->second.c_str()));

    std::unique_ptr<PlainTableReader> new_reader(new TestPlainTableReader(
-        table_reader_options.env_options,
-        table_reader_options.internal_comparator, encoding_type, file_size,
+        env_options, internal_comparator, encoding_type, file_size,
        bloom_bits_per_key_, hash_table_ratio_, index_sparseness_, props,
-        std::move(file), table_reader_options.ioptions, expect_bloom_not_match_,
+        std::move(file), ioptions, expect_bloom_not_match_,
        store_index_in_file_));

    *table = std::move(new_reader);
@@ -314,7 +306,7 @@ class TestPlainTableFactory : public PlainTableFactory {
  bool* expect_bloom_not_match_;
 };

-TEST_P(PlainTableDBTest, Flush) {
+TEST_F(PlainTableDBTest, Flush) {
  for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
       huge_page_tlb_size += 2 * 1024 * 1024) {
    for (EncodingType encoding_type : {kPlain, kPrefix}) {
@@ -401,7 +393,7 @@ TEST_P(PlainTableDBTest, Flush) {
  }
 }

-TEST_P(PlainTableDBTest, Flush2) {
+TEST_F(PlainTableDBTest, Flush2) {
  for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
       huge_page_tlb_size += 2 * 1024 * 1024) {
    for (EncodingType encoding_type : {kPlain, kPrefix}) {
@@ -481,7 +473,7 @@ TEST_P(PlainTableDBTest, Flush2) {
  }
 }

-TEST_P(PlainTableDBTest, Iterator) {
+TEST_F(PlainTableDBTest, Iterator) {
  for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
       huge_page_tlb_size += 2 * 1024 * 1024) {
    for (EncodingType encoding_type : {kPlain, kPrefix}) {
@@ -615,7 +607,7 @@ std::string MakeLongKey(size_t length, char c) {
 }
 }  // namespace

-TEST_P(PlainTableDBTest, IteratorLargeKeys) {
+TEST_F(PlainTableDBTest, IteratorLargeKeys) {
  Options options = CurrentOptions();

  PlainTableOptions plain_table_options;
@@ -665,7 +657,7 @@ std::string MakeLongKeyWithPrefix(size_t length, char c) {
 }
 }  // namespace

-TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
+TEST_F(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
  Options options = CurrentOptions();

  PlainTableOptions plain_table_options;
@@ -707,7 +699,7 @@ TEST_P(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
  delete iter;
 }

-TEST_P(PlainTableDBTest, IteratorReverseSuffixComparator) {
+TEST_F(PlainTableDBTest, IteratorReverseSuffixComparator) {
  Options options = CurrentOptions();
  options.create_if_missing = true;
  // Set only one bucket to force bucket conflict.
@@ -776,7 +768,7 @@ TEST_P(PlainTableDBTest, IteratorReverseSuffixComparator) {
  delete iter;
 }

-TEST_P(PlainTableDBTest, HashBucketConflict) {
+TEST_F(PlainTableDBTest, HashBucketConflict) {
  for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
       huge_page_tlb_size += 2 * 1024 * 1024) {
    for (unsigned char i = 1; i <= 3; i++) {
@@ -869,7 +861,7 @@ TEST_P(PlainTableDBTest, HashBucketConflict) {
  }
 }

-TEST_P(PlainTableDBTest, HashBucketConflictReverseSuffixComparator) {
+TEST_F(PlainTableDBTest, HashBucketConflictReverseSuffixComparator) {
  for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
       huge_page_tlb_size += 2 * 1024 * 1024) {
    for (unsigned char i = 1; i <= 3; i++) {
@@ -962,7 +954,7 @@ TEST_P(PlainTableDBTest, HashBucketConflictReverseSuffixComparator) {
  }
 }

-TEST_P(PlainTableDBTest, NonExistingKeyToNonEmptyBucket) {
+TEST_F(PlainTableDBTest, NonExistingKeyToNonEmptyBucket) {
  Options options = CurrentOptions();
  options.create_if_missing = true;
  // Set only one bucket to force bucket conflict.
@@ -1018,7 +1010,7 @@ static std::string RandomString(Random* rnd, int len) {
  return r;
 }

-TEST_P(PlainTableDBTest, CompactionTrigger) {
+TEST_F(PlainTableDBTest, CompactionTrigger) {
  Options options = CurrentOptions();
  options.write_buffer_size = 120 << 10;  // 100KB
  options.num_levels = 3;
@@ -1053,7 +1045,7 @@ TEST_P(PlainTableDBTest, CompactionTrigger) {
  ASSERT_EQ(NumTableFilesAtLevel(1), 1);
 }

-TEST_P(PlainTableDBTest, AdaptiveTable) {
+TEST_F(PlainTableDBTest, AdaptiveTable) {
  Options options = CurrentOptions();
  options.create_if_missing = true;

@@ -1096,8 +1088,6 @@ TEST_P(PlainTableDBTest, AdaptiveTable) {
  ASSERT_NE("v5", Get("3000000000000bar"));
 }

-INSTANTIATE_TEST_CASE_P(PlainTableDBTest, PlainTableDBTest, ::testing::Bool());
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -8,8 +8,8 @@
 #ifndef GFLAGS
 #include <cstdio>
 int main() {
-  fprintf(stderr, "Please install gflags to run this test... Skipping...\n");
-  return 0;
+  fprintf(stderr, "Please install gflags to run rocksdb tools\n");
+  return 1;
 }
 #else

@@ -15,7 +15,6 @@

 #include "rocksdb/statistics.h"
 #include "table/iterator_wrapper.h"
-#include "table/table_builder.h"
 #include "table/table_reader.h"
 #include "table/get_context.h"
 #include "util/coding.h"
@@ -107,8 +106,8 @@ Status TableCache::GetTableReader(
                                   ioptions_.statistics, record_read_stats,
                                   file_read_hist));
    s = ioptions_.table_factory->NewTableReader(
-        TableReaderOptions(ioptions_, env_options, internal_comparator),
-        std::move(file_reader), fd.GetFileSize(), table_reader);
+        ioptions_, env_options, internal_comparator, std::move(file_reader),
+        fd.GetFileSize(), table_reader);
    TEST_SYNC_POINT("TableCache::GetTableReader:0");
  }
  return s;
@@ -19,9 +19,7 @@ Status InternalKeyPropertiesCollector::InternalAdd(const Slice& key,
    return Status::InvalidArgument("Invalid internal key");
  }

-  // Note: We count both, deletions and single deletions here.
-  if (ikey.type == ValueType::kTypeDeletion ||
-      ikey.type == ValueType::kTypeSingleDeletion) {
+  if (ikey.type == ValueType::kTypeDeletion) {
    ++deleted_keys_;
  }

@@ -49,22 +47,18 @@ InternalKeyPropertiesCollector::GetReadableProperties() const {
 }

 namespace {
-
 EntryType GetEntryType(ValueType value_type) {
  switch (value_type) {
    case kTypeValue:
      return kEntryPut;
    case kTypeDeletion:
      return kEntryDelete;
-    case kTypeSingleDeletion:
-      return kEntrySingleDelete;
    case kTypeMerge:
      return kEntryMerge;
    default:
      return kEntryOther;
  }
 }
-
 }  // namespace

 Status UserKeyTablePropertiesCollector::InternalAdd(const Slice& key,
@@ -36,7 +36,7 @@ class IntTblPropCollector {
  virtual bool NeedCompact() const { return false; }
 };

-// Factory for internal table properties collector.
+// Facrtory for internal table properties collector.
 class IntTblPropCollectorFactory {
 public:
  virtual ~IntTblPropCollectorFactory() {}
@@ -6,7 +6,6 @@
 #include <map>
 #include <memory>
 #include <string>
-#include <utility>
 #include <vector>

 #include "db/db_impl.h"
@@ -59,19 +58,16 @@ class RegularKeysStartWithA: public TablePropertiesCollector {
     std::string encoded;
     std::string encoded_num_puts;
     std::string encoded_num_deletes;
-     std::string encoded_num_single_deletes;
     std::string encoded_num_size_changes;
     PutVarint32(&encoded, count_);
     PutVarint32(&encoded_num_puts, num_puts_);
     PutVarint32(&encoded_num_deletes, num_deletes_);
-     PutVarint32(&encoded_num_single_deletes, num_single_deletes_);
     PutVarint32(&encoded_num_size_changes, num_size_changes_);
     *properties = UserCollectedProperties{
         {"TablePropertiesTest", message_},
         {"Count", encoded},
         {"NumPuts", encoded_num_puts},
         {"NumDeletes", encoded_num_deletes},
-         {"NumSingleDeletes", encoded_num_single_deletes},
         {"NumSizeChanges", encoded_num_size_changes},
     };
     return Status::OK();
@@ -87,8 +83,6 @@ class RegularKeysStartWithA: public TablePropertiesCollector {
      num_puts_++;
    } else if (type == kEntryDelete) {
      num_deletes_++;
-    } else if (type == kEntrySingleDelete) {
-      num_single_deletes_++;
    }
    if (file_size < file_size_) {
      message_ = "File size should not decrease.";
@@ -108,7 +102,6 @@ class RegularKeysStartWithA: public TablePropertiesCollector {
  uint32_t count_ = 0;
  uint32_t num_puts_ = 0;
  uint32_t num_deletes_ = 0;
-  uint32_t num_single_deletes_ = 0;
  uint32_t num_size_changes_ = 0;
  uint64_t file_size_ = 0;
 };
@@ -224,18 +217,18 @@ namespace {
 void TestCustomizedTablePropertiesCollector(
    bool backward_mode, uint64_t magic_number, bool test_int_tbl_prop_collector,
    const Options& options, const InternalKeyComparator& internal_comparator) {
+  const std::string kDeleteFlag = "D";
  // make sure the entries will be inserted with order.
-  std::map<std::pair<std::string, ValueType>, std::string> kvs = {
-      {{"About   ", kTypeValue}, "val5"},  // starts with 'A'
-      {{"Abstract", kTypeValue}, "val2"},  // starts with 'A'
-      {{"Around  ", kTypeValue}, "val7"},  // starts with 'A'
-      {{"Beyond  ", kTypeValue}, "val3"},
-      {{"Builder ", kTypeValue}, "val1"},
-      {{"Love    ", kTypeDeletion}, ""},
-      {{"Cancel  ", kTypeValue}, "val4"},
-      {{"Find    ", kTypeValue}, "val6"},
-      {{"Rocks   ", kTypeDeletion}, ""},
-      {{"Foo     ", kTypeSingleDeletion}, ""},
+  std::map<std::string, std::string> kvs = {
+      {"About   ", "val5"},  // starts with 'A'
+      {"Abstract", "val2"},  // starts with 'A'
+      {"Around  ", "val7"},  // starts with 'A'
+      {"Beyond  ", "val3"},
+      {"Builder ", "val1"},
+      {"Love    ", kDeleteFlag},
+      {"Cancel  ", "val4"},
+      {"Find    ", "val6"},
+      {"Rocks   ", kDeleteFlag},
  };

  // -- Step 1: build table
@@ -255,7 +248,9 @@ void TestCustomizedTablePropertiesCollector(

  SequenceNumber seqNum = 0U;
  for (const auto& kv : kvs) {
-    InternalKey ikey(kv.first.first, seqNum++, kv.first.second);
+    InternalKey ikey(kv.first, seqNum++, (kv.second != kDeleteFlag)
+                                             ? ValueType::kTypeValue
+                                             : ValueType::kTypeDeletion);
    builder->Add(ikey.Encode(), kv.second);
  }
  ASSERT_OK(builder->Finish());
@@ -275,36 +270,31 @@ void TestCustomizedTablePropertiesCollector(

  auto user_collected = props->user_collected_properties;

-  ASSERT_NE(user_collected.find("TablePropertiesTest"), user_collected.end());
+  ASSERT_TRUE(user_collected.find("TablePropertiesTest") !=
+              user_collected.end());
  ASSERT_EQ("Rocksdb", user_collected.at("TablePropertiesTest"));

  uint32_t starts_with_A = 0;
-  ASSERT_NE(user_collected.find("Count"), user_collected.end());
+  ASSERT_TRUE(user_collected.find("Count") != user_collected.end());
  Slice key(user_collected.at("Count"));
  ASSERT_TRUE(GetVarint32(&key, &starts_with_A));
  ASSERT_EQ(3u, starts_with_A);

  if (!backward_mode && !test_int_tbl_prop_collector) {
-    uint32_t num_puts;
-    ASSERT_NE(user_collected.find("NumPuts"), user_collected.end());
-    Slice key_puts(user_collected.at("NumPuts"));
-    ASSERT_TRUE(GetVarint32(&key_puts, &num_puts));
-    ASSERT_EQ(7u, num_puts);
-
    uint32_t num_deletes;
-    ASSERT_NE(user_collected.find("NumDeletes"), user_collected.end());
+    ASSERT_TRUE(user_collected.find("NumDeletes") != user_collected.end());
    Slice key_deletes(user_collected.at("NumDeletes"));
    ASSERT_TRUE(GetVarint32(&key_deletes, &num_deletes));
    ASSERT_EQ(2u, num_deletes);

-    uint32_t num_single_deletes;
-    ASSERT_NE(user_collected.find("NumSingleDeletes"), user_collected.end());
-    Slice key_single_deletes(user_collected.at("NumSingleDeletes"));
-    ASSERT_TRUE(GetVarint32(&key_single_deletes, &num_single_deletes));
-    ASSERT_EQ(1u, num_single_deletes);
+    uint32_t num_puts;
+    ASSERT_TRUE(user_collected.find("NumPuts") != user_collected.end());
+    Slice key_puts(user_collected.at("NumPuts"));
+    ASSERT_TRUE(GetVarint32(&key_puts, &num_puts));
+    ASSERT_EQ(7u, num_puts);

    uint32_t num_size_changes;
-    ASSERT_NE(user_collected.find("NumSizeChanges"), user_collected.end());
+    ASSERT_TRUE(user_collected.find("NumSizeChanges") != user_collected.end());
    Slice key_size_changes(user_collected.at("NumSizeChanges"));
    ASSERT_TRUE(GetVarint32(&key_size_changes, &num_size_changes));
    ASSERT_GE(num_size_changes, 2u);
@@ -360,7 +350,6 @@ void TestInternalKeyPropertiesCollector(
      InternalKey("X       ", 4, ValueType::kTypeDeletion),
      InternalKey("Y       ", 5, ValueType::kTypeDeletion),
      InternalKey("Z       ", 6, ValueType::kTypeDeletion),
-      InternalKey("a       ", 7, ValueType::kTypeSingleDeletion),
  };

  std::unique_ptr<TableBuilder> builder;
@@ -414,34 +403,27 @@ void TestInternalKeyPropertiesCollector(
    std::unique_ptr<TableProperties> props_guard(props);
    auto user_collected = props->user_collected_properties;
    uint64_t deleted = GetDeletedKeys(user_collected);
-    ASSERT_EQ(5u, deleted);  // deletes + single-deletes
+    ASSERT_EQ(4u, deleted);

    if (sanitized) {
      uint32_t starts_with_A = 0;
-      ASSERT_NE(user_collected.find("Count"), user_collected.end());
+      ASSERT_TRUE(user_collected.find("Count") != user_collected.end());
      Slice key(user_collected.at("Count"));
      ASSERT_TRUE(GetVarint32(&key, &starts_with_A));
      ASSERT_EQ(1u, starts_with_A);

      if (!backward_mode) {
-        uint32_t num_puts;
-        ASSERT_NE(user_collected.find("NumPuts"), user_collected.end());
-        Slice key_puts(user_collected.at("NumPuts"));
-        ASSERT_TRUE(GetVarint32(&key_puts, &num_puts));
-        ASSERT_EQ(3u, num_puts);
-
        uint32_t num_deletes;
-        ASSERT_NE(user_collected.find("NumDeletes"), user_collected.end());
+        ASSERT_TRUE(user_collected.find("NumDeletes") != user_collected.end());
        Slice key_deletes(user_collected.at("NumDeletes"));
        ASSERT_TRUE(GetVarint32(&key_deletes, &num_deletes));
        ASSERT_EQ(4u, num_deletes);

-        uint32_t num_single_deletes;
-        ASSERT_NE(user_collected.find("NumSingleDeletes"),
-                  user_collected.end());
-        Slice key_single_deletes(user_collected.at("NumSingleDeletes"));
-        ASSERT_TRUE(GetVarint32(&key_single_deletes, &num_single_deletes));
-        ASSERT_EQ(1u, num_single_deletes);
+        uint32_t num_puts;
+        ASSERT_TRUE(user_collected.find("NumPuts") != user_collected.end());
+        Slice key_puts(user_collected.at("NumPuts"));
+        ASSERT_TRUE(GetVarint32(&key_puts, &num_puts));
+        ASSERT_EQ(3u, num_puts);
      }
    }
  }
@@ -77,7 +77,7 @@ class VersionBuilderTest : public testing::Test {
  }

  void UpdateVersionStorageInfo() {
-    vstorage_.UpdateFilesByCompactionPri(mutable_cf_options_);
+    vstorage_.UpdateFilesBySize();
    vstorage_.UpdateNumNonEmptyLevels();
    vstorage_.GenerateFileIndexer();
    vstorage_.GenerateLevelFilesBrief();
@@ -782,7 +782,7 @@ VersionStorageInfo::VersionStorageInfo(
      compaction_style_(compaction_style),
      files_(new std::vector<FileMetaData*>[num_levels_]),
      base_level_(num_levels_ == 1 ? -1 : 1),
-      files_by_compaction_pri_(num_levels_),
+      files_by_size_(num_levels_),
      level0_non_overlapping_(false),
      next_file_to_compact_by_size_(num_levels_),
      compaction_score_(num_levels_),
@@ -923,7 +923,7 @@ void Version::PrepareApply(
  UpdateAccumulatedStats(update_stats);
  storage_info_.UpdateNumNonEmptyLevels();
  storage_info_.CalculateBaseBytes(*cfd_->ioptions(), mutable_cf_options);
-  storage_info_.UpdateFilesByCompactionPri(mutable_cf_options);
+  storage_info_.UpdateFilesBySize();
  storage_info_.GenerateFileIndexer();
  storage_info_.GenerateLevelFilesBrief();
  storage_info_.GenerateLevel0NonOverlapping();
@@ -1227,6 +1227,7 @@ bool CompareCompensatedSizeDescending(const Fsize& first, const Fsize& second) {
  return (first.file->compensated_file_size >
      second.file->compensated_file_size);
 }
+
 } // anonymous namespace

 void VersionStorageInfo::AddFile(int level, FileMetaData* f) {
@@ -1244,7 +1245,7 @@ void VersionStorageInfo::AddFile(int level, FileMetaData* f) {
 // following functions called:
 // 1. UpdateNumNonEmptyLevels();
 // 2. CalculateBaseBytes();
-// 3. UpdateFilesByCompactionPri();
+// 3. UpdateFilesBySize();
 // 4. GenerateFileIndexer();
 // 5. GenerateLevelFilesBrief();
 // 6. GenerateLevel0NonOverlapping();
@@ -1296,8 +1297,7 @@ void VersionStorageInfo::UpdateNumNonEmptyLevels() {
  }
 }

-void VersionStorageInfo::UpdateFilesByCompactionPri(
-    const MutableCFOptions& mutable_cf_options) {
+void VersionStorageInfo::UpdateFilesBySize() {
  if (compaction_style_ == kCompactionStyleFIFO ||
      compaction_style_ == kCompactionStyleUniversal) {
    // don't need this
@@ -1306,8 +1306,8 @@ void VersionStorageInfo::UpdateFilesByCompactionPri(
  // No need to sort the highest level because it is never compacted.
  for (int level = 0; level < num_levels() - 1; level++) {
    const std::vector<FileMetaData*>& files = files_[level];
-    auto& files_by_compaction_pri = files_by_compaction_pri_[level];
-    assert(files_by_compaction_pri.size() == 0);
+    auto& files_by_size = files_by_size_[level];
+    assert(files_by_size.size() == 0);

    // populate a temp vector for sorting based on size
    std::vector<Fsize> temp(files.size());
@@ -1321,28 +1321,16 @@ void VersionStorageInfo::UpdateFilesByCompactionPri(
    if (num > temp.size()) {
      num = temp.size();
    }
-    switch (mutable_cf_options.compaction_pri) {
-      case kCompactionPriByCompensatedSize:
-        std::partial_sort(temp.begin(), temp.begin() + num, temp.end(),
-                          CompareCompensatedSizeDescending);
-        break;
-      case kCompactionPriByLargestSeq:
-        std::sort(temp.begin(), temp.end(),
-                  [this](const Fsize& f1, const Fsize& f2) -> bool {
-                    return f1.file->largest_seqno < f2.file->largest_seqno;
-                  });
-        break;
-      default:
-        assert(false);
-    }
+    std::partial_sort(temp.begin(), temp.begin() + num, temp.end(),
+                      CompareCompensatedSizeDescending);
    assert(temp.size() == files.size());

-    // initialize files_by_compaction_pri_
+    // initialize files_by_size_
    for (unsigned int i = 0; i < temp.size(); i++) {
-      files_by_compaction_pri.push_back(temp[i].index);
+      files_by_size.push_back(temp[i].index);
    }
    next_file_to_compact_by_size_[level] = 0;
-    assert(files_[level].size() == files_by_compaction_pri_[level].size());
+    assert(files_[level].size() == files_by_size_[level].size());
  }
 }

@@ -132,9 +132,8 @@ class VersionStorageInfo {
  // Generate level_files_brief_ from files_
  void GenerateLevelFilesBrief();
  // Sort all files for this version based on their file size and
-  // record results in files_by_compaction_pri_. The largest files are listed
-  // first.
-  void UpdateFilesByCompactionPri(const MutableCFOptions& mutable_cf_options);
+  // record results in files_by_size_. The largest files are listed first.
+  void UpdateFilesBySize();

  void GenerateLevel0NonOverlapping();
  bool level0_non_overlapping() const {
@@ -227,9 +226,9 @@ class VersionStorageInfo {
  }

  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  const std::vector<int>& FilesByCompactionPri(int level) const {
+  const std::vector<int>& FilesBySize(int level) const {
    assert(finalized_);
-    return files_by_compaction_pri_[level];
+    return files_by_size_[level];
  }

  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
@@ -243,7 +242,7 @@ class VersionStorageInfo {
  int base_level() const { return base_level_; }

  // REQUIRES: lock is held
-  // Set the index that is used to offset into files_by_compaction_pri_ to find
+  // Set the index that is used to offset into files_by_size_ to find
  // the next compaction candidate file.
  void SetNextCompactionIndex(int level, int index) {
    next_file_to_compact_by_size_[level] = index;
@@ -260,7 +259,7 @@ class VersionStorageInfo {
    return file_indexer_;
  }

-  // Only the first few entries of files_by_compaction_pri_ are sorted.
+  // Only the first few entries of files_by_size_ are sorted.
  // There is no need to sort all the files because it is likely
  // that on a running system, we need to look at only the first
  // few largest files because a new version is created every few
@@ -300,8 +299,7 @@ class VersionStorageInfo {

  uint64_t GetEstimatedActiveKeys() const;

-  // re-initializes the index that is used to offset into
-  // files_by_compaction_pri_
+  // re-initializes the index that is used to offset into files_by_size_
  // to find the next compaction candidate file.
  void ResetNextCompactionIndex(int level) {
    next_file_to_compact_by_size_[level] = 0;
@@ -353,16 +351,16 @@ class VersionStorageInfo {
  // but files in each level are now sorted based on file
  // size. The file with the largest size is at the front.
  // This vector stores the index of the file from files_.
-  std::vector<std::vector<int>> files_by_compaction_pri_;
+  std::vector<std::vector<int>> files_by_size_;

  // If true, means that files in L0 have keys with non overlapping ranges
  bool level0_non_overlapping_;

-  // An index into files_by_compaction_pri_ that specifies the first
+  // An index into files_by_size_ that specifies the first
  // file that is not yet compacted
  std::vector<int> next_file_to_compact_by_size_;

-  // Only the first few entries of files_by_compaction_pri_ are sorted.
+  // Only the first few entries of files_by_size_ are sorted.
  // There is no need to sort all the files because it is likely
  // that on a running system, we need to look at only the first
  // few largest files because a new version is created every few
@@ -515,9 +513,8 @@ class Version {
  void UpdateAccumulatedStats(bool update_stats);

  // Sort all files for this version based on their file size and
-  // record results in files_by_compaction_pri_. The largest files are listed
-  // first.
-  void UpdateFilesByCompactionPri();
+  // record results in files_by_size_. The largest files are listed first.
+  void UpdateFilesBySize();

  ColumnFamilyData* cfd_;  // ColumnFamilyData to which this Version belongs
  Logger* info_log_;
@@ -666,7 +663,6 @@ class VersionSet {
  Status GetMetadataForFile(uint64_t number, int* filelevel,
                            FileMetaData** metadata, ColumnFamilyData** cfd);

-  // This function doesn't support leveldb SST filenames
  void GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata);

  void GetObsoleteFiles(std::vector<FileMetaData*>* files,
@@ -13,13 +13,11 @@
 //    data: record[count]
 // record :=
 //    kTypeValue varstring varstring
-//    kTypeDeletion varstring
-//    kTypeSingleDeletion varstring
 //    kTypeMerge varstring varstring
+//    kTypeDeletion varstring
 //    kTypeColumnFamilyValue varint32 varstring varstring
-//    kTypeColumnFamilyDeletion varint32 varstring varstring
-//    kTypeColumnFamilySingleDeletion varint32 varstring varstring
 //    kTypeColumnFamilyMerge varint32 varstring varstring
+//    kTypeColumnFamilyDeletion varint32 varstring varstring
 // varstring :=
 //    len: varint32
 //    data: uint8[len]
@@ -112,13 +110,11 @@ Status ReadRecordFromWriteBatch(Slice* input, char* tag,
      }
      break;
    case kTypeColumnFamilyDeletion:
-    case kTypeColumnFamilySingleDeletion:
      if (!GetVarint32(input, column_family)) {
        return Status::Corruption("bad WriteBatch Delete");
      }
    // intentional fallthrough
    case kTypeDeletion:
-    case kTypeSingleDeletion:
      if (!GetLengthPrefixedSlice(input, key)) {
        return Status::Corruption("bad WriteBatch Delete");
      }
@@ -177,11 +173,6 @@ Status WriteBatch::Iterate(Handler* handler) const {
        s = handler->DeleteCF(column_family, key);
        found++;
        break;
-      case kTypeColumnFamilySingleDeletion:
-      case kTypeSingleDeletion:
-        s = handler->SingleDeleteCF(column_family, key);
-        found++;
-        break;
      case kTypeColumnFamilyMerge:
      case kTypeMerge:
        s = handler->MergeCF(column_family, key, value);
@@ -291,40 +282,6 @@ void WriteBatch::Delete(ColumnFamilyHandle* column_family,
  WriteBatchInternal::Delete(this, GetColumnFamilyID(column_family), key);
 }

-void WriteBatchInternal::SingleDelete(WriteBatch* b, uint32_t column_family_id,
-                                      const Slice& key) {
-  WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
-  if (column_family_id == 0) {
-    b->rep_.push_back(static_cast<char>(kTypeSingleDeletion));
-  } else {
-    b->rep_.push_back(static_cast<char>(kTypeColumnFamilySingleDeletion));
-    PutVarint32(&b->rep_, column_family_id);
-  }
-  PutLengthPrefixedSlice(&b->rep_, key);
-}
-
-void WriteBatch::SingleDelete(ColumnFamilyHandle* column_family,
-                              const Slice& key) {
-  WriteBatchInternal::SingleDelete(this, GetColumnFamilyID(column_family), key);
-}
-
-void WriteBatchInternal::SingleDelete(WriteBatch* b, uint32_t column_family_id,
-                                      const SliceParts& key) {
-  WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
-  if (column_family_id == 0) {
-    b->rep_.push_back(static_cast<char>(kTypeSingleDeletion));
-  } else {
-    b->rep_.push_back(static_cast<char>(kTypeColumnFamilySingleDeletion));
-    PutVarint32(&b->rep_, column_family_id);
-  }
-  PutLengthPrefixedSliceParts(&b->rep_, key);
-}
-
-void WriteBatch::SingleDelete(ColumnFamilyHandle* column_family,
-                              const SliceParts& key) {
-  WriteBatchInternal::SingleDelete(this, GetColumnFamilyID(column_family), key);
-}
-
 void WriteBatchInternal::Merge(WriteBatch* b, uint32_t column_family_id,
                               const Slice& key, const Slice& value) {
  WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
@@ -509,66 +466,6 @@ class MemTableInserter : public WriteBatch::Handler {
    return Status::OK();
  }

-  virtual Status DeleteCF(uint32_t column_family_id,
-                          const Slice& key) override {
-    Status seek_status;
-    if (!SeekToColumnFamily(column_family_id, &seek_status)) {
-      ++sequence_;
-      return seek_status;
-    }
-    MemTable* mem = cf_mems_->GetMemTable();
-    auto* moptions = mem->GetMemTableOptions();
-    if (!dont_filter_deletes_ && moptions->filter_deletes) {
-      SnapshotImpl read_from_snapshot;
-      read_from_snapshot.number_ = sequence_;
-      ReadOptions ropts;
-      ropts.snapshot = &read_from_snapshot;
-      std::string value;
-      auto cf_handle = cf_mems_->GetColumnFamilyHandle();
-      if (cf_handle == nullptr) {
-        cf_handle = db_->DefaultColumnFamily();
-      }
-      if (!db_->KeyMayExist(ropts, cf_handle, key, &value)) {
-        RecordTick(moptions->statistics, NUMBER_FILTERED_DELETES);
-        return Status::OK();
-      }
-    }
-    mem->Add(sequence_, kTypeDeletion, key, Slice());
-    sequence_++;
-    cf_mems_->CheckMemtableFull();
-    return Status::OK();
-  }
-
-  virtual Status SingleDeleteCF(uint32_t column_family_id,
-                                const Slice& key) override {
-    Status seek_status;
-    if (!SeekToColumnFamily(column_family_id, &seek_status)) {
-      ++sequence_;
-      return seek_status;
-    }
-    MemTable* mem = cf_mems_->GetMemTable();
-    auto* moptions = mem->GetMemTableOptions();
-    if (!dont_filter_deletes_ && moptions->filter_deletes) {
-      SnapshotImpl read_from_snapshot;
-      read_from_snapshot.number_ = sequence_;
-      ReadOptions ropts;
-      ropts.snapshot = &read_from_snapshot;
-      std::string value;
-      auto cf_handle = cf_mems_->GetColumnFamilyHandle();
-      if (cf_handle == nullptr) {
-        cf_handle = db_->DefaultColumnFamily();
-      }
-      if (!db_->KeyMayExist(ropts, cf_handle, key, &value)) {
-        RecordTick(moptions->statistics, NUMBER_FILTERED_DELETES);
-        return Status::OK();
-      }
-    }
-    mem->Add(sequence_, kTypeSingleDeletion, key, Slice());
-    sequence_++;
-    cf_mems_->CheckMemtableFull();
-    return Status::OK();
-  }
-
  virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
                         const Slice& value) override {
    Status seek_status;
@@ -648,6 +545,36 @@ class MemTableInserter : public WriteBatch::Handler {
    cf_mems_->CheckMemtableFull();
    return Status::OK();
  }
+
+  virtual Status DeleteCF(uint32_t column_family_id,
+                          const Slice& key) override {
+    Status seek_status;
+    if (!SeekToColumnFamily(column_family_id, &seek_status)) {
+      ++sequence_;
+      return seek_status;
+    }
+    MemTable* mem = cf_mems_->GetMemTable();
+    auto* moptions = mem->GetMemTableOptions();
+    if (!dont_filter_deletes_ && moptions->filter_deletes) {
+      SnapshotImpl read_from_snapshot;
+      read_from_snapshot.number_ = sequence_;
+      ReadOptions ropts;
+      ropts.snapshot = &read_from_snapshot;
+      std::string value;
+      auto cf_handle = cf_mems_->GetColumnFamilyHandle();
+      if (cf_handle == nullptr) {
+        cf_handle = db_->DefaultColumnFamily();
+      }
+      if (!db_->KeyMayExist(ropts, cf_handle, key, &value)) {
+        RecordTick(moptions->statistics, NUMBER_FILTERED_DELETES);
+        return Status::OK();
+      }
+    }
+    mem->Add(sequence_, kTypeDeletion, key, Slice());
+    sequence_++;
+    cf_mems_->CheckMemtableFull();
+    return Status::OK();
+  }
 };
 }  // namespace

@@ -43,19 +43,6 @@ void WriteBatchBase::Delete(const SliceParts& key) {
  Delete(key_slice);
 }

-void WriteBatchBase::SingleDelete(ColumnFamilyHandle* column_family,
-                                  const SliceParts& key) {
-  std::string key_buf;
-  Slice key_slice(key, &key_buf);
-  SingleDelete(column_family, key_slice);
-}
-
-void WriteBatchBase::SingleDelete(const SliceParts& key) {
-  std::string key_buf;
-  Slice key_slice(key, &key_buf);
-  SingleDelete(key_slice);
-}
-
 void WriteBatchBase::Merge(ColumnFamilyHandle* column_family,
                         const SliceParts& key, const SliceParts& value) {
  std::string key_buf, value_buf;
@@ -73,12 +73,6 @@ class WriteBatchInternal {
  static void Delete(WriteBatch* batch, uint32_t column_family_id,
                     const Slice& key);

-  static void SingleDelete(WriteBatch* batch, uint32_t column_family_id,
-                           const SliceParts& key);
-
-  static void SingleDelete(WriteBatch* batch, uint32_t column_family_id,
-                           const Slice& key);
-
  static void Merge(WriteBatch* batch, uint32_t column_family_id,
                    const Slice& key, const Slice& value);

@@ -54,18 +54,6 @@ static std::string PrintContents(WriteBatch* b) {
        state.append(")");
        count++;
        break;
-      case kTypeDeletion:
-        state.append("Delete(");
-        state.append(ikey.user_key.ToString());
-        state.append(")");
-        count++;
-        break;
-      case kTypeSingleDeletion:
-        state.append("SingleDelete(");
-        state.append(ikey.user_key.ToString());
-        state.append(")");
-        count++;
-        break;
      case kTypeMerge:
        state.append("Merge(");
        state.append(ikey.user_key.ToString());
@@ -74,6 +62,12 @@ static std::string PrintContents(WriteBatch* b) {
        state.append(")");
        count++;
        break;
+      case kTypeDeletion:
+        state.append("Delete(");
+        state.append(ikey.user_key.ToString());
+        state.append(")");
+        count++;
+        break;
      default:
        assert(false);
        break;
@@ -157,22 +151,6 @@ TEST_F(WriteBatchTest, Append) {
  ASSERT_EQ(4, b1.Count());
 }

-TEST_F(WriteBatchTest, SingleDeletion) {
-  WriteBatch batch;
-  WriteBatchInternal::SetSequence(&batch, 100);
-  ASSERT_EQ("", PrintContents(&batch));
-  ASSERT_EQ(0, batch.Count());
-  batch.Put("a", "va");
-  ASSERT_EQ("Put(a, va)@100", PrintContents(&batch));
-  ASSERT_EQ(1, batch.Count());
-  batch.SingleDelete("a");
-  ASSERT_EQ(
-      "SingleDelete(a)@101"
-      "Put(a, va)@100",
-      PrintContents(&batch));
-  ASSERT_EQ(2, batch.Count());
-}
-
 namespace {
  struct TestHandler : public WriteBatch::Handler {
    std::string seen;
@@ -186,26 +164,6 @@ namespace {
      }
      return Status::OK();
    }
-    virtual Status DeleteCF(uint32_t column_family_id,
-                            const Slice& key) override {
-      if (column_family_id == 0) {
-        seen += "Delete(" + key.ToString() + ")";
-      } else {
-        seen += "DeleteCF(" + ToString(column_family_id) + ", " +
-                key.ToString() + ")";
-      }
-      return Status::OK();
-    }
-    virtual Status SingleDeleteCF(uint32_t column_family_id,
-                                  const Slice& key) override {
-      if (column_family_id == 0) {
-        seen += "SingleDelete(" + key.ToString() + ")";
-      } else {
-        seen += "SingleDeleteCF(" + ToString(column_family_id) + ", " +
-                key.ToString() + ")";
-      }
-      return Status::OK();
-    }
    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
                           const Slice& value) override {
      if (column_family_id == 0) {
@@ -219,14 +177,36 @@ namespace {
    virtual void LogData(const Slice& blob) override {
      seen += "LogData(" + blob.ToString() + ")";
    }
+    virtual Status DeleteCF(uint32_t column_family_id,
+                            const Slice& key) override {
+      if (column_family_id == 0) {
+        seen += "Delete(" + key.ToString() + ")";
+      } else {
+        seen += "DeleteCF(" + ToString(column_family_id) + ", " +
+                key.ToString() + ")";
+      }
+      return Status::OK();
+    }
  };
 }

+TEST_F(WriteBatchTest, MergeNotImplemented) {
+  WriteBatch batch;
+  batch.Merge(Slice("foo"), Slice("bar"));
+  ASSERT_EQ(1, batch.Count());
+  ASSERT_EQ("Merge(foo, bar)@0",
+            PrintContents(&batch));
+
+  WriteBatch::Handler handler;
+  ASSERT_OK(batch.Iterate(&handler));
+}
+
 TEST_F(WriteBatchTest, PutNotImplemented) {
  WriteBatch batch;
  batch.Put(Slice("k1"), Slice("v1"));
  ASSERT_EQ(1, batch.Count());
-  ASSERT_EQ("Put(k1, v1)@0", PrintContents(&batch));
+  ASSERT_EQ("Put(k1, v1)@0",
+            PrintContents(&batch));

  WriteBatch::Handler handler;
  ASSERT_OK(batch.Iterate(&handler));
@@ -236,27 +216,8 @@ TEST_F(WriteBatchTest, DeleteNotImplemented) {
  WriteBatch batch;
  batch.Delete(Slice("k2"));
  ASSERT_EQ(1, batch.Count());
-  ASSERT_EQ("Delete(k2)@0", PrintContents(&batch));
-
-  WriteBatch::Handler handler;
-  ASSERT_OK(batch.Iterate(&handler));
-}
-
-TEST_F(WriteBatchTest, SingleDeleteNotImplemented) {
-  WriteBatch batch;
-  batch.SingleDelete(Slice("k2"));
-  ASSERT_EQ(1, batch.Count());
-  ASSERT_EQ("SingleDelete(k2)@0", PrintContents(&batch));
-
-  WriteBatch::Handler handler;
-  ASSERT_OK(batch.Iterate(&handler));
-}
-
-TEST_F(WriteBatchTest, MergeNotImplemented) {
-  WriteBatch batch;
-  batch.Merge(Slice("foo"), Slice("bar"));
-  ASSERT_EQ(1, batch.Count());
-  ASSERT_EQ("Merge(foo, bar)@0", PrintContents(&batch));
+  ASSERT_EQ("Delete(k2)@0",
+            PrintContents(&batch));

  WriteBatch::Handler handler;
  ASSERT_OK(batch.Iterate(&handler));
@@ -269,31 +230,27 @@ TEST_F(WriteBatchTest, Blob) {
  batch.Put(Slice("k3"), Slice("v3"));
  batch.PutLogData(Slice("blob1"));
  batch.Delete(Slice("k2"));
-  batch.SingleDelete(Slice("k3"));
  batch.PutLogData(Slice("blob2"));
  batch.Merge(Slice("foo"), Slice("bar"));
-  ASSERT_EQ(6, batch.Count());
-  ASSERT_EQ(
-      "Merge(foo, bar)@5"
-      "Put(k1, v1)@0"
-      "Delete(k2)@3"
-      "Put(k2, v2)@1"
-      "SingleDelete(k3)@4"
-      "Put(k3, v3)@2",
-      PrintContents(&batch));
+  ASSERT_EQ(5, batch.Count());
+  ASSERT_EQ("Merge(foo, bar)@4"
+            "Put(k1, v1)@0"
+            "Delete(k2)@3"
+            "Put(k2, v2)@1"
+            "Put(k3, v3)@2",
+            PrintContents(&batch));

  TestHandler handler;
  batch.Iterate(&handler);
  ASSERT_EQ(
-      "Put(k1, v1)"
-      "Put(k2, v2)"
-      "Put(k3, v3)"
-      "LogData(blob1)"
-      "Delete(k2)"
-      "SingleDelete(k3)"
-      "LogData(blob2)"
-      "Merge(foo, bar)",
-      handler.seen);
+            "Put(k1, v1)"
+            "Put(k2, v2)"
+            "Put(k3, v3)"
+            "LogData(blob1)"
+            "Delete(k2)"
+            "LogData(blob2)"
+            "Merge(foo, bar)",
+            handler.seen);
 }

 TEST_F(WriteBatchTest, Continue) {
@@ -306,16 +263,6 @@ TEST_F(WriteBatchTest, Continue) {
      ++num_seen;
      return TestHandler::PutCF(column_family_id, key, value);
    }
-    virtual Status DeleteCF(uint32_t column_family_id,
-                            const Slice& key) override {
-      ++num_seen;
-      return TestHandler::DeleteCF(column_family_id, key);
-    }
-    virtual Status SingleDeleteCF(uint32_t column_family_id,
-                                  const Slice& key) override {
-      ++num_seen;
-      return TestHandler::SingleDeleteCF(column_family_id, key);
-    }
    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
                           const Slice& value) override {
      ++num_seen;
@@ -325,24 +272,27 @@ TEST_F(WriteBatchTest, Continue) {
      ++num_seen;
      TestHandler::LogData(blob);
    }
-    virtual bool Continue() override { return num_seen < 5; }
+    virtual Status DeleteCF(uint32_t column_family_id,
+                            const Slice& key) override {
+      ++num_seen;
+      return TestHandler::DeleteCF(column_family_id, key);
+    }
+    virtual bool Continue() override {
+      return num_seen < 3;
+    }
  } handler;

  batch.Put(Slice("k1"), Slice("v1"));
-  batch.Put(Slice("k2"), Slice("v2"));
  batch.PutLogData(Slice("blob1"));
  batch.Delete(Slice("k1"));
-  batch.SingleDelete(Slice("k2"));
  batch.PutLogData(Slice("blob2"));
  batch.Merge(Slice("foo"), Slice("bar"));
  batch.Iterate(&handler);
  ASSERT_EQ(
-      "Put(k1, v1)"
-      "Put(k2, v2)"
-      "LogData(blob1)"
-      "Delete(k1)"
-      "SingleDelete(k2)",
-      handler.seen);
+            "Put(k1, v1)"
+            "LogData(blob1)"
+            "Delete(k1)",
+            handler.seen);
 }

 TEST_F(WriteBatchTest, PutGatherSlices) {
@@ -395,7 +345,6 @@ TEST_F(WriteBatchTest, ColumnFamiliesBatchTest) {
  batch.Put(&two, Slice("twofoo"), Slice("bar2"));
  batch.Put(&eight, Slice("eightfoo"), Slice("bar8"));
  batch.Delete(&eight, Slice("eightfoo"));
-  batch.SingleDelete(&two, Slice("twofoo"));
  batch.Merge(&three, Slice("threethree"), Slice("3three"));
  batch.Put(&zero, Slice("foo"), Slice("bar"));
  batch.Merge(Slice("omom"), Slice("nom"));
@@ -407,7 +356,6 @@ TEST_F(WriteBatchTest, ColumnFamiliesBatchTest) {
      "PutCF(2, twofoo, bar2)"
      "PutCF(8, eightfoo, bar8)"
      "DeleteCF(8, eightfoo)"
-      "SingleDeleteCF(2, twofoo)"
      "MergeCF(3, threethree, 3three)"
      "Put(foo, bar)"
      "Merge(omom, nom)",
@@ -422,7 +370,6 @@ TEST_F(WriteBatchTest, ColumnFamiliesBatchWithIndexTest) {
  batch.Put(&two, Slice("twofoo"), Slice("bar2"));
  batch.Put(&eight, Slice("eightfoo"), Slice("bar8"));
  batch.Delete(&eight, Slice("eightfoo"));
-  batch.SingleDelete(&two, Slice("twofoo"));
  batch.Merge(&three, Slice("threethree"), Slice("3three"));
  batch.Put(&zero, Slice("foo"), Slice("bar"));
  batch.Merge(Slice("omom"), Slice("nom"));
@@ -447,24 +394,6 @@ TEST_F(WriteBatchTest, ColumnFamiliesBatchWithIndexTest) {
  ASSERT_OK(iter->status());
  ASSERT_TRUE(!iter->Valid());

-  iter.reset(batch.NewIterator(&two));
-  iter->Seek("twofoo");
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(WriteType::kPutRecord, iter->Entry().type);
-  ASSERT_EQ("twofoo", iter->Entry().key.ToString());
-  ASSERT_EQ("bar2", iter->Entry().value.ToString());
-
-  iter->Next();
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(WriteType::kSingleDeleteRecord, iter->Entry().type);
-  ASSERT_EQ("twofoo", iter->Entry().key.ToString());
-
-  iter->Next();
-  ASSERT_OK(iter->status());
-  ASSERT_TRUE(!iter->Valid());
-
  iter.reset(batch.NewIterator());
  iter->Seek("gggg");
  ASSERT_OK(iter->status());
@@ -510,7 +439,6 @@ TEST_F(WriteBatchTest, ColumnFamiliesBatchWithIndexTest) {
      "PutCF(2, twofoo, bar2)"
      "PutCF(8, eightfoo, bar8)"
      "DeleteCF(8, eightfoo)"
-      "SingleDeleteCF(2, twofoo)"
      "MergeCF(3, threethree, 3three)"
      "Put(foo, bar)"
      "Merge(omom, nom)",
@@ -1,7 +1,6 @@
-c_simple_example
 column_families_example
-compact_files_example
-compaction_filter_example
-optimistic_transaction_example
 simple_example
+c_simple_example
+compact_files_example
 transaction_example
+optimistic_transaction_example
@@ -10,9 +10,6 @@ simple_example: librocksdb simple_example.cc
 column_families_example: librocksdb column_families_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

-compaction_filter_example: librocksdb compaction_filter_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
-
 compact_files_example: librocksdb compact_files_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

@@ -29,7 +26,7 @@ transaction_example: librocksdb transaction_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)

 clean:
-	rm -rf ./simple_example ./column_families_example ./compact_files_example ./compaction_filter_example ./c_simple_example c_simple_example.o ./optimistic_transaction_example ./transaction_example
+	rm -rf ./simple_example ./column_families_example ./compact_files_example ./c_simple_example c_simple_example.o ./optimistic_transaction_example ./transaction_example

 librocksdb:
 	cd .. && $(MAKE) librocksdb.a
@@ -1,84 +0,0 @@
-// Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
-
-#include <rocksdb/compaction_filter.h>
-#include <rocksdb/db.h>
-#include <rocksdb/merge_operator.h>
-#include <rocksdb/options.h>
-
-class MyMerge : public rocksdb::MergeOperator {
- public:
-  bool FullMerge(const rocksdb::Slice& key,
-                 const rocksdb::Slice* existing_value,
-                 const std::deque<std::string>& operand_list,
-                 std::string* new_value,
-                 rocksdb::Logger* logger) const override {
-    new_value->clear();
-    if (existing_value != nullptr) {
-      new_value->assign(existing_value->data(), existing_value->size());
-    }
-    for (const std::string& m : operand_list) {
-      fprintf(stderr, "Merge(%s)\n", m.c_str());
-      assert(m != "bad");  // the compaction filter filters out bad values
-      new_value->assign(m);
-    }
-    return true;
-  }
-
-  const char* Name() const override { return "MyMerge"; }
-};
-
-class MyFilter : public rocksdb::CompactionFilter {
- public:
-  bool Filter(int level, const rocksdb::Slice& key,
-              const rocksdb::Slice& existing_value, std::string* new_value,
-              bool* value_changed) const override {
-    fprintf(stderr, "Filter(%s)\n", key.ToString().c_str());
-    ++count_;
-    assert(*value_changed == false);
-    return false;
-  }
-
-  bool FilterMergeOperand(int level, const rocksdb::Slice& key,
-                          const rocksdb::Slice& existing_value) const override {
-    fprintf(stderr, "FilterMerge(%s)\n", key.ToString().c_str());
-    ++merge_count_;
-    return existing_value == "bad";
-  }
-
-  const char* Name() const override { return "MyFilter"; }
-
-  mutable int count_ = 0;
-  mutable int merge_count_ = 0;
-};
-
-int main() {
-  rocksdb::DB* raw_db;
-  rocksdb::Status status;
-
-  MyFilter filter;
-
-  system("rm -rf /tmp/rocksmergetest");
-  rocksdb::Options options;
-  options.create_if_missing = true;
-  options.merge_operator.reset(new MyMerge);
-  options.compaction_filter = &filter;
-  status = rocksdb::DB::Open(options, "/tmp/rocksmergetest", &raw_db);
-  assert(status.ok());
-  std::unique_ptr<rocksdb::DB> db(raw_db);
-
-  rocksdb::WriteOptions wopts;
-  db->Merge(wopts, "0", "bad");  // This is filtered out
-  db->Merge(wopts, "1", "data1");
-  db->Merge(wopts, "1", "bad");
-  db->Merge(wopts, "1", "data2");
-  db->Merge(wopts, "1", "bad");
-  db->Merge(wopts, "3", "data3");
-  db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr);
-  fprintf(stderr, "filter.count_ = %d\n", filter.count_);
-  assert(filter.count_ == 1);
-  fprintf(stderr, "filter.merge_count_ = %d\n", filter.merge_count_);
-  assert(filter.merge_count_ == 5);
-}
@@ -1,53 +0,0 @@
-# This is a RocksDB option file.
-#
-# A typical RocksDB options file has three sections, which are
-# Version, DBOptions, and more than one CFOptions.  The RocksDB
-# options file in general follows the basic INI file format
-# with the following extensions / modifications:
-#
-#  * Escaped characters
-#    We escaped the following characters:
-#     - \n -- line feed - new line
-#     - \r -- carriage return
-#     - \\ -- backslash \
-#     - \: -- colon symbol :
-#     - \# -- hash tag #
-#  * Comments
-#    We support # style comments.  Comments can appear at the ending
-#    part of a line.
-#  * Statements
-#    A statement is of the form option_name = value.
-#    Each statement contains a '=', where extra white-spaces
-#    are supported. However, we don't support multi-lined statement.
-#    Furthermore, each line can only contain at most one statement.
-#  * Section
-#    Sections are of the form [SecitonTitle "SectionArgument"],
-#    where section argument is optional.
-#  * List
-#    We use colon-separated string to represent a list.
-#    For instance, n1:n2:n3:n4 is a list containing four values.
-#
-# Below is an example of a RocksDB options file:
-[Version]
-  # The Version section stores the version information about rocksdb
-  # and option file.  This is used for handling potential format
-  # change in the future.
-  rocksdb_version=4.0.0  # We support "#" style comment.
-  options_file_version=1.0
-[DBOptions]
-  # Followed by the Version section is the DBOptions section.
-  # The value of an options can be assigned using a statement.
-  # Note that for those options that is not set in the options file,
-  # we will use the default value.
-  max_open_files=12345
-  max_background_flushes=301
-[CFOptions "default"]
-  # ColumnFamilyOptions section must follow the format of
-  # [CFOptions "cf name"].  If a rocksdb instance
-  # has multiple column families, then its CFOptions must be
-  # specified in the same order as column family creation order.
-[CFOptions "the second column family"]
-  # Each column family must have one section in the RocksDB option
-  # file even all the options of this column family are set to
-  # default value.
-[CFOptions "the third column family"]
@@ -581,6 +581,8 @@ rocksdb_options_set_rate_limit_delay_max_milliseconds(rocksdb_options_t*,
                                                      unsigned int);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_manifest_file_size(
    rocksdb_options_t*, size_t);
+extern ROCKSDB_LIBRARY_API void rocksdb_options_set_no_block_cache(
+    rocksdb_options_t*, unsigned char);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_table_cache_numshardbits(
    rocksdb_options_t*, int);
 extern ROCKSDB_LIBRARY_API void
@@ -615,6 +617,8 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_skip_log_error_on_recovery(
    rocksdb_options_t*, unsigned char);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_stats_dump_period_sec(
    rocksdb_options_t*, unsigned int);
+extern ROCKSDB_LIBRARY_API void rocksdb_options_set_block_size_deviation(
+    rocksdb_options_t*, int);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_advise_random_on_open(
    rocksdb_options_t*, unsigned char);
 extern ROCKSDB_LIBRARY_API void
@@ -104,9 +104,6 @@ class Cache {
  // returns the memory size for the entries residing in the cache.
  virtual size_t GetUsage() const = 0;

-  // returns the memory size for a specific entry in the cache.
-  virtual size_t GetUsage(Handle* handle) const = 0;
-
  // returns the memory size for the entries in use by the system
  virtual size_t GetPinnedUsage() const = 0;

@@ -51,24 +51,10 @@ class CompactionFilter {
  // output of the compaction.  The application can inspect
  // the existing value of the key and make decision based on it.
  //
-  // Key-Values that are results of merge operation during compaction are not
-  // passed into this function. Currently, when you have a mix of Put()s and
-  // Merge()s on a same key, we only guarantee to process the merge operands
-  // through the compaction filters. Put()s might be processed, or might not.
-  //
  // When the value is to be preserved, the application has the option
  // to modify the existing_value and pass it back through new_value.
  // value_changed needs to be set to true in this case.
  //
-  // If you use snapshot feature of RocksDB (i.e. call GetSnapshot() API on a
-  // DB* object), CompactionFilter might not be very useful for you. Due to
-  // guarantees we need to maintain, compaction process will not call Filter()
-  // on any keys that were written before the latest snapshot. In other words,
-  // compaction will only call Filter() on keys written after your most recent
-  // call to GetSnapshot(). In most cases, Filter() will not be called very
-  // often. This is something we're fixing. See the discussion at:
-  // https://www.facebook.com/groups/mysqlonrocksdb/permalink/999723240091865/
-  //
  // If multithreaded compaction is being used *and* a single CompactionFilter
  // instance was supplied via Options::compaction_filter, this method may be
  // called from different threads concurrently.  The application must ensure
@@ -78,23 +64,44 @@ class CompactionFilter {
  // be used by a single thread that is doing the compaction run, and this
  // call does not need to be thread-safe.  However, multiple filters may be
  // in existence and operating concurrently.
-  //
-  // The last paragraph is not true if you set max_subcompactions to more than
-  // 1. In that case, subcompaction from multiple threads may call a single
-  // CompactionFilter concurrently.
  virtual bool Filter(int level,
                      const Slice& key,
                      const Slice& existing_value,
                      std::string* new_value,
                      bool* value_changed) const = 0;

-  // The compaction process invokes this method on every merge operand. If this
-  // method returns true, the merge operand will be ignored and not written out
-  // in the compaction output
-  virtual bool FilterMergeOperand(int level, const Slice& key,
-                                  const Slice& operand) const {
-    return false;
-  }
+  // Returns a name that identifies this compaction filter.
+  // The name will be printed to LOG file on start up for diagnosis.
+  virtual const char* Name() const = 0;
+};
+
+// CompactionFilterV2 that buffers kv pairs sharing the same prefix and let
+// application layer to make individual decisions for all the kv pairs in the
+// buffer.
+class CompactionFilterV2 {
+ public:
+  virtual ~CompactionFilterV2() {}
+
+  // The compaction process invokes this method for all the kv pairs
+  // sharing the same prefix. It is a "roll-up" version of CompactionFilter.
+  //
+  // Each entry in the return vector indicates if the corresponding kv should
+  // be preserved in the output of this compaction run. The application can
+  // inspect the existing values of the keys and make decision based on it.
+  //
+  // When a value is to be preserved, the application has the option
+  // to modify the entry in existing_values and pass it back through an entry
+  // in new_values. A corresponding values_changed entry needs to be set to
+  // true in this case. Note that the new_values vector contains only changed
+  // values, i.e. new_values.size() <= values_changed.size().
+  //
+  typedef std::vector<Slice> SliceVector;
+  virtual std::vector<bool> Filter(int level,
+                                   const SliceVector& keys,
+                                   const SliceVector& existing_values,
+                                   std::vector<std::string>* new_values,
+                                   std::vector<bool>* values_changed)
+    const = 0;

  // Returns a name that identifies this compaction filter.
  // The name will be printed to LOG file on start up for diagnosis.
@@ -128,6 +135,65 @@ class DefaultCompactionFilterFactory : public CompactionFilterFactory {
  }
 };

+// Each compaction will create a new CompactionFilterV2
+//
+// CompactionFilterFactoryV2 enables application to specify a prefix and use
+// CompactionFilterV2 to filter kv-pairs in batches. Each batch contains all
+// the kv-pairs sharing the same prefix.
+//
+// This is useful for applications that require grouping kv-pairs in
+// compaction filter to make a purge/no-purge decision. For example, if the
+// key prefix is user id and the rest of key represents the type of value.
+// This batching filter will come in handy if the application's compaction
+// filter requires knowledge of all types of values for any user id.
+//
+class CompactionFilterFactoryV2 {
+ public:
+  // NOTE: CompactionFilterFactoryV2 will not delete prefix_extractor
+  explicit CompactionFilterFactoryV2(const SliceTransform* prefix_extractor)
+    : prefix_extractor_(prefix_extractor) { }
+
+  virtual ~CompactionFilterFactoryV2() { }
+
+  virtual std::unique_ptr<CompactionFilterV2> CreateCompactionFilterV2(
+    const CompactionFilterContext& context) = 0;
+
+  // Returns a name that identifies this compaction filter factory.
+  virtual const char* Name() const = 0;
+
+  const SliceTransform* GetPrefixExtractor() const {
+    return prefix_extractor_;
+  }
+
+  void SetPrefixExtractor(const SliceTransform* prefix_extractor) {
+    prefix_extractor_ = prefix_extractor;
+  }
+
+ private:
+  // Prefix extractor for compaction filter v2
+  // Keys sharing the same prefix will be buffered internally.
+  // Client can implement a Filter callback function to operate on the buffer
+  const SliceTransform* prefix_extractor_;
+};
+
+// Default implementation of CompactionFilterFactoryV2 which does not
+// return any filter
+class DefaultCompactionFilterFactoryV2 : public CompactionFilterFactoryV2 {
+ public:
+  explicit DefaultCompactionFilterFactoryV2()
+      : CompactionFilterFactoryV2(nullptr) { }
+
+  virtual std::unique_ptr<CompactionFilterV2>
+  CreateCompactionFilterV2(
+      const CompactionFilterContext& context) override {
+    return std::unique_ptr<CompactionFilterV2>(nullptr);
+  }
+
+  virtual const char* Name() const override {
+    return "DefaultCompactionFilterFactoryV2";
+  }
+};
+
 }  // namespace rocksdb

 #endif  // STORAGE_ROCKSDB_INCLUDE_COMPACTION_FILTER_H_
@@ -14,28 +14,16 @@ namespace rocksdb {

 #ifndef ROCKSDB_LITE
 // Take a map of option name and option value, apply them into the
-// base_options, and return the new options as a result.
-//
-// If input_strings_escaped is set to true, then each escaped characters
-// prefixed by '\' in the the values of the opts_map will be further
-// converted back to the raw string before assigning to the associated
-// options.
+// base_options, and return the new options as a result
 Status GetColumnFamilyOptionsFromMap(
    const ColumnFamilyOptions& base_options,
    const std::unordered_map<std::string, std::string>& opts_map,
-    ColumnFamilyOptions* new_options, bool input_strings_escaped = false);
+    ColumnFamilyOptions* new_options);

-// Take a map of option name and option value, apply them into the
-// base_options, and return the new options as a result.
-//
-// If input_strings_escaped is set to true, then each escaped characters
-// prefixed by '\' in the the values of the opts_map will be further
-// converted back to the raw string before assigning to the associated
-// options.
 Status GetDBOptionsFromMap(
    const DBOptions& base_options,
    const std::unordered_map<std::string, std::string>& opts_map,
-    DBOptions* new_options, bool input_strings_escaped = false);
+    DBOptions* new_options);

 Status GetBlockBasedTableOptionsFromMap(
    const BlockBasedTableOptions& table_options,
@@ -60,13 +48,11 @@ Status GetDBOptionsFromString(
    const std::string& opts_str,
    DBOptions* new_options);

-Status GetStringFromDBOptions(std::string* opts_str,
-                              const DBOptions& db_options,
-                              const std::string& delimiter = ";  ");
+Status GetStringFromDBOptions(const DBOptions& db_options,
+                              std::string* opts_str);

-Status GetStringFromColumnFamilyOptions(std::string* opts_str,
-                                        const ColumnFamilyOptions& db_options,
-                                        const std::string& delimiter = ";  ");
+Status GetStringFromColumnFamilyOptions(const ColumnFamilyOptions& db_options,
+                                        std::string* opts_str);

 Status GetBlockBasedTableOptionsFromString(
    const BlockBasedTableOptions& table_options,
@@ -42,7 +42,6 @@ struct FlushOptions;
 struct CompactionOptions;
 struct CompactRangeOptions;
 struct TableProperties;
-struct ExternalSstFileInfo;
 class WriteBatch;
 class Env;
 class EventListener;
@@ -185,17 +184,6 @@ class DB {
    return Delete(options, DefaultColumnFamily(), key);
  }

-  // Remove the database entry for "key". Requires that the key exists
-  // and was not overwritten. Returns OK on success, and a non-OK status
-  // on error.  It is not an error if "key" did not exist in the database.
-  // Note: consider setting options.sync = true.
-  virtual Status SingleDelete(const WriteOptions& options,
-                              ColumnFamilyHandle* column_family,
-                              const Slice& key) = 0;
-  virtual Status SingleDelete(const WriteOptions& options, const Slice& key) {
-    return SingleDelete(options, DefaultColumnFamily(), key);
-  }
-
  // Merge the database entry for "key" with "value".  Returns OK on success,
  // and a non-OK status on error. The semantics of this operation is
  // determined by the user provided merge_operator when opening DB.
@@ -524,13 +512,6 @@ class DB {
    return CompactFiles(compact_options, DefaultColumnFamily(),
                        input_file_names, output_level, output_path_id);
  }
-
-  // This function will wait until all currently running background processes
-  // finish. After it returns, no background process will be run until
-  // UnblockBackgroundWork is called
-  virtual Status PauseBackgroundWork() = 0;
-  virtual Status ContinueBackgroundWork() = 0;
-
  // Number of levels used for this DB.
  virtual int NumberLevels(ColumnFamilyHandle* column_family) = 0;
  virtual int NumberLevels() { return NumberLevels(DefaultColumnFamily()); }
@@ -663,36 +644,6 @@ class DB {
      ColumnFamilyMetaData* metadata) {
    GetColumnFamilyMetaData(DefaultColumnFamily(), metadata);
  }
-
-  // Load table file located at "file_path" into "column_family", a pointer to
-  // ExternalSstFileInfo can be used instead of "file_path" to do a blind add
-  // that wont need to read the file, move_file can be set to true to
-  // move the file instead of copying it.
-  //
-  // Current Requirements:
-  // (1) Memtable is empty.
-  // (2) All existing files (if any) have sequence number = 0.
-  // (3) Key range in loaded table file don't overlap with existing
-  //     files key ranges.
-  // (4) No other writes happen during AddFile call, otherwise
-  //     DB may get corrupted.
-  // (5) Database have at least 2 levels.
-  virtual Status AddFile(ColumnFamilyHandle* column_family,
-                         const std::string& file_path,
-                         bool move_file = false) = 0;
-  virtual Status AddFile(const std::string& file_path, bool move_file = false) {
-    return AddFile(DefaultColumnFamily(), file_path, move_file);
-  }
-
-  // Load table file with information "file_info" into "column_family"
-  virtual Status AddFile(ColumnFamilyHandle* column_family,
-                         const ExternalSstFileInfo* file_info,
-                         bool move_file = false) = 0;
-  virtual Status AddFile(const ExternalSstFileInfo* file_info,
-                         bool move_file = false) {
-    return AddFile(DefaultColumnFamily(), file_info, move_file);
-  }
-
 #endif  // ROCKSDB_LITE

  // Sets the globally unique ID created at database creation time by invoking
@@ -1,45 +0,0 @@
-//  Copyright (c) 2015, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#pragma once
-#ifndef ROCKSDB_LITE
-
-#include <string>
-
-#include "rocksdb/db.h"
-
-namespace rocksdb {
-
-struct DumpOptions {
-  // Database that will be dumped
-  std::string db_path;
-  // File location that will contain dump output
-  std::string dump_location;
-  // Dont include db information header in the dump
-  bool anonymous = false;
-};
-
-class DbDumpTool {
- public:
-  bool Run(const DumpOptions& dump_options,
-           rocksdb::Options options = rocksdb::Options());
-};
-
-struct UndumpOptions {
-  // Database that we will load the dumped file into
-  std::string db_path;
-  // File location of the dumped file that will be loaded
-  std::string dump_location;
-  // Compact the db after loading the dumped file
-  bool compact_db = false;
-};
-
-class DbUndumpTool {
- public:
-  bool Run(const UndumpOptions& undump_options,
-           rocksdb::Options options = rocksdb::Options());
-};
-}  // namespace rocksdb
-#endif  // ROCKSDB_LITE
@@ -68,9 +68,6 @@ struct EnvOptions {
   // If true, then use mmap to write data
  bool use_mmap_writes = true;

-  // If false, fallocate() calls are bypassed
-  bool allow_fallocate = true;
-
  // If true, set the FD_CLOEXEC on open fd.
  bool set_fd_cloexec = true;

@@ -396,12 +393,6 @@ class RandomAccessFile {
  virtual Status Read(uint64_t offset, size_t n, Slice* result,
                      char* scratch) const = 0;

-  // Used by the file_reader_writer to decide if the ReadAhead wrapper
-  // should simply forward the call and do not enact buffering or locking.
-  virtual bool ShouldForwardRawRequest() const {
-    return false;
-  }
-
  // Tries to get an unique ID for this file that will be the same each time
  // the file is opened (and will stay the same while the file is open).
  // Furthermore, it tries to make this ID at most "max_size" bytes. If such an
@@ -422,6 +413,7 @@ class RandomAccessFile {
              // compatibility.
  };

+
  enum AccessPattern { NORMAL, RANDOM, SEQUENTIAL, WILLNEED, DONTNEED };

  virtual void Hint(AccessPattern pattern) {}
@@ -446,35 +438,7 @@ class WritableFile {
  }
  virtual ~WritableFile();

-  // Indicates if the class makes use of unbuffered I/O
-  virtual bool UseOSBuffer() const {
-    return true;
-  }
-
-  const size_t c_DefaultPageSize = 4 * 1024;
-
-  // This is needed when you want to allocate
-  // AlignedBuffer for use with file I/O classes
-  // Used for unbuffered file I/O when UseOSBuffer() returns false
-  virtual size_t GetRequiredBufferAlignment() const {
-    return c_DefaultPageSize;
-  }
-
  virtual Status Append(const Slice& data) = 0;
-
-  // Positioned write for unbuffered access default forward
-  // to simple append as most of the tests are buffered by default
-  virtual Status PositionedAppend(const Slice& /* data */, uint64_t /* offset */) {
-    return Status::NotSupported();
-  }
-
-  // Truncate is necessary to trim the file to the correct size
-  // before closing. It is not always possible to keep track of the file
-  // size due to whole pages writes. The behavior is undefined if called
-  // with other writes to follow.
-  virtual Status Truncate(uint64_t size) {
-    return Status::OK();
-  }
  virtual Status Close() = 0;
  virtual Status Flush() = 0;
  virtual Status Sync() = 0; // sync data
@@ -875,10 +839,6 @@ class WritableFileWrapper : public WritableFile {
  explicit WritableFileWrapper(WritableFile* t) : target_(t) { }

  Status Append(const Slice& data) override { return target_->Append(data); }
-  Status PositionedAppend(const Slice& data, uint64_t offset) override {
-    return target_->PositionedAppend(data, offset);
-  }
-  Status Truncate(uint64_t size) override { return target_->Truncate(size); }
  Status Close() override { return target_->Close(); }
  Status Flush() override { return target_->Flush(); }
  Status Sync() override { return target_->Sync(); }
@@ -30,6 +30,7 @@ namespace rocksdb {
 class Cache;
 class CompactionFilter;
 class CompactionFilterFactory;
+class CompactionFilterFactoryV2;
 class Comparator;
 class Env;
 enum InfoLogLevel : unsigned char;
@@ -79,13 +80,6 @@ enum CompactionStyle : char {
  kCompactionStyleNone = 0x3,
 };

-enum CompactionPri : char {
-  // Slightly Priotize larger files by size compensated by #deletes
-  kCompactionPriByCompensatedSize = 0x0,
-  // First compact files whose data is oldest.
-  kCompactionPriByLargestSeq = 0x1,
-};
-
 enum class WALRecoveryMode : char {
  // Original levelDB recovery
  // We tolerate incomplete record in trailing data on all logs
@@ -94,7 +88,7 @@ enum class WALRecoveryMode : char {
  // Recover from clean shutdown
  // We don't expect to find any corruption in the WAL
  // Use case : This is ideal for unit tests and rare applications that
-  // can require high consistency guarantee
+  // can require high consistency gaurantee
  kAbsoluteConsistency = 0x01,
  // Recover to point-in-time consistency
  // We stop the WAL playback on discovering WAL inconsistency
@@ -102,7 +96,7 @@ enum class WALRecoveryMode : char {
  // hard disk, SSD without super capacitor that store related data
  kPointInTimeRecovery = 0x02,
  // Recovery after a disaster
-  // We ignore any corruption in the WAL and try to salvage as much data as
+  // We ignore any corruption in the  WAL and try to salvage as much data as
  // possible
  // Use case : Ideal for last ditch effort to recover data or systems that
  // operate with low grade unrelated data
@@ -225,6 +219,10 @@ struct ColumnFamilyOptions {
  // Default: nullptr
  std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;

+  // This is deprecated. Talk to us if you depend on
+  // compaction_filter_factory_v2 and we'll put it back
+  // std::shared_ptr<CompactionFilterFactoryV2> compaction_filter_factory_v2;
+
  // -------------------
  // Parameters that affect performance

@@ -501,6 +499,8 @@ struct ColumnFamilyOptions {

  // Puts are delayed to options.delayed_write_rate when any level has a
  // compaction score that exceeds soft_rate_limit. This is ignored when == 0.0.
+  // CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not
+  // hold, RocksDB will set soft_rate_limit = hard_rate_limit
  //
  // Default: 0 (disabled)
  //
@@ -510,12 +510,6 @@ struct ColumnFamilyOptions {
  // DEPRECATED -- this options is no longer usde
  double hard_rate_limit;

-  // All writes are stopped if estimated bytes needed to be compaction exceed
-  // this threshold.
-  //
-  // Default: 0 (disabled)
-  uint64_t hard_pending_compaction_bytes_limit;
-
  // DEPRECATED -- this options is no longer used
  unsigned int rate_limit_delay_max_milliseconds;

@@ -549,11 +543,6 @@ struct ColumnFamilyOptions {
  // The compaction style. Default: kCompactionStyleLevel
  CompactionStyle compaction_style;

-  // If level compaction_style = kCompactionStyleLevel, for each level,
-  // which files are prioritized to be picked to compact.
-  // Default: kCompactionPriByCompensatedSize
-  CompactionPri compaction_pri;
-
  // If true, compaction will verify checksum on every read that happens
  // as part of compaction
  //
@@ -917,7 +906,7 @@ struct DBOptions {
  // Default: 1
  int max_background_compactions;

-  // This value represents the maximum number of threads that will
+  // This integer represents the maximum number of threads that will
  // concurrently perform a compaction job by breaking it into multiple,
  // smaller ones that are run simultaneously.
  // Default: 1 (i.e. no subcompactions)
@@ -1001,9 +990,6 @@ struct DBOptions {
  // Default: false
  bool allow_mmap_writes;

-  // If false, fallocate() calls are bypassed
-  bool allow_fallocate;
-
  // Disable child process inherit open files. Default: true
  bool is_fd_close_on_exec;

@@ -1107,7 +1093,7 @@ struct DBOptions {
  bool enable_thread_tracking;

  // The limited write rate to DB if soft_rate_limit or
-  // level0_slowdown_writes_trigger is triggered. It is calculated using
+  // level0_slowdown_writes_trigger is triggered. It is calcualted using
  // size of user write requests before compression.
  // Unit: byte per second.
  //
@@ -1116,7 +1102,7 @@ struct DBOptions {

  // If true, then DB::Open() will not update the statistics used to optimize
  // compaction decision by loading table properties from many files.
-  // Turning off this feature will improve DBOpen time especially in
+  // Turning off this feature will improve DBOpen time espcially in
  // disk environment.
  //
  // Default: false
@@ -1135,7 +1121,9 @@ struct DBOptions {
 // Options to control the behavior of a database (passed to DB::Open)
 struct Options : public DBOptions, public ColumnFamilyOptions {
  // Create an Options object with default values for all fields.
-  Options() : DBOptions(), ColumnFamilyOptions() {}
+  Options() :
+    DBOptions(),
+    ColumnFamilyOptions() {}

  Options(const DBOptions& db_options,
          const ColumnFamilyOptions& column_family_options)
@@ -33,7 +33,7 @@ struct PerfContext {
  // total number of internal keys skipped over during iteration (overwritten or
  // deleted, to be more specific, hidden by a put or delete of the same key)
  uint64_t internal_key_skipped_count;
-  // total number of deletes and single deletes skipped over during iteration
+  // total number of deletes skipped over during iteration
  uint64_t internal_delete_skipped_count;

  uint64_t get_snapshot_time;          // total time spent on getting snapshot
@@ -83,14 +83,6 @@ struct PerfContext {
  uint64_t block_seek_nanos;
  // Time spent on finding or creating a table reader
  uint64_t find_table_nanos;
-  // total number of mem table bloom hits
-  uint64_t bloom_memtable_hit_count;
-  // total number of mem table bloom misses
-  uint64_t bloom_memtable_miss_count;
-  // total number of SST table bloom hits
-  uint64_t bloom_sst_hit_count;
-  // total number of SST table bloom misses
-  uint64_t bloom_sst_miss_count;
 };

 #if defined(NPERF_CONTEXT) || defined(IOS_CROSS_COMPILE)
@@ -88,9 +88,6 @@ class Slice {
            (memcmp(data_, x.data_, x.size_) == 0));
  }

-  // Compare two slices and returns the first byte where they differ
-  size_t difference_offset(const Slice& b) const;
-
 // private: make these public for rocksdbjni access
  const char* data_;
  size_t size_;
@@ -128,15 +125,6 @@ inline int Slice::compare(const Slice& b) const {
  return r;
 }

-inline size_t Slice::difference_offset(const Slice& b) const {
-  size_t off = 0;
-  const size_t len = (size_ < b.size_) ? size_ : b.size_;
-  for (; off < len; off++) {
-    if (data_[off] != b.data_[off]) break;
-  }
-  return off;
-}
-
 }  // namespace rocksdb

 #endif  // STORAGE_ROCKSDB_INCLUDE_SLICE_H_
@@ -1,77 +0,0 @@
-//  Copyright (c) 2015, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#pragma once
-#include <string>
-#include "rocksdb/env.h"
-#include "rocksdb/immutable_options.h"
-#include "rocksdb/types.h"
-
-namespace rocksdb {
-
-class Comparator;
-
-// Table Properties that are specific to tables created by SstFileWriter.
-struct ExternalSstFilePropertyNames {
-  // value of this property is a fixed int32 number.
-  static const std::string kVersion;
-};
-
-// ExternalSstFileInfo include information about sst files created
-// using SstFileWriter
-struct ExternalSstFileInfo {
-  ExternalSstFileInfo() {}
-  ExternalSstFileInfo(const std::string& _file_path,
-                      const std::string& _smallest_key,
-                      const std::string& _largest_key,
-                      SequenceNumber _sequence_number, uint64_t _file_size,
-                      int32_t _num_entries, int32_t _version)
-      : file_path(_file_path),
-        smallest_key(_smallest_key),
-        largest_key(_largest_key),
-        sequence_number(_sequence_number),
-        file_size(_file_size),
-        num_entries(_num_entries),
-        version(_version) {}
-
-  std::string file_path;           // external sst file path
-  std::string smallest_key;        // smallest user key in file
-  std::string largest_key;         // largest user key in file
-  SequenceNumber sequence_number;  // sequence number of all keys in file
-  uint64_t file_size;              // file size in bytes
-  uint64_t num_entries;            // number of entries in file
-  int32_t version;                 // file version
-};
-
-// SstFileWriter is used to create sst files that can be added to database later
-// All keys in files generated by SstFileWriter will have sequence number = 0
-class SstFileWriter {
- public:
-  SstFileWriter(const EnvOptions& env_options,
-                const ImmutableCFOptions& ioptions,
-                const Comparator* user_comparator);
-
-  ~SstFileWriter();
-
-  // Prepare SstFileWriter to write into file located at "file_path".
-  Status Open(const std::string& file_path);
-
-  // Add key, value to currently opened file
-  // REQUIRES: key is after any previously added key according to comparator.
-  Status Add(const Slice& user_key, const Slice& value);
-
-  // Finalize writing to sst file and close file.
-  //
-  // An optional ExternalSstFileInfo pointer can be passed to the function
-  // which will be populated with information about the created sst file
-  Status Finish(ExternalSstFileInfo* file_info = nullptr);
-
- private:
-  class SstFileWriterPropertiesCollectorFactory;
-  class SstFileWriterPropertiesCollector;
-  struct Rep;
-  Rep* rep_;
-};
-}  // namespace rocksdb
@@ -45,10 +45,6 @@ enum Tickers : uint32_t {
  BLOCK_CACHE_DATA_MISS,
  // # of times cache hit when accessing data block from block cache.
  BLOCK_CACHE_DATA_HIT,
-  // # of bytes read from cache.
-  BLOCK_CACHE_BYTES_READ,
-  // # of bytes written into cache.
-  BLOCK_CACHE_BYTES_WRITE,
  // # of times bloom filter has avoided file reads.
  BLOOM_FILTER_USEFUL,

@@ -86,17 +82,6 @@ enum Tickers : uint32_t {
  // For the number of logical bytes read from DB::MultiGet(),
  // please use NUMBER_MULTIGET_BYTES_READ.
  BYTES_READ,
-  // The number of calls to seek/next/prev
-  NUMBER_DB_SEEK,
-  NUMBER_DB_NEXT,
-  NUMBER_DB_PREV,
-  // The number of calls to seek/next/prev that returned data
-  NUMBER_DB_SEEK_FOUND,
-  NUMBER_DB_NEXT_FOUND,
-  NUMBER_DB_PREV_FOUND,
-  // The number of uncompressed bytes read from an iterator.
-  // Includes size of key and value.
-  ITER_BYTES_READ,
  NO_FILE_CLOSES,
  NO_FILE_OPENS,
  NO_FILE_ERRORS,
@@ -181,8 +166,6 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
    {BLOCK_CACHE_FILTER_HIT, "rocksdb.block.cache.filter.hit"},
    {BLOCK_CACHE_DATA_MISS, "rocksdb.block.cache.data.miss"},
    {BLOCK_CACHE_DATA_HIT, "rocksdb.block.cache.data.hit"},
-    {BLOCK_CACHE_BYTES_READ, "rocksdb.block.cache.bytes.read"},
-    {BLOCK_CACHE_BYTES_WRITE, "rocksdb.block.cache.bytes.write"},
    {BLOOM_FILTER_USEFUL, "rocksdb.bloom.filter.useful"},
    {MEMTABLE_HIT, "rocksdb.memtable.hit"},
    {MEMTABLE_MISS, "rocksdb.memtable.miss"},
@@ -197,13 +180,6 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
    {NUMBER_KEYS_UPDATED, "rocksdb.number.keys.updated"},
    {BYTES_WRITTEN, "rocksdb.bytes.written"},
    {BYTES_READ, "rocksdb.bytes.read"},
-    {NUMBER_DB_SEEK, "rocksdb.number.db.seek"},
-    {NUMBER_DB_NEXT, "rocksdb.number.db.next"},
-    {NUMBER_DB_PREV, "rocksdb.number.db.prev"},
-    {NUMBER_DB_SEEK_FOUND, "rocksdb.number.db.seek.found"},
-    {NUMBER_DB_NEXT_FOUND, "rocksdb.number.db.next.found"},
-    {NUMBER_DB_PREV_FOUND, "rocksdb.number.db.prev.found"},
-    {ITER_BYTES_READ, "rocksdb.db.iter.bytes.read"},
    {NO_FILE_CLOSES, "rocksdb.no.file.closes"},
    {NO_FILE_OPENS, "rocksdb.no.file.opens"},
    {NO_FILE_ERRORS, "rocksdb.no.file.errors"},
@@ -277,8 +253,6 @@ enum Histograms : uint32_t {
  DB_SEEK,
  WRITE_STALL,
  SST_READ_MICROS,
-  // The number of subcompactions actually scheduled during a compaction
-  NUM_SUBCOMPACTIONS_SCHEDULED,
  HISTOGRAM_ENUM_MAX,  // TODO(ldemailly): enforce HistogramsNameMap match
 };

@@ -305,7 +279,6 @@ const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
    {DB_SEEK, "rocksdb.db.seek.micros"},
    {WRITE_STALL, "rocksdb.db.write.stall"},
    {SST_READ_MICROS, "rocksdb.sst.read.micros"},
-    {NUM_SUBCOMPACTIONS_SCHEDULED, "rocksdb.num.subcompactions.scheduled"},
 };

 struct HistogramData {
@@ -31,7 +31,6 @@ namespace rocksdb {
 // -- Block-based Table
 class FlushBlockPolicyFactory;
 class RandomAccessFile;
-struct TableReaderOptions;
 struct TableBuilderOptions;
 class TableBuilder;
 class TableReader;
@@ -336,21 +335,21 @@ class TableFactory {
  // in parameter file. It's the caller's responsibility to make sure
  // file is in the correct format.
  //
-  // NewTableReader() is called in three places:
+  // NewTableReader() is called in two places:
  // (1) TableCache::FindTable() calls the function when table cache miss
  //     and cache the table object returned.
-  // (2) SstFileReader (for SST Dump) opens the table and dump the table
+  // (1) SstFileReader (for SST Dump) opens the table and dump the table
  //     contents using the interator of the table.
-  // (3) DBImpl::AddFile() calls this function to read the contents of
-  //     the sst file it's attempting to add
-  //
-  // table_reader_options is a TableReaderOptions which contain all the
-  //    needed parameters and configuration to open the table.
-  // file is a file handler to handle the file for the table.
-  // file_size is the physical file size of the file.
-  // table_reader is the output table reader.
+  // ImmutableCFOptions is a subset of Options that can not be altered.
+  // EnvOptions is a subset of Options that will be used by Env.
+  // Multiple configured can be accessed from there, including and not
+  // limited to block cache and key comparators.
+  // file is a file handler to handle the file for the table
+  // file_size is the physical file size of the file
+  // table_reader is the output table reader
  virtual Status NewTableReader(
-      const TableReaderOptions& table_reader_options,
+      const ImmutableCFOptions& ioptions, const EnvOptions& env_options,
+      const InternalKeyComparator& internal_comparator,
      unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
      unique_ptr<TableReader>* table_reader) const = 0;

@@ -86,7 +86,6 @@ extern const std::string kPropertiesBlock;
 enum EntryType {
  kEntryPut,
  kEntryDelete,
-  kEntrySingleDelete,
  kEntryMerge,
  kEntryOther,
 };
@@ -18,7 +18,6 @@
 #include <string>
 #include <map>
 #include <vector>
-#include <functional>

 #include "rocksdb/utilities/stackable_db.h"

@@ -93,20 +92,17 @@ struct BackupableDBOptions {
  // Default: 1
  int max_background_operations;

-  // During backup user can get callback every time next
-  // callback_trigger_interval_size bytes being copied.
-  // Default: 4194304
-  uint64_t callback_trigger_interval_size;
-
  void Dump(Logger* logger) const;

-  explicit BackupableDBOptions(
-      const std::string& _backup_dir, Env* _backup_env = nullptr,
-      bool _share_table_files = true, Logger* _info_log = nullptr,
-      bool _sync = true, bool _destroy_old_data = false,
-      bool _backup_log_files = true, uint64_t _backup_rate_limit = 0,
-      uint64_t _restore_rate_limit = 0, int _max_background_operations = 1,
-      uint64_t _callback_trigger_interval_size = 4 * 1024 * 1024)
+  explicit BackupableDBOptions(const std::string& _backup_dir,
+                               Env* _backup_env = nullptr,
+                               bool _share_table_files = true,
+                               Logger* _info_log = nullptr, bool _sync = true,
+                               bool _destroy_old_data = false,
+                               bool _backup_log_files = true,
+                               uint64_t _backup_rate_limit = 0,
+                               uint64_t _restore_rate_limit = 0,
+                               int _max_background_operations = 1)
      : backup_dir(_backup_dir),
        backup_env(_backup_env),
        share_table_files(_share_table_files),
@@ -117,8 +113,7 @@ struct BackupableDBOptions {
        backup_rate_limit(_backup_rate_limit),
        restore_rate_limit(_restore_rate_limit),
        share_files_with_checksum(false),
-        max_background_operations(_max_background_operations),
-        callback_trigger_interval_size(_callback_trigger_interval_size) {
+        max_background_operations(_max_background_operations) {
    assert(share_table_files || !share_files_with_checksum);
  }
 };
@@ -218,9 +213,7 @@ class BackupEngine {
                     const BackupableDBOptions& options,
                     BackupEngine** backup_engine_ptr);

-  virtual Status CreateNewBackup(
-      DB* db, bool flush_before_backup = false,
-      std::function<void()> progress_callback = []() {}) = 0;
+  virtual Status CreateNewBackup(DB* db, bool flush_before_backup = false) = 0;
  virtual Status PurgeOldBackups(uint32_t num_backups_to_keep) = 0;
  virtual Status DeleteBackup(BackupID backup_id) = 0;
  virtual void StopBackup() = 0;
@@ -63,18 +63,6 @@ class StackableDB : public DB {
    return db_->MultiGet(options, column_family, keys, values);
  }

-  using DB::AddFile;
-  virtual Status AddFile(ColumnFamilyHandle* column_family,
-                         const ExternalSstFileInfo* file_info,
-                         bool move_file) override {
-    return db_->AddFile(column_family, file_info, move_file);
-  }
-  virtual Status AddFile(ColumnFamilyHandle* column_family,
-                         const std::string& file_path,
-                         bool move_file) override {
-    return db_->AddFile(column_family, file_path, move_file);
-  }
-
  using DB::KeyMayExist;
  virtual bool KeyMayExist(const ReadOptions& options,
                           ColumnFamilyHandle* column_family, const Slice& key,
@@ -90,13 +78,6 @@ class StackableDB : public DB {
    return db_->Delete(wopts, column_family, key);
  }

-  using DB::SingleDelete;
-  virtual Status SingleDelete(const WriteOptions& wopts,
-                              ColumnFamilyHandle* column_family,
-                              const Slice& key) override {
-    return db_->SingleDelete(wopts, column_family, key);
-  }
-
  using DB::Merge;
  virtual Status Merge(const WriteOptions& options,
                       ColumnFamilyHandle* column_family, const Slice& key,
@@ -169,13 +150,6 @@ class StackableDB : public DB {
        output_level, output_path_id);
  }

-  virtual Status PauseBackgroundWork() override {
-    return db_->PauseBackgroundWork();
-  }
-  virtual Status ContinueBackgroundWork() override {
-    return db_->ContinueBackgroundWork();
-  }
-
  using DB::NumberLevels;
  virtual int NumberLevels(ColumnFamilyHandle* column_family) override {
    return db_->NumberLevels(column_family);
@@ -191,7 +191,7 @@ class Transaction {
  virtual Iterator* GetIterator(const ReadOptions& read_options,
                                ColumnFamilyHandle* column_family) = 0;

-  // Put, Merge, Delete, and SingleDelete behave similarly to the corresponding
+  // Put, Merge, and Delete behave similarly to their corresponding
  // functions in WriteBatch, but will also do conflict checking on the
  // keys being written.
  //
@@ -224,13 +224,6 @@ class Transaction {
                        const SliceParts& key) = 0;
  virtual Status Delete(const SliceParts& key) = 0;

-  virtual Status SingleDelete(ColumnFamilyHandle* column_family,
-                              const Slice& key) = 0;
-  virtual Status SingleDelete(const Slice& key) = 0;
-  virtual Status SingleDelete(ColumnFamilyHandle* column_family,
-                              const SliceParts& key) = 0;
-  virtual Status SingleDelete(const SliceParts& key) = 0;
-
  // PutUntracked() will write a Put to the batch of operations to be committed
  // in this transaction.  This write will only happen if this transaction
  // gets committed successfully.  But unlike Transaction::Put(),
@@ -29,16 +29,10 @@ class DB;
 struct ReadOptions;
 struct DBOptions;

-enum WriteType {
-  kPutRecord,
-  kMergeRecord,
-  kDeleteRecord,
-  kSingleDeleteRecord,
-  kLogDataRecord
-};
+enum WriteType { kPutRecord, kMergeRecord, kDeleteRecord, kLogDataRecord };

-// an entry for Put, Merge, Delete, or SingleDelete entry for write batches.
-// Used in WBWIIterator.
+// an entry for Put, Merge or Delete entry for write batches. Used in
+// WBWIIterator.
 struct WriteEntry {
  WriteType type;
  Slice key;
@@ -71,8 +65,8 @@ class WBWIIterator {

 // A WriteBatchWithIndex with a binary searchable index built for all the keys
 // inserted.
-// In Put(), Merge() Delete(), or SingleDelete(), the same function of the
-// wrapped will be called. At the same time, indexes will be built.
+// In Put(), Merge() or Delete(), the same function of the wrapped will be
+// called. At the same time, indexes will be built.
 // By calling GetWriteBatch(), a user will get the WriteBatch for the data
 // they inserted, which can be used for DB::Write().
 // A user can call NewIterator() to create an iterator.
@@ -107,11 +101,6 @@ class WriteBatchWithIndex : public WriteBatchBase {
  void Delete(ColumnFamilyHandle* column_family, const Slice& key) override;
  void Delete(const Slice& key) override;

-  using WriteBatchBase::SingleDelete;
-  void SingleDelete(ColumnFamilyHandle* column_family,
-                    const Slice& key) override;
-  void SingleDelete(const Slice& key) override;
-
  using WriteBatchBase::PutLogData;
  void PutLogData(const Slice& blob) override;

@@ -126,8 +115,6 @@ class WriteBatchWithIndex : public WriteBatchBase {
  // order given by index_comparator. For multiple updates on the same key,
  // each update will be returned as a separate entry, in the order of update
  // time.
-  //
-  // The returned iterator should be deleted by the caller.
  WBWIIterator* NewIterator(ColumnFamilyHandle* column_family);
  // Create an iterator of the default column family.
  WBWIIterator* NewIterator();
@@ -181,9 +168,8 @@ class WriteBatchWithIndex : public WriteBatchBase {
  // May be called multiple times to set multiple save points.
  void SetSavePoint() override;

-  // Remove all entries in this batch (Put, Merge, Delete, SingleDelete,
-  // PutLogData) since the most recent call to SetSavePoint() and removes the
-  // most recent save point.
+  // Remove all entries in this batch (Put, Merge, Delete, PutLogData) since the
+  // most recent call to SetSavePoint() and removes the most recent save point.
  // If there is no previous call to SetSavePoint(), behaves the same as
  // Clear().
  //
@@ -5,7 +5,7 @@
 #pragma once

 #define ROCKSDB_MAJOR 4
-#define ROCKSDB_MINOR 1
+#define ROCKSDB_MINOR 0
 #define ROCKSDB_PATCH 0

 // Do not use these. We made the mistake of declaring macros starting with
@@ -60,30 +60,6 @@ class WriteBatch : public WriteBatchBase {
    Put(nullptr, key, value);
  }

-  using WriteBatchBase::Delete;
-  // If the database contains a mapping for "key", erase it.  Else do nothing.
-  void Delete(ColumnFamilyHandle* column_family, const Slice& key) override;
-  void Delete(const Slice& key) override { Delete(nullptr, key); }
-
-  // variant that takes SliceParts
-  void Delete(ColumnFamilyHandle* column_family,
-              const SliceParts& key) override;
-  void Delete(const SliceParts& key) override { Delete(nullptr, key); }
-
-  using WriteBatchBase::SingleDelete;
-  // If the database contains a mapping for "key", erase it. Expects that the
-  // key was not overwritten. Else do nothing.
-  void SingleDelete(ColumnFamilyHandle* column_family,
-                    const Slice& key) override;
-  void SingleDelete(const Slice& key) override { SingleDelete(nullptr, key); }
-
-  // variant that takes SliceParts
-  void SingleDelete(ColumnFamilyHandle* column_family,
-                    const SliceParts& key) override;
-  void SingleDelete(const SliceParts& key) override {
-    SingleDelete(nullptr, key);
-  }
-
  using WriteBatchBase::Merge;
  // Merge "value" with the existing value of "key" in the database.
  // "key->merge(existing, value)"
@@ -100,6 +76,16 @@ class WriteBatch : public WriteBatchBase {
    Merge(nullptr, key, value);
  }

+  using WriteBatchBase::Delete;
+  // If the database contains a mapping for "key", erase it.  Else do nothing.
+  void Delete(ColumnFamilyHandle* column_family, const Slice& key) override;
+  void Delete(const Slice& key) override { Delete(nullptr, key); }
+
+  // variant that takes SliceParts
+  void Delete(ColumnFamilyHandle* column_family,
+              const SliceParts& key) override;
+  void Delete(const SliceParts& key) override { Delete(nullptr, key); }
+
  using WriteBatchBase::PutLogData;
  // Append a blob of arbitrary size to the records in this batch. The blob will
  // be stored in the transaction log but not in any other file. In particular,
@@ -149,26 +135,6 @@ class WriteBatch : public WriteBatchBase {
    }
    virtual void Put(const Slice& key, const Slice& value) {}

-    virtual Status DeleteCF(uint32_t column_family_id, const Slice& key) {
-      if (column_family_id == 0) {
-        Delete(key);
-        return Status::OK();
-      }
-      return Status::InvalidArgument(
-          "non-default column family and DeleteCF not implemented");
-    }
-    virtual void Delete(const Slice& key) {}
-
-    virtual Status SingleDeleteCF(uint32_t column_family_id, const Slice& key) {
-      if (column_family_id == 0) {
-        SingleDelete(key);
-        return Status::OK();
-      }
-      return Status::InvalidArgument(
-          "non-default column family and SingleDeleteCF not implemented");
-    }
-    virtual void SingleDelete(const Slice& key) {}
-
    // Merge and LogData are not pure virtual. Otherwise, we would break
    // existing clients of Handler on a source code level. The default
    // implementation of Merge does nothing.
@@ -185,6 +151,15 @@ class WriteBatch : public WriteBatchBase {

    // The default implementation of LogData does nothing.
    virtual void LogData(const Slice& blob);
+    virtual Status DeleteCF(uint32_t column_family_id, const Slice& key) {
+      if (column_family_id == 0) {
+        Delete(key);
+        return Status::OK();
+      }
+      return Status::InvalidArgument(
+          "non-default column family and DeleteCF not implemented");
+    }
+    virtual void Delete(const Slice& key) {}

    // Continue is called by WriteBatch::Iterate. If it returns false,
    // iteration is halted. Otherwise, it continues iterating. The default
@@ -206,8 +181,7 @@ class WriteBatch : public WriteBatchBase {
  WriteBatch* GetWriteBatch() override { return this; }

  // Constructor with a serialized string object
-  explicit WriteBatch(const std::string& rep)
-      : save_points_(nullptr), rep_(rep) {}
+  explicit WriteBatch(std::string rep) : save_points_(nullptr), rep_(rep) {}

 private:
  friend class WriteBatchInternal;
@@ -54,17 +54,6 @@ class WriteBatchBase {
  virtual void Delete(ColumnFamilyHandle* column_family, const SliceParts& key);
  virtual void Delete(const SliceParts& key);

-  // If the database contains a mapping for "key", erase it. Expects that the
-  // key was not overwritten. Else do nothing.
-  virtual void SingleDelete(ColumnFamilyHandle* column_family,
-                            const Slice& key) = 0;
-  virtual void SingleDelete(const Slice& key) = 0;
-
-  // variant that takes SliceParts
-  virtual void SingleDelete(ColumnFamilyHandle* column_family,
-                            const SliceParts& key);
-  virtual void SingleDelete(const SliceParts& key);
-
  // Append a blob of arbitrary size to the records in this batch. The blob will
  // be stored in the transaction log but not in any other file. In particular,
  // it will not be persisted to the SST files. When iterating over this
@@ -128,43 +128,43 @@ JAVA_TESTCLASSPATH = $(JAVA_JUNIT_JAR):$(JAVA_HAMCR_JAR):$(JAVA_MOCKITO_JAR):$(J
 MVN_LOCAL = ~/.m2/repository

 clean:
-	$(AM_V_at)rm -rf include/*
-	$(AM_V_at)rm -rf test-libs/
-	$(AM_V_at)rm -rf $(OUTPUT)
-	$(AM_V_at)rm -rf $(BENCHMARK_OUTPUT)
-	$(AM_V_at)rm -rf $(SAMPLES_OUTPUT)
+	rm -rf include/*
+	rm -rf test-libs/
+	rm -rf $(OUTPUT)
+	rm -rf $(BENCHMARK_OUTPUT)
+	rm -rf $(SAMPLES_OUTPUT)


 javadocs:
-	$(AM_V_GEN)mkdir -p $(JAVADOC)
-	$(AM_V_at)javadoc -d $(JAVADOC) -sourcepath $(MAIN_SRC) -subpackages org
+	mkdir -p $(JAVADOC)
+	javadoc -d $(JAVADOC) -sourcepath $(MAIN_SRC) -subpackages org

 javalib: java java_test javadocs

 java:
-	$(AM_V_GEN)mkdir -p $(MAIN_CLASSES)
-	$(AM_V_at)javac -d $(MAIN_CLASSES)\
+	mkdir -p $(MAIN_CLASSES)
+	javac -d $(MAIN_CLASSES)\
 		$(MAIN_SRC)/org/rocksdb/util/*.java\
 		$(MAIN_SRC)/org/rocksdb/*.java
-	$(AM_V_at)@cp ../HISTORY.md ./HISTORY-CPP.md
-	$(AM_V_at)@rm -f ./HISTORY-CPP.md
-	$(AM_V_at)javah -cp $(MAIN_CLASSES) -d $(NATIVE_INCLUDE) -jni $(NATIVE_JAVA_CLASSES)
+	@cp ../HISTORY.md ./HISTORY-CPP.md
+	@rm -f ./HISTORY-CPP.md
+	javah -cp $(MAIN_CLASSES) -d $(NATIVE_INCLUDE) -jni $(NATIVE_JAVA_CLASSES)

 sample: java
-	$(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES)
-	$(AM_V_at)javac -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBSample.java
-	$(AM_V_at)@rm -rf /tmp/rocksdbjni
-	$(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found
+	mkdir -p $(SAMPLES_MAIN_CLASSES)
+	javac -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBSample.java
+	@rm -rf /tmp/rocksdbjni
+	@rm -rf /tmp/rocksdbjni_not_found
 	java -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBSample /tmp/rocksdbjni
-	$(AM_V_at)@rm -rf /tmp/rocksdbjni
-	$(AM_V_at)@rm -rf /tmp/rocksdbjni_not_found
+	@rm -rf /tmp/rocksdbjni
+	@rm -rf /tmp/rocksdbjni_not_found

 column_family_sample: java
-	$(AM_V_GEN)mkdir -p $(SAMPLES_MAIN_CLASSES)
-	$(AM_V_at)javac -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBColumnFamilySample.java
-	$(AM_V_at)@rm -rf /tmp/rocksdbjni
+	mkdir -p $(SAMPLES_MAIN_CLASSES)
+	javac -cp $(MAIN_CLASSES) -d $(SAMPLES_MAIN_CLASSES) $(SAMPLES_MAIN_SRC)/RocksDBColumnFamilySample.java
+	@rm -rf /tmp/rocksdbjni
 	java -ea -Xcheck:jni -Djava.library.path=target -cp $(MAIN_CLASSES):$(SAMPLES_MAIN_CLASSES) RocksDBColumnFamilySample /tmp/rocksdbjni
-	$(AM_V_at)@rm -rf /tmp/rocksdbjni
+	@rm -rf /tmp/rocksdbjni

 resolve_test_deps:
 	test -s "$(JAVA_TEST_LIBDIR)" || mkdir -p "$(JAVA_TEST_LIBDIR)"
@@ -175,16 +175,16 @@ resolve_test_deps:
 	test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" http://central.maven.org/maven2/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar

 java_test: resolve_test_deps
-	$(AM_V_GEN)mkdir -p $(TEST_CLASSES)
-	$(AM_V_at)javac -cp $(MAIN_CLASSES):$(JAVA_TESTCLASSPATH) -d $(TEST_CLASSES)\
+	mkdir -p $(TEST_CLASSES)
+	javac -cp $(MAIN_CLASSES):$(JAVA_TESTCLASSPATH) -d $(TEST_CLASSES)\
 		$(TEST_SRC)/org/rocksdb/test/*.java\
 		$(TEST_SRC)/org/rocksdb/util/*.java\
 		$(TEST_SRC)/org/rocksdb/*.java
-	$(AM_V_at)javah -cp $(MAIN_CLASSES):$(TEST_CLASSES) -d $(NATIVE_INCLUDE) -jni $(NATIVE_JAVA_TEST_CLASSES)
+	javah -cp $(MAIN_CLASSES):$(TEST_CLASSES) -d $(NATIVE_INCLUDE) -jni $(NATIVE_JAVA_TEST_CLASSES)

 test: java resolve_test_deps java_test
 	java -ea -Xcheck:jni -Djava.library.path=target -cp "$(MAIN_CLASSES):$(TEST_CLASSES):$(JAVA_TESTCLASSPATH):target/*" org.rocksdb.test.RocksJunitRunner $(JAVA_TESTS)

 db_bench: java
-	$(AM_V_GEN)mkdir -p $(BENCHMARK_MAIN_CLASSES)
-	$(AM_V_at)javac -cp $(MAIN_CLASSES) -d $(BENCHMARK_MAIN_CLASSES) $(BENCHMARK_MAIN_SRC)/org/rocksdb/benchmark/*.java
+	mkdir -p $(BENCHMARK_MAIN_CLASSES)
+	javac -cp $(MAIN_CLASSES) -d $(BENCHMARK_MAIN_CLASSES) $(BENCHMARK_MAIN_SRC)/org/rocksdb/benchmark/*.java
@@ -15,7 +15,7 @@ Once you have these items, run this make command from RocksDB's root source dire

 This command will build RocksDB natively on OSX, and will then spin up two Vagrant Virtualbox Ubuntu images to build RocksDB for both 32-bit and 64-bit Linux. 

-You can find all native binaries and JARs in the java directory upon completion:
+You can find all native binaries and JARs in the java/target directory upon completion:

    librocksdbjni-linux32.so
    librocksdbjni-linux64.so
@@ -20,7 +20,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  end

  config.vm.provision :shell, path: "build-linux-centos.sh"
-  config.vm.synced_folder "../", "/rocksdb-build"
+  config.vm.synced_folder "../target", "/rocksdb-build"
  config.vm.synced_folder "../..", "/rocksdb", type: "rsync"
  config.vm.boot_timeout = 1200
 end
@@ -1,15 +1,15 @@
 #!/usr/bin/env bash
 # install all required packages for rocksdb that are available through yum
 ARCH=$(uname -i)
-sudo yum -y install openssl java-1.7.0-openjdk-devel.$ARCH zlib zlib-devel bzip2 bzip2-devel
+sudo yum -y install openssl java-1.7.0-openjdk-devel.$ARCH 

 # install gcc/g++ 4.8.2 via CERN (http://linux.web.cern.ch/linux/devtoolset/)
 sudo wget -O /etc/yum.repos.d/slc5-devtoolset.repo http://linuxsoft.cern.ch/cern/devtoolset/slc5-devtoolset.repo
 sudo wget -O /etc/pki/rpm-gpg/RPM-GPG-KEY-cern http://ftp.mirrorservice.org/sites/ftp.scientificlinux.org/linux/scientific/51/i386/RPM-GPG-KEYs/RPM-GPG-KEY-cern
 sudo yum -y install devtoolset-2

-wget http://gflags.googlecode.com/files/gflags-1.6.tar.gz
-tar xvfz gflags-1.6.tar.gz; cd gflags-1.6; scl enable devtoolset-2 ./configure; scl enable devtoolset-2 make; sudo make install
+wget http://gflags.googlecode.com/files/gflags-2.0-no-svn-files.tar.gz
+tar xvfz gflags-2.0-no-svn-files.tar.gz; cd gflags-2.0; scl enable devtoolset-2 ./configure; scl enable devtoolset-2 make; sudo make install
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib

 # set java home so we can build rocksdb jars
@@ -18,7 +18,7 @@ export JAVA_HOME=/usr/lib/jvm/java-1.7.0
 # build rocksdb
 cd /rocksdb
 scl enable devtoolset-2 'make jclean clean'
-scl enable devtoolset-2 'make rocksdbjavastatic'
+scl enable devtoolset-2 'PORTABLE=1 make rocksdbjavastatic'
 cp /rocksdb/java/target/librocksdbjni-* /rocksdb-build
 cp /rocksdb/java/target/rocksdbjni-* /rocksdb-build

@@ -27,7 +27,7 @@ public class InfoLogLevelTest {
    try {
      db = RocksDB.open(dbFolder.getRoot().getAbsolutePath());
      db.put("key".getBytes(), "value".getBytes());
-      assertThat(getLogContentsWithoutHeader()).isNotEmpty();
+      assertThat(getLogContents()).isNotEmpty();
    } finally {
      if (db != null) {
        db.close();
@@ -51,7 +51,7 @@ public class InfoLogLevelTest {
      db.put("key".getBytes(), "value".getBytes());
      // As InfoLogLevel is set to FATAL_LEVEL, here we expect the log
      // content to be empty.
-      assertThat(getLogContentsWithoutHeader()).isEmpty();
+      assertThat(getLogContents()).isEmpty();
    } finally {
      if (db != null) {
        db.close();
@@ -81,7 +81,7 @@ public class InfoLogLevelTest {
      db = RocksDB.open(options,
          dbFolder.getRoot().getAbsolutePath());
      db.put("key".getBytes(), "value".getBytes());
-      assertThat(getLogContentsWithoutHeader()).isEmpty();
+      assertThat(getLogContents()).isEmpty();
    } finally {
      if (db != null) {
        db.close();
@@ -112,23 +112,8 @@ public class InfoLogLevelTest {
   * @return LOG file contents as String.
   * @throws IOException if file is not found.
   */
-  private String getLogContentsWithoutHeader() throws IOException {
-    final String separator = System.getProperty("line.separator");
-    final String[] lines = new String(readAllBytes(get(
-        dbFolder.getRoot().getAbsolutePath()+ "/LOG"))).split(separator);
-
-    int first_non_header = lines.length;
-    // Identify the last line of the header
-    for (int i = lines.length - 1; i >= 0; --i) {
-      if (lines[i].indexOf("Options.") >= 0 && lines[i].indexOf(':') >= 0) {
-        first_non_header = i + 1;
-        break;
-      }
-    }
-    StringBuilder builder = new StringBuilder();
-    for (int i = first_non_header; i < lines.length; ++i) {
-      builder.append(lines[i]).append(separator);
-    }
-    return builder.toString();
+  private String getLogContents() throws IOException {
+    return new String(readAllBytes(get(
+        dbFolder.getRoot().getAbsolutePath()+ "/LOG")));
  }
 }
@@ -9,7 +9,6 @@ import org.junit.internal.RealSystem;
 import org.junit.internal.TextListener;
 import org.junit.runner.Description;
 import org.junit.runner.JUnitCore;
-import org.junit.runner.Result;

 import java.util.ArrayList;
 import java.util.List;
@@ -57,10 +56,8 @@ public class RocksJunitRunner {
      for (String arg : args) {
        classes.add(Class.forName(arg));
      }
-      final Result result = runner.run(classes.toArray(new Class[1]));
-      if(!result.wasSuccessful()) {
-        System.exit(-1);
-      }
+      runner.run(classes.toArray(new Class[1]));
+
    } catch (ClassNotFoundException e) {
      e.printStackTrace();
    }
@@ -82,7 +82,6 @@ namespace port {
 // For use at db/file_indexer.h kLevelMaxIndex
 const int kMaxInt32 = std::numeric_limits<int32_t>::max();
 const uint64_t kMaxUint64 = std::numeric_limits<uint64_t>::max();
-const size_t kMaxSizet = std::numeric_limits<size_t>::max();

 static const bool kLittleEndian = PLATFORM_IS_LITTLE_ENDIAN;
 #undef PLATFORM_IS_LITTLE_ENDIAN
@@ -30,7 +30,6 @@
 #include "util/iostats_context_imp.h"
 #include "util/rate_limiter.h"
 #include "util/sync_point.h"
-#include "util/aligned_buffer.h"

 #include "util/thread_status_updater.h"
 #include "util/thread_status_util.h"
@@ -162,6 +161,15 @@ inline int fsync(HANDLE hFile) {
  return 0;
 }

+inline size_t TruncateToPageBoundary(size_t page_size, size_t s) {
+  s -= (s & (page_size - 1));
+  assert((s % page_size) == 0);
+  return s;
+}
+
+// Roundup x to a multiple of y
+inline size_t Roundup(size_t x, size_t y) { return ((x + y - 1) / y) * y; }
+
 // SetFileInformationByHandle() is capable of fast pre-allocates.
 // However, this does not change the file end position unless the file is
 // truncated and the pre-allocated space is not considered filled with zeros.
@@ -233,14 +241,13 @@ class WinMmapReadableFile : public RandomAccessFile {
                      char* scratch) const override {
    Status s;

-    if (offset > length_) {
+    if (offset + n > length_) {
      *result = Slice();
-      return IOError(fileName_, EINVAL);
-    } else if (offset + n > length_) {
-      n = length_ - offset;
+      s = IOError(fileName_, EINVAL);
+    } else {
+      *result =
+          Slice(reinterpret_cast<const char*>(mapped_region_) + offset, n);
    }
-    *result =
-        Slice(reinterpret_cast<const char*>(mapped_region_) + offset, n);
    return s;
  }

@@ -485,6 +492,7 @@ class WinMmapFile : public WritableFile {

      size_t n = std::min(left, avail);
      memcpy(dst_, src, n);
+      IOSTATS_ADD(bytes_written, n);
      dst_ += n;
      src += n;
      left -= n;
@@ -494,12 +502,6 @@ class WinMmapFile : public WritableFile {
    return Status::OK();
  }

-  // Means Close() will properly take care of truncate
-  // and it does not need any additional information
-  virtual Status Truncate(uint64_t size) override {
-    return Status::OK();
-  }
-
  virtual Status Close() override {
    Status s;

@@ -610,6 +612,94 @@ class WinMmapFile : public WritableFile {
  }
 };

+// This class is to manage an aligned user
+// allocated buffer for unbuffered I/O purposes
+// though it does not make a difference if you need a buffer.
+class AlignedBuffer {
+  const size_t alignment_;
+  std::unique_ptr<char[]> buf_;
+  size_t capacity_;
+  size_t cursize_;
+  char* bufstart_;
+
+ public:
+  explicit AlignedBuffer(size_t alignment)
+      : alignment_(alignment), capacity_(0), cursize_(0), bufstart_(nullptr) {
+    assert(alignment > 0);
+    assert((alignment & (alignment - 1)) == 0);
+  }
+
+  size_t GetAlignment() const { return alignment_; }
+
+  size_t GetCapacity() const { return capacity_; }
+
+  size_t GetCurrentSize() const { return cursize_; }
+
+  const char* GetBufferStart() const { return bufstart_; }
+
+  void Clear() { cursize_ = 0; }
+
+  // Allocates a new buffer and sets bufstart_ to the aligned first byte
+  void AllocateNewBuffer(size_t requestedCapacity) {
+    size_t size = Roundup(requestedCapacity, alignment_);
+    buf_.reset(new char[size + alignment_]);
+
+    char* p = buf_.get();
+    bufstart_ = reinterpret_cast<char*>(
+        (reinterpret_cast<uintptr_t>(p) + (alignment_ - 1)) &
+        ~static_cast<uintptr_t>(alignment_ - 1));
+    capacity_ = size;
+    cursize_ = 0;
+  }
+
+  // Used for write
+  // Returns the number of bytes appended
+  size_t Append(const char* src, size_t append_size) {
+    size_t buffer_remaining = capacity_ - cursize_;
+    size_t to_copy = std::min(append_size, buffer_remaining);
+
+    if (to_copy > 0) {
+      memcpy(bufstart_ + cursize_, src, to_copy);
+      cursize_ += to_copy;
+    }
+    return to_copy;
+  }
+
+  size_t Read(char* dest, size_t offset, size_t read_size) const {
+    assert(offset < cursize_);
+    size_t to_read = std::min(cursize_ - offset, read_size);
+    if (to_read > 0) {
+      memcpy(dest, bufstart_ + offset, to_read);
+    }
+    return to_read;
+  }
+
+  /// Pad to alignment
+  void PadToAlignmentWith(int padding) {
+    size_t total_size = Roundup(cursize_, alignment_);
+    size_t pad_size = total_size - cursize_;
+
+    if (pad_size > 0) {
+      assert((pad_size + cursize_) <= capacity_);
+      memset(bufstart_ + cursize_, padding, pad_size);
+      cursize_ += pad_size;
+    }
+  }
+
+  // After a partial flush move the tail to the beginning of the buffer
+  void RefitTail(size_t tail_offset, size_t tail_size) {
+    if (tail_size > 0) {
+      memmove(bufstart_, bufstart_ + tail_offset, tail_size);
+    }
+    cursize_ = tail_size;
+  }
+
+  // Returns place to start writing
+  char* GetDestination() { return bufstart_ + cursize_; }
+
+  void SetSize(size_t cursize) { cursize_ = cursize; }
+};
+
 class WinSequentialFile : public SequentialFile {
 private:
  const std::string filename_;
@@ -644,7 +734,7 @@ class WinSequentialFile : public SequentialFile {
    // Windows ReadFile API accepts a DWORD.
    // While it is possible to read in a loop if n is > UINT_MAX
    // it is a highly unlikely case.
-    if (n > UINT_MAX) {
+    if (n > UINT_MAX) {      
      return IOErrorFromWindowsError(filename_, ERROR_INVALID_PARAMETER);
    }

@@ -657,6 +747,8 @@ class WinSequentialFile : public SequentialFile {
      return IOErrorFromWindowsError(filename_, GetLastError());
    }

+    IOSTATS_ADD(bytes_read, r);
+
    *result = Slice(scratch, r);

    return s;
@@ -699,13 +791,12 @@ class WinRandomAccessFile : public RandomAccessFile {
      : filename_(fname),
        hFile_(hFile),
        use_os_buffer_(options.use_os_buffer),
-        buffer_(),
+        buffer_(alignment),
        buffered_start_(0) {
    assert(!options.use_mmap_reads);

    // Unbuffered access, use internal buffer for reads
    if (!use_os_buffer_) {
-      buffer_.Alignment(alignment);
      // Random read, no need in a big buffer
      // We read things in database blocks which are likely to be similar to
      // the alignment we use.
@@ -735,7 +826,7 @@ class WinRandomAccessFile : public RandomAccessFile {
      // Let's see if at least some of the requested data is already
      // in the buffer
      if (offset >= buffered_start_ &&
-          offset < (buffered_start_ + buffer_.CurrentSize())) {
+          offset < (buffered_start_ + buffer_.GetCurrentSize())) {
        size_t buffer_offset = offset - buffered_start_;
        r = buffer_.Read(dest, buffer_offset, left);
        assert(r >= 0);
@@ -748,7 +839,7 @@ class WinRandomAccessFile : public RandomAccessFile {
      // Still some left or none was buffered
      if (left > 0) {
        // Figure out the start/end offset for reading and amount to read
-        const size_t alignment = buffer_.Alignment();
+        const size_t alignment = buffer_.GetAlignment();
        const size_t start_page_start =
            TruncateToPageBoundary(alignment, offset);
        const size_t end_page_start =
@@ -756,18 +847,21 @@ class WinRandomAccessFile : public RandomAccessFile {
        const size_t actual_bytes_toread =
            (end_page_start - start_page_start) + alignment;

-        if (buffer_.Capacity() < actual_bytes_toread) {
+        if (buffer_.GetCapacity() < actual_bytes_toread) {
          buffer_.AllocateNewBuffer(actual_bytes_toread);
        } else {
          buffer_.Clear();
        }

        SSIZE_T read = 0;
-        read = pread(hFile_, buffer_.Destination(), actual_bytes_toread,
-                      start_page_start);
+        {
+          IOSTATS_TIMER_GUARD(read_nanos);
+          read = pread(hFile_, buffer_.GetDestination(), actual_bytes_toread,
+                       start_page_start);
+        }

        if (read > 0) {
-          buffer_.Size(read);
+          buffer_.SetSize(read);
          buffered_start_ = start_page_start;

          // Let's figure out how much we read from the users standpoint
@@ -790,6 +884,7 @@ class WinRandomAccessFile : public RandomAccessFile {
      }
    }

+    IOSTATS_ADD_IF_POSITIVE(bytes_read, n - left);
    *result = Slice(scratch, (r < 0) ? 0 : n - left);

    if (r < 0) {
@@ -798,10 +893,6 @@ class WinRandomAccessFile : public RandomAccessFile {
    return s;
  }

-  virtual bool ShouldForwardRawRequest() const override {
-    return true;
-  }
-
  virtual void Hint(AccessPattern pattern) override {}

  virtual Status InvalidateCache(size_t offset, size_t length) override {
@@ -824,23 +915,33 @@ class WinRandomAccessFile : public RandomAccessFile {
 class WinWritableFile : public WritableFile {
 private:
  const std::string filename_;
-  HANDLE            hFile_;
-  const bool        use_os_buffer_;  // Used to indicate unbuffered access, the file
-  const uint64_t    alignment_;
-  // must be opened as unbuffered if false
-  uint64_t          filesize_;      // How much data is actually written disk
-  uint64_t          reservedsize_;  // how far we have reserved space
+  HANDLE hFile_;
+  AlignedBuffer buffer_;
+
+  uint64_t filesize_;      // How much data is actually written disk
+  uint64_t reservedsize_;  // how far we have reserved space
+
+  bool pending_sync_;
+
+  RateLimiter* rate_limiter_;
+
+  const bool use_os_buffer_;  // Used to indicate unbuffered access, the file
+                              // must be opened as unbuffered if false

 public:
  WinWritableFile(const std::string& fname, HANDLE hFile, size_t alignment,
                  size_t capacity, const EnvOptions& options)
      : filename_(fname),
        hFile_(hFile),
-        use_os_buffer_(options.use_os_buffer),
-        alignment_(alignment),
+        buffer_(alignment),
        filesize_(0),
-        reservedsize_(0) {
+        reservedsize_(0),
+        pending_sync_(false),
+        rate_limiter_(options.rate_limiter),
+        use_os_buffer_(options.use_os_buffer) {
    assert(!options.use_mmap_writes);
+
+    buffer_.AllocateNewBuffer(capacity);
  }

  ~WinWritableFile() {
@@ -849,84 +950,106 @@ class WinWritableFile : public WritableFile {
    }
  }

-  // Indicates if the class makes use of unbuffered I/O
-  virtual bool UseOSBuffer() const override {
-    return use_os_buffer_;
-  }
-
-  virtual size_t GetRequiredBufferAlignment() const override {
-    return alignment_;
-  }
-
  virtual Status Append(const Slice& data) override {
+    const char* src = data.data();

-    // Used for buffered access ONLY
-    assert(use_os_buffer_);
-    assert(data.size() < std::numeric_limits<int>::max());
+    assert(data.size() < INT_MAX);

+    size_t left = data.size();
    Status s;
+    pending_sync_ = true;

-    DWORD bytesWritten = 0;
-    if (!WriteFile(hFile_, data.data(),
-        data.size(), &bytesWritten, NULL)) {
-      auto lastError = GetLastError();
-      s = IOErrorFromWindowsError(
-        "Failed to WriteFile: " + filename_,
-        lastError);
+    // This would call Alloc() if we are out of blocks
+    PrepareWrite(GetFileSize(), left);
+
+    // Flush only when I/O is buffered
+    if (use_os_buffer_ &&
+        (buffer_.GetCapacity() - buffer_.GetCurrentSize()) < left) {
+      if (buffer_.GetCurrentSize() > 0) {
+        s = Flush();
+        if (!s.ok()) {
+          return s;
+        }
+      }
+
+      if (buffer_.GetCapacity() < c_OneMB) {
+        size_t desiredCapacity = buffer_.GetCapacity() * 2;
+        desiredCapacity = std::min(desiredCapacity, c_OneMB);
+        buffer_.AllocateNewBuffer(desiredCapacity);
+      }
+    }
+
+    // We always use the internal buffer for the unbuffered I/O
+    // or we simply use it for its original purpose to accumulate many small
+    // chunks
+    if (!use_os_buffer_ || (buffer_.GetCapacity() >= left)) {
+      while (left > 0) {
+        size_t appended = buffer_.Append(src, left);
+        left -= appended;
+        src += appended;
+
+        if (left > 0) {
+          s = Flush();
+          if (!s.ok()) {
+            break;
+          }
+
+          size_t cursize = buffer_.GetCurrentSize();
+          size_t capacity = buffer_.GetCapacity();
+
+          // We double the buffer here because
+          // Flush calls do not keep up with the incoming bytes
+          // This is the only place when buffer is changed with unbuffered I/O
+          if (cursize == 0 && capacity < c_OneMB) {
+            size_t desiredCapacity = capacity * 2;
+            desiredCapacity = std::min(desiredCapacity, c_OneMB);
+            buffer_.AllocateNewBuffer(desiredCapacity);
+          }
+        }
+      }
    } else {
-      assert(size_t(bytesWritten) == data.size());
-      filesize_ += data.size();
+      // Writing directly to file bypassing what is in the buffer
+      assert(buffer_.GetCurrentSize() == 0);
+      // Use rate limiter for normal I/O very large request if available
+      s = WriteBuffered(src, left);
    }

    return s;
  }

-  virtual Status PositionedAppend(const Slice& data, uint64_t offset) override {
-    Status s;
-
-    SSIZE_T ret = pwrite(hFile_, data.data(), 
-      data.size(), offset);
-
-    // Error break
-    if (ret < 0) {
-      auto lastError = GetLastError();
-      s = IOErrorFromWindowsError(
-        "Failed to pwrite for: " + filename_, lastError);
-    } else {
-      // With positional write it is not clear at all
-      // if this actually extends the filesize
-      assert(size_t(ret) == data.size());
-      filesize_ += data.size();
-    }
-    return s;
-  }
-
-  // Need to implement this so the file is truncated correctly
-  // when buffered and unbuffered mode
-  virtual Status Truncate(uint64_t size) override {
-    Status s =  ftruncate(filename_, hFile_, size);
-    if (s.ok()) {
-      filesize_ = size;
-    }
-    return s;
-  }
-
  virtual Status Close() override {
-
    Status s;

-    assert(INVALID_HANDLE_VALUE != hFile_);
+    // If there is any data in the cache not written we need to deal with it
+    const size_t cursize = buffer_.GetCurrentSize();
+    const uint64_t final_size = filesize_ + cursize;

-    if (fsync(hFile_) < 0) {
+    if (cursize > 0) {
+      // If OS buffering is on, we just flush the remainder, otherwise need
+      if (!use_os_buffer_) {
+        s = WriteUnbuffered();
+      } else {
+        s = WriteBuffered(buffer_.GetBufferStart(), cursize);
+      }
+    }
+
+    if (s.ok()) {
+      s = ftruncate(filename_, hFile_, final_size);
+    }
+
+    // Sync data if buffer was flushed
+    if (s.ok() && (cursize > 0) && fsync(hFile_) < 0) {
      auto lastError = GetLastError();
      s = IOErrorFromWindowsError("fsync failed at Close() for: " + filename_,
-        lastError);
+                                  lastError);
    }

    if (FALSE == ::CloseHandle(hFile_)) {
-      auto lastError = GetLastError();
-      s = IOErrorFromWindowsError("CloseHandle failed for: " + filename_,
-                                  lastError);
+      if (s.ok()) {
+        auto lastError = GetLastError();
+        s = IOErrorFromWindowsError("CloseHandle failed for: " + filename_,
+                                    lastError);
+      }
    }

    hFile_ = INVALID_HANDLE_VALUE;
@@ -934,18 +1057,36 @@ class WinWritableFile : public WritableFile {
  }

  // write out the cached data to the OS cache
-  // This is now taken care of the WritableFileWriter
  virtual Status Flush() override {
-    return Status::OK();
+    Status status;
+
+    if (buffer_.GetCurrentSize() > 0) {
+      if (!use_os_buffer_) {
+        status = WriteUnbuffered();
+      } else {
+        status =
+            WriteBuffered(buffer_.GetBufferStart(), buffer_.GetCurrentSize());
+        if (status.ok()) {
+          buffer_.SetSize(0);
+        }
+      }
+    }
+    return status;
  }

  virtual Status Sync() override {
-    Status s;
+    Status s = Flush();
+    if (!s.ok()) {
+      return s;
+    }
+
    // Calls flush buffers
-    if (fsync(hFile_) < 0) {
+    if (pending_sync_ && fsync(hFile_) < 0) {
      auto lastError = GetLastError();
      s = IOErrorFromWindowsError("fsync failed at Sync() for: " + filename_,
                                  lastError);
+    } else {
+      pending_sync_ = false;
    }
    return s;
  }
@@ -953,12 +1094,7 @@ class WinWritableFile : public WritableFile {
  virtual Status Fsync() override { return Sync(); }

  virtual uint64_t GetFileSize() override {
-    // Double accounting now here with WritableFileWriter
-    // and this size will be wrong when unbuffered access is used
-    // but tests implement their own writable files and do not use WritableFileWrapper
-    // so we need to squeeze a square peg through
-    // a round hole here.
-    return filesize_;
+    return filesize_ + buffer_.GetCurrentSize();
  }

  virtual Status Allocate(off_t offset, off_t len) override {
@@ -968,7 +1104,7 @@ class WinWritableFile : public WritableFile {
    // Make sure that we reserve an aligned amount of space
    // since the reservation block size is driven outside so we want
    // to check if we are ok with reservation here
-    size_t spaceToReserve = Roundup(offset + len, alignment_);
+    size_t spaceToReserve = Roundup(offset + len, buffer_.GetAlignment());
    // Nothing to do
    if (spaceToReserve <= reservedsize_) {
      return status;
@@ -981,6 +1117,133 @@ class WinWritableFile : public WritableFile {
    }
    return status;
  }
+
+ private:
+  // This method writes to disk the specified data and makes use of the rate
+  // limiter
+  // if available
+  Status WriteBuffered(const char* data, size_t size) {
+    Status s;
+    assert(use_os_buffer_);
+    const char* src = data;
+    size_t left = size;
+
+    size_t actually_written = 0;
+
+    while (left > 0) {
+      size_t bytes_allowed = RequestToken(left, false);
+
+      DWORD bytesWritten = 0;
+      if (!WriteFile(hFile_, src, bytes_allowed, &bytesWritten, NULL)) {
+        auto lastError = GetLastError();
+        s = IOErrorFromWindowsError(
+            "Failed to write buffered via rate_limiter: " + filename_,
+            lastError);
+        break;
+      } else {
+        actually_written += bytesWritten;
+        src += bytesWritten;
+        left -= bytesWritten;
+      }
+    }
+
+    IOSTATS_ADD(bytes_written, actually_written);
+    filesize_ += actually_written;
+
+    return s;
+  }
+
+  // This flushes the accumulated data in the buffer. We pad data with zeros if
+  // necessary to the whole page.
+  // However, during automatic flushes padding would not be necessary.
+  // We always use RateLimiter if available. We move (Refit) any buffer bytes
+  // that are left over the
+  // whole number of pages to be written again on the next flush because we can
+  // only write on aligned
+  // offsets.
+  Status WriteUnbuffered() {
+    Status s;
+
+    assert(!use_os_buffer_);
+    size_t alignment = buffer_.GetAlignment();
+    assert((filesize_ % alignment) == 0);
+
+    // Calculate whole page final file advance if all writes succeed
+    size_t file_advance =
+        TruncateToPageBoundary(alignment, buffer_.GetCurrentSize());
+
+    // Calculate the leftover tail, we write it here padded with zeros BUT we
+    // will write
+    // it again in the future either on Close() OR when the current whole page
+    // fills out
+    size_t leftover_tail = buffer_.GetCurrentSize() - file_advance;
+
+    // Round up and pad
+    buffer_.PadToAlignmentWith(0);
+
+    const char* src = buffer_.GetBufferStart();
+    size_t left = buffer_.GetCurrentSize();
+    uint64_t file_offset = filesize_;
+    size_t actually_written = 0;
+
+    while (left > 0) {
+      // Request how much is allowed. If this is less than one alignment we may
+      // be blocking a lot on every write
+      // because we can not write less than one alignment (page) unit thus check
+      // the configuration.
+      size_t bytes_allowed = RequestToken(left, true);
+      SSIZE_T ret = pwrite(hFile_, buffer_.GetBufferStart() + actually_written,
+                           bytes_allowed, file_offset);
+
+      // Error break
+      if (ret < 0) {
+        auto lastError = GetLastError();
+        s = IOErrorFromWindowsError(
+            "Failed to pwrite for unbuffered: " + filename_, lastError);
+        buffer_.SetSize(file_advance + leftover_tail);
+        break;
+      }
+      actually_written += ret;
+      file_offset += ret;
+      left -= ret;
+    }
+
+    IOSTATS_ADD(bytes_written, actually_written);
+
+    if (s.ok()) {
+      // Move the tail to the beginning of the buffer
+      // This never happens during normal Append but rather during
+      // explicit call to Flush()/Sync() or Close()
+      buffer_.RefitTail(file_advance, leftover_tail);
+      // This is where we start writing next time which may or not be
+      // the actual file size on disk. They match if the buffer size
+      // is a multiple of whole pages otherwise filesize_ is leftover_tail
+      // behind
+      filesize_ += file_advance;
+    }
+    return s;
+  }
+
+  // This truncates the request to a single burst bytes
+  // and then goes through the request to make sure we are
+  // satisfied in the order of the I/O priority
+  size_t RequestToken(size_t bytes, bool align) const {
+    if (rate_limiter_ && io_priority_ < Env::IO_TOTAL) {
+      bytes = std::min(
+          bytes, static_cast<size_t>(rate_limiter_->GetSingleBurstBytes()));
+
+      if (align) {
+        // Here we may actually require more than burst and block
+        // but we can not write less than one page at a time on unbuffered
+        // thus we may want not to use ratelimiter s
+        size_t alignment = buffer_.GetAlignment();
+        bytes = std::max(alignment, TruncateToPageBoundary(alignment, bytes));
+      }
+
+      rate_limiter_->Request(bytes, io_priority_);
+    }
+    return bytes;
+  }
 };

 class WinDirectory : public Directory {
@@ -1829,7 +2092,7 @@ class WinEnv : public Env {
      ThreadPool* thread_pool_;
      size_t thread_id_;  // Thread count in the thread.

-      BGThreadMetadata(ThreadPool* thread_pool, size_t thread_id)
+      explicit BGThreadMetadata(ThreadPool* thread_pool, size_t thread_id)
          : thread_pool_(thread_pool), thread_id_(thread_id) {}
    };

@@ -99,13 +99,6 @@ namespace port {
 // For use at db/file_indexer.h kLevelMaxIndex
 const int kMaxInt32 = INT32_MAX;
 const uint64_t kMaxUint64 = UINT64_MAX;
-// std::numeric_limits<size_t>::max() is not constexpr just yet
-// therefore, use the same limits
-#ifdef _WIN64
-const size_t kMaxSizet = UINT64_MAX;
-#else
-const size_t kMaxSizet = UINT_MAX;
-#endif

 const bool kLittleEndian = true;

@@ -68,7 +68,6 @@ LIB_SOURCES =                                                   \
  table/iterator.cc                                             \
  table/merger.cc                                               \
  table/meta_blocks.cc                                          \
-  table/sst_file_writer.cc                                      \
  table/plain_table_builder.cc                                  \
  table/plain_table_factory.cc                                  \
  table/plain_table_index.cc                                    \
@@ -76,7 +75,6 @@ LIB_SOURCES =                                                   \
  table/plain_table_reader.cc                                   \
  table/table_properties.cc                                     \
  table/two_level_iterator.cc                                   \
-  tools/dump/db_dump_tool.cc                                    \
  util/arena.cc                                                 \
  util/auto_roll_logger.cc                                      \
  util/bloom.cc                                                 \
@@ -131,6 +129,8 @@ LIB_SOURCES =                                                   \
  utilities/write_batch_with_index/write_batch_with_index.cc    \
  utilities/write_batch_with_index/write_batch_with_index_internal.cc    \
  util/event_logger.cc                                          \
+  util/ldb_cmd.cc                                               \
+  util/ldb_tool.cc                                              \
  util/log_buffer.cc                                            \
  util/logging.cc                                               \
  util/memenv.cc                                                \
@@ -139,12 +139,12 @@ LIB_SOURCES =                                                   \
  util/options_builder.cc                                       \
  util/options.cc                                               \
  util/options_helper.cc                                        \
-  util/options_parser.cc                                        \
  util/perf_context.cc                                          \
-  util/perf_level.cc                                            \
+  util/perf_level.cc                                          \
  util/rate_limiter.cc                                          \
  util/skiplistrep.cc                                           \
  util/slice.cc                                                 \
+  util/sst_dump_tool.cc                                         \
  util/statistics.cc                                            \
  util/status.cc                                                \
  util/status_message.cc                                        \
@@ -160,11 +160,6 @@ LIB_SOURCES =                                                   \
  util/xfunc.cc                                                 \
  util/xxhash.cc                                                \

-TOOL_SOURCES = \
-  util/ldb_cmd.cc                                               \
-  util/ldb_tool.cc                                              \
-  util/sst_dump_tool.cc                                         \
-
 MOCK_SOURCES = \
  table/mock_table.cc \
  util/mock_env.cc
@@ -263,8 +258,7 @@ TEST_BENCH_SOURCES =                                                    \
  util/testharness.cc                                                   \
  util/testutil.cc                                                      \
  util/thread_list_test.cc                                              \
-  util/thread_local_test.cc                                             \
-  util/ldb_cmd_test.cc
+  util/thread_local_test.cc

 JNI_NATIVE_SOURCES =                                          \
  java/rocksjni/backupenginejni.cc                            \
@@ -5,7 +5,6 @@
 #ifndef ROCKSDB_LITE
 #include "table/adaptive_table_factory.h"

-#include "table/table_builder.h"
 #include "table/format.h"
 #include "port/port.h"

@@ -41,7 +40,8 @@ extern const uint64_t kLegacyBlockBasedTableMagicNumber;
 extern const uint64_t kCuckooTableMagicNumber;

 Status AdaptiveTableFactory::NewTableReader(
-    const TableReaderOptions& table_reader_options,
+    const ImmutableCFOptions& ioptions, const EnvOptions& env_options,
+    const InternalKeyComparator& icomp,
    unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
    unique_ptr<TableReader>* table) const {
  Footer footer;
@@ -52,14 +52,14 @@ Status AdaptiveTableFactory::NewTableReader(
  if (footer.table_magic_number() == kPlainTableMagicNumber ||
      footer.table_magic_number() == kLegacyPlainTableMagicNumber) {
    return plain_table_factory_->NewTableReader(
-        table_reader_options, std::move(file), file_size, table);
+        ioptions, env_options, icomp, std::move(file), file_size, table);
  } else if (footer.table_magic_number() == kBlockBasedTableMagicNumber ||
      footer.table_magic_number() == kLegacyBlockBasedTableMagicNumber) {
    return block_based_table_factory_->NewTableReader(
-        table_reader_options, std::move(file), file_size, table);
+        ioptions, env_options, icomp, std::move(file), file_size, table);
  } else if (footer.table_magic_number() == kCuckooTableMagicNumber) {
    return cuckoo_table_factory_->NewTableReader(
-        table_reader_options, std::move(file), file_size, table);
+        ioptions, env_options, icomp, std::move(file), file_size, table);
  } else {
    return Status::NotSupported("Unidentified table format");
  }
@@ -33,7 +33,9 @@ class AdaptiveTableFactory : public TableFactory {

  const char* Name() const override { return "AdaptiveTableFactory"; }

-  Status NewTableReader(const TableReaderOptions& table_reader_options,
+  Status NewTableReader(const ImmutableCFOptions& ioptions,
+                        const EnvOptions& env_options,
+                        const InternalKeyComparator& internal_comparator,
                        unique_ptr<RandomAccessFileReader>&& file,
                        uint64_t file_size,
                        unique_ptr<TableReader>* table) const override;
@@ -45,6 +47,10 @@ class AdaptiveTableFactory : public TableFactory {
  // Sanitizes the specified DB Options.
  Status SanitizeOptions(const DBOptions& db_opts,
                         const ColumnFamilyOptions& cf_opts) const override {
+    if (db_opts.allow_mmap_reads == false) {
+      return Status::NotSupported(
+          "AdaptiveTable with allow_mmap_reads == false is not supported.");
+    }
    return Status::OK();
  }

@@ -7,13 +7,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

-#include "table/block_based_filter_block.h"
 #include <algorithm>
+#include "table/block_based_filter_block.h"

 #include "db/dbformat.h"
 #include "rocksdb/filter_policy.h"
 #include "util/coding.h"
-#include "util/perf_context_imp.h"
 #include "util/string_util.h"

 namespace rocksdb {
@@ -220,14 +219,7 @@ bool BlockBasedFilterBlockReader::MayMatch(const Slice& entry,
    uint32_t limit = DecodeFixed32(offset_ + index * 4 + 4);
    if (start <= limit && limit <= (uint32_t)(offset_ - data_)) {
      Slice filter = Slice(data_ + start, limit - start);
-      bool const may_match = policy_->KeyMayMatch(entry, filter);
-      if (may_match) {
-        PERF_COUNTER_ADD(bloom_sst_hit_count, 1);
-        return true;
-      } else {
-        PERF_COUNTER_ADD(bloom_sst_miss_count, 1);
-        return false;
-      }
+      return policy_->KeyMayMatch(entry, filter);
    } else if (start == limit) {
      // Empty filters do not match any entries
      return false;
@@ -42,22 +42,13 @@ BlockBasedTableFactory::BlockBasedTableFactory(
 }

 Status BlockBasedTableFactory::NewTableReader(
-    const TableReaderOptions& table_reader_options,
-    unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
-    unique_ptr<TableReader>* table_reader) const {
-  return NewTableReader(table_reader_options, std::move(file), file_size,
-                        table_reader,
-                        /*prefetch_index_and_filter=*/true);
-}
-
-Status BlockBasedTableFactory::NewTableReader(
-    const TableReaderOptions& table_reader_options,
+    const ImmutableCFOptions& ioptions, const EnvOptions& soptions,
+    const InternalKeyComparator& internal_comparator,
    unique_ptr<RandomAccessFileReader>&& file, uint64_t file_size,
    unique_ptr<TableReader>* table_reader, const bool prefetch_enabled) const {
-  return BlockBasedTable::Open(
-      table_reader_options.ioptions, table_reader_options.env_options,
-      table_options_, table_reader_options.internal_comparator, std::move(file),
-      file_size, table_reader, prefetch_enabled);
+  return BlockBasedTable::Open(ioptions, soptions, table_options_,
+                               internal_comparator, std::move(file), file_size,
+                               table_reader, prefetch_enabled);
 }

 TableBuilder* BlockBasedTableFactory::NewTableBuilder(
@@ -33,14 +33,22 @@ class BlockBasedTableFactory : public TableFactory {

  const char* Name() const override { return "BlockBasedTable"; }

-  Status NewTableReader(const TableReaderOptions& table_reader_options,
+  Status NewTableReader(const ImmutableCFOptions& ioptions,
+                        const EnvOptions& soptions,
+                        const InternalKeyComparator& internal_comparator,
                        unique_ptr<RandomAccessFileReader>&& file,
                        uint64_t file_size,
-                        unique_ptr<TableReader>* table_reader) const override;
+                        unique_ptr<TableReader>* table_reader) const override {
+    return NewTableReader(ioptions, soptions, internal_comparator,
+                          std::move(file), file_size, table_reader,
+                          /*prefetch_index_and_filter=*/true);
+  }

  // This is a variant of virtual member function NewTableReader function with
  // added capability to disable pre-fetching of blocks on BlockBasedTable::Open
-  Status NewTableReader(const TableReaderOptions& table_reader_options,
+  Status NewTableReader(const ImmutableCFOptions& ioptions,
+                        const EnvOptions& soptions,
+                        const InternalKeyComparator& internal_comparator,
                        unique_ptr<RandomAccessFileReader>&& file,
                        uint64_t file_size,
                        unique_ptr<TableReader>* table_reader,
@@ -117,9 +117,6 @@ Cache::Handle* GetEntryFromCache(Cache* block_cache, const Slice& key,
    PERF_COUNTER_ADD(block_cache_hit_count, 1);
    // overall cache hit
    RecordTick(statistics, BLOCK_CACHE_HIT);
-    // total bytes read from cache
-    RecordTick(statistics, BLOCK_CACHE_BYTES_READ,
-               block_cache->GetUsage(cache_handle));
    // block-type specific cache hit
    RecordTick(statistics, block_cache_hit_ticker);
  } else {
@@ -798,8 +795,6 @@ Status BlockBasedTable::PutDataBlockToCache(
                                              block->value->usable_size(),
                                              &DeleteCachedEntry<Block>);
    RecordTick(statistics, BLOCK_CACHE_ADD);
-    RecordTick(statistics, BLOCK_CACHE_BYTES_WRITE,
-               block->value->usable_size());
    assert(reinterpret_cast<Block*>(block_cache->Value(block->cache_handle)) ==
           block->value);
  }
@@ -891,7 +886,6 @@ BlockBasedTable::CachableEntry<FilterBlockReader> BlockBasedTable::GetFilter(
      cache_handle = block_cache->Insert(key, filter, filter_size,
                                         &DeleteCachedEntry<FilterBlockReader>);
      RecordTick(statistics, BLOCK_CACHE_ADD);
-      RecordTick(statistics, BLOCK_CACHE_BYTES_WRITE, filter_size);
    }
  }

@@ -950,8 +944,6 @@ Iterator* BlockBasedTable::NewIndexIterator(const ReadOptions& read_options,
        block_cache->Insert(key, index_reader, index_reader->usable_size(),
                            &DeleteCachedEntry<IndexReader>);
    RecordTick(statistics, BLOCK_CACHE_ADD);
-    RecordTick(statistics, BLOCK_CACHE_BYTES_WRITE,
-               index_reader->usable_size());
  }

  assert(cache_handle);
Author	SHA1	Message	Date
Yueh-Hsuan Chiang	a9a6a2f434	Merge pull request #764 from dmittendorf/fix-java-static-packaging Fix crossbuild jar packaging Conflicts: Makefile java/crossbuild/build-linux-centos.sh	2015-10-15 15:05:20 -07:00
Yueh-Hsuan Chiang	be5cb800ef	Merge pull request #759 from jwlent55/statically-load-compression-libraries Ensure that the compression libraries are statically linked into dyna…	2015-10-15 15:04:53 -07:00
agiardullo	0fb70a800e	Return MergeInProgress when fetching from transactions or WBWI with overwrite_key Summary: WriteBatchWithIndex::GetFromBatchAndDB only works correctly for overwrite_key=false. Transactions use overwrite_key=true (since WriteBatchWithIndex::GetIteratorWithBase only works when overwrite_key=true). So currently, Transactions could return incorrectly merged results when calling Get/GetForUpdate(). Until a permanent fix can be put in place, Transaction::Get[ForUpdate] and WriteBatchWithIndex::GetFromBatch[AndDB] will now return MergeInProgress if the most recent write to a key in the batch is a Merge. Test Plan: more tests Reviewers: sdong, yhchiang, rven, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D47817	2015-09-30 11:45:32 -07:00
agiardullo	8dbf5a39bd	Fix accidental object copy in transactions Summary: Should have used auto& instead of auto. Also needed to change the code a bit due to const correctness. Test Plan: unit tests Reviewers: sdong, igor, yoshinorim, yhchiang Reviewed By: yhchiang Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D47787	2015-09-30 11:37:27 -07:00
Yueh-Hsuan Chiang	0798a49a28	Make CompactionJobStatsTest.UniversalCompactionTest more robust Summary: CompactionJobStatsTest.UniversalCompactionTest assumes compaction kicks in when the number of L0 files equals to the compaction trigger. However, in some case, the compaction might not catch up the write speed and thus compaction might not kick in until the number of L0 files is GREATER than the compaction trigger. This patch tries to fix this corner case by making the Put thread wait for a potential compaction whenever it flushes. Test Plan: ./compaction_job_stats_test Reviewers: sdong, anthony, IslamAbdelRahman, igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D47589	2015-09-30 11:15:35 -07:00
Islam AbdelRahman	555f3be85b	Clear SyncPoint Trace in DeleteSchedulerTests Summary: DeleteSchedulerTests is running the same test with different rates, After the first iteraton sync points become useless because ClearTrace was not being called Test Plan: Run the test Reviewers: sdong, yhchiang, igor Reviewed By: igor Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D47709	2015-09-29 22:29:09 -07:00
Yueh-Hsuan Chiang	e29a73b89e	Fixed a tsan warning in db_stress.cc Summary: Fixed the following tsan warning in db_stress.cc WARNING: ThreadSanitizer: data race (pid=3163194) Read of size 8 at 0x7fd1797cb518 by thread T32: #0 VerifyDb tools/db_stress.cc:1731 (db_stress+0x000000040674) #1 rocksdb::StressTest::ThreadBody(void) tools/db_stress.cc:1191 (db_stress+0x0000000625a9) #2 StartThreadWrapper util/env_posix.cc:1648 (db_stress+0x00000028bbbd) Previous write of size 8 at 0x7fd1797cb518 by thread T31: #0 VerifyDb tools/db_stress.cc:1726 (db_stress+0x00000004072a) #1 rocksdb::StressTest::ThreadBody(void) tools/db_stress.cc:1191 (db_stress+0x0000000625a9) #2 StartThreadWrapper util/env_posix.cc:1648 (db_stress+0x00000028bbbd) The cause is that in VerifyDb(), the static local const variable long max_key can be read and written at the same time. This patch fixed it by making it non-static. Test Plan: db_stress Reviewers: igor, sdong, IslamAbdelRahman, anthony Reviewed By: anthony Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D47703	2015-09-29 22:28:45 -07:00
Yueh-Hsuan Chiang	0b7dd9131f	Fixed a bug which causes rocksdb.flush.write.bytes stat is always zero Summary: Fixed a bug which causes rocksdb.flush.write.bytes stat is always zero Test Plan: augment existing db_test Reviewers: sdong, anthony, IslamAbdelRahman, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D47595	2015-09-28 18:39:35 -07:00
sdong	8dcaa16b54	RandomAccessFileReader should not inherit RandomAccessFile Summary: RandomAccessFileReader unnecessarily inherited RandomAccessFile, which can introduce unnecessarily extra costs. Remove it. Test Plan: Run all existing tests Reviewers: yhchiang, anthony, igor, kradhakrishnan, rven, IslamAbdelRahman Reviewed By: IslamAbdelRahman Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D47409	2015-09-28 18:38:54 -07:00
Andres Noetzli	1d2d22e738	Fix clang-format on Travis Summary: Commit `1b598213aa` does not quite work as intended. This patch fixes Travis by adding clang-format to the packages list and excluding clang-format from the OSX build. In addition the build does not stop when bad formatting is detected but the build still fails. Test Plan: See https://travis-ci.org/facebook/rocksdb/builds/81037125 Reviewers: igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D47223	2015-09-28 18:37:54 -07:00
Andres Noetzli	fbd60cf3c4	Check formatting in Travis Summary: Check code formatting before running tests. Goal is to make it easier to deal with external pull requests. Test Plan: See https://travis-ci.org/facebook/rocksdb/builds/80952963 Reviewers: igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D47211	2015-09-28 18:37:38 -07:00
Andres Noetzli	c5024c03b6	Fix gflags build in Travis script Summary: There were two minor issues with `9566342d28`: - The install command was doing `cmake` instead of `cmake .` - `LIBRARY_PATH` was not set Fix both issues. Test Plan: Run tests on Travis: https://travis-ci.org/facebook/rocksdb/jobs/80699627 (bloom_test works now). Reviewers: igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D47103	2015-09-28 18:37:08 -07:00
Andres Noetzli	074154582a	Fix wrong constants in db_test_util Summary: Fix two constants in WaitFor() that multiply a value with 000 instead of 1000. Test Plan: make clean all check Reviewers: rven, anthony, yhchiang, aekmekji, sdong, MarkCallaghan, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D47091	2015-09-28 18:36:54 -07:00
Andres Noetzli	3967a4f0d1	Build gflags from source for Travis Summary: Unfortunately it is not easily possible to install gflags as a package on Travis. Change the Travis script to download and build gflags. This patch avoids "sudo make install" so that we can continue to use Travis' container-based infrastructure. Test Plan: Run tests on Travis. Reviewers: sdong, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D47037	2015-09-28 18:36:02 -07:00
Andres Noetzli	9a4d86b3da	Initialize variable to avoid warning Summary: RocksDB debug version failed to build under gcc-4.8.1 on sandcastle with the following error: ``` db/db_compaction_filter_test.cc:570:33: error: â€˜snapshotâ€™ may be used uninitialized in this function [-Werror=maybe-uninitialized] ``` Test Plan: make db_compaction_filter_test && ./db_compaction_filter_test Reviewers: rven, anthony, yhchiang, aekmekji, igor, sdong Reviewed By: igor, sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D46725	2015-09-16 15:55:30 -07:00
Igor Canadi	0833b75104	LogAndApply() should fail if the column family has been dropped Summary: This patch finally fixes the ColumnFamilyTest.ReadDroppedColumnFamily test. The test has been failing very sporadically and it was hard to repro. However, I managed to write a new tests that reproes the failure deterministically. Here's what happens: 1. We start the flush for the column family 2. We check if the column family was dropped here: https://github.com/facebook/rocksdb/blob/a3fc49bfddcdb1ff29409aacd06c04df56c7a1d7/db/flush_job.cc#L149 3. This check goes through, ends up in InstallMemtableFlushResults() and it goes into LogAndApply() 4. At about this time, we start dropping the column family. Dropping the column family process gets to LogAndApply() at about the same time as LogAndApply() from flush process 5. Drop column family goes through LogAndApply() first, marking the column family as dropped. 6. Flush process gets woken up and gets a chance to write to the MANIFEST. However, this is where it gets stuck: https://github.com/facebook/rocksdb/blob/a3fc49bfddcdb1ff29409aacd06c04df56c7a1d7/db/version_set.cc#L1975 7. We see that the column family was dropped, so there is no need to write to the MANIFEST. We return OK. 8. Flush gets OK back from LogAndApply() and it deletes the memtable, thinking that the data is now safely persisted to sst file. The fix is pretty simple. Instead of OK, we return ShutdownInProgress. This is not really true, but we have been using this status code to also mean "this operation was canceled because the column family has been dropped". The fix is only one LOC. All other code is related to tests. I added a new test that reproes the failure. I also moved SleepingBackgroundTask to util/testutil.h (because I needed it in column_family_test for my new test). There's plenty of other places where we reimplement SleepingBackgroundTask, but I'll address that in a separate commit. Test Plan: 1. new test 2. make check 3. Make sure the ColumnFamilyTest.ReadDroppedColumnFamily doesn't fail on Travis: https://travis-ci.org/facebook/rocksdb/jobs/79952386 Reviewers: yhchiang, anthony, IslamAbdelRahman, kradhakrishnan, rven, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D46773	2015-09-15 11:30:44 -07:00
Dmitry Marakasov	664acfa087	Fix printf format for size_t	2015-09-15 11:00:52 -07:00
Dmitry Marakasov	66a007f832	Fix `integer overflow in expression' error	2015-09-15 11:00:45 -07:00
sdong	5559488e92	DBImpl::FindObsoleteFiles() shouldn't release mutex between getting min_pending_output and scanning files Summary: Releasing mutex between getting min_pending_output and scanning files may cause min_pending_output to be max but some non-final files are found in file scanning, ending up with deleting wrong files. As a recent regression, mutex can be released while waiting for log sync. We move it to after file scanning. Test Plan: Run all existing tests. Don't think it is easy to write a unit test. Maybe we should find a way to assert lock not released so that we can have some test verification for similar cases. Reviewers: igor, anthony, IslamAbdelRahman, kradhakrishnan, yhchiang, kolmike, rven Reviewed By: rven Subscribers: leveldb, dhruba Differential Revision: https://reviews.facebook.net/D46899	2015-09-15 10:57:00 -07:00
Islam AbdelRahman	d0ce28bb27	Fix compaction_job_stats under ROCKSDB_LITE Summary: Fix compaction_job_stats under ROCKSDB_LITE Test Plan: compile using ROCKSDB_LITE Reviewers: yhchiang, igor, sdong Reviewed By: sdong Subscribers: dhruba Differential Revision: https://reviews.facebook.net/D46887	2015-09-15 10:50:14 -07:00
Andres Noetzli	248802038b	Relax asserts in arena_test Summary: Commit `c67d206898` did not fix all test conditions which use Arena::MemoryAllocatedBytes() (see Travis failure https://travis-ci.org/facebook/rocksdb/jobs/79957700). The assumption of that commit was that aligned allocations do not call Arena::AllocateNewBlock(), so malloc_usable_block_size() would not be used for Arena::MemoryAllocatedBytes(). However, there is a code path where Arena::AllocateAligned() calls AllocateFallback() which in turn calls Arena::AllocateNewBlock(), so Arena::MemoryAllocatedBytes() may return a greater value than expected even for aligned requests. Test Plan: make arena_test && ./arena_test Reviewers: rven, anthony, yhchiang, aekmekji, igor, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D46869	2015-09-15 10:50:09 -07:00
agiardullo	af06d1a871	Transactions: Release Locks when rolling back to a savepoint Summary: Transaction::RollbackToSavePoint() will now release any locks that were taken since the previous SavePoint. To do this cleanly, I moved tracked_keys_ management into TransactionBase. Test Plan: New Transaction test. Reviewers: igor, rven, sdong Reviewed By: sdong Subscribers: dhruba, spetrunia, leveldb Differential Revision: https://reviews.facebook.net/D46761	2015-09-15 10:50:01 -07:00
Andres Noetzli	5266ed3cc5	Fixed arena_test failure due to malloc_usable_size() Summary: ArenaTest.MemoryAllocatedBytes on Travis failed: https://travis-ci.org/facebook/rocksdb/jobs/79887849 . This is probably due to malloc_usable_size() returning a value greater than the requested size. From the man page: The value returned by malloc_usable_size() may be greater than the requested size of the allocation because of alignment and minimum size constraints. Although the excess bytes can be overwritten by the application without ill effects, this is not good programming practice: the number of excess bytes in an allocation depends on the underlying implementation. Test Plan: make arena_test && ./arena_test Reviewers: rven, anthony, yhchiang, aekmekji, sdong, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D46743	2015-09-15 10:49:53 -07:00
Andres Noetzli	7584091c7f	Fixed bug in compaction iterator Summary: During the refactoring, the condition that makes sure that compaction filters are only applied to records newer than the latest snapshot got butchered. This patch fixes the condition and adds a test case. Test Plan: make db_compaction_filter_test && ./db_compaction_filter_test Reviewers: rven, anthony, yhchiang, sdong, aekmekji, igor Reviewed By: igor Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D46707	2015-09-15 10:49:37 -07:00